source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_irene-amd.ksh @ 1624

Last change on this file since 1624 was 1624, checked in by jgipsl, 4 months ago

Added possibility to declare a restart file as optional which means that libIGCM want stop if the file is not found. Only a warning will be written.

For example, in the following case, if the routing_restart file is not available when starting the model or if it is not produced, libIGCM will not stop. But if the file sechiba_rest is not available, libIGCM will stop as usual.

[RestartFiles]
List=   (sechiba_rest_out.nc, sechiba_rest.nc, sechiba_rest_in.nc) ,\
        (routing_restart.nc, routing_restart.nc, routing_start.nc, OPTIONAL)
  • Read optional argument OPTIONAL on the line where the restart file is declared.
  • Each time when a restart file is not found for coping in or out, if the argument OPTIONAL is set, then just continue. If not set, the file is considered as MANDATORY and libIGCM will exist if it is missing as before.
  • on irene-amd and jean-zay, corrected a bug for the case in config.card DataProject?=DEFAULT


File size: 66.5 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision:: 1512                                     $ Revision of last commit
7# $Author:: acosce                                     $ Author of last commit
8# $Date:: 2020-01-14 18:52:20 +0100 (Tue, 14 Jan 2020) $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Irene-amd
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40if ([ X${DRYRUN} != X ] && [ ${DRYRUN} = 4 ]) ; then
41  typeset -r DRYRUN=0
42  typeset -r DRYRUN_DEBUG=4
43else
44  typeset -r DRYRUN=${DRYRUN:=0}
45  typeset -r DRYRUN_DEBUG=0
46fi
47
48# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
49# -------------------------------------------------------------------------------------
50# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
51# |          |  Cp/Exe/param/files |            |         |                           |
52# |          |  Chmod Qsub         |            |         |                           |
53# -------------------------------------------------------------------------------------
54# |    0     |       yes           |    yes     |  yes    |      yes                  |
55# -------------------------------------------------------------------------------------
56# |    1     |       yes           |    yes     |  yes    |      no                   |
57# -------------------------------------------------------------------------------------
58# |    2     |       yes           |    yes     |  no     |      no                   |
59# -------------------------------------------------------------------------------------
60# |    3     |       yes           |    no      |  no     |      no                   |
61# -------------------------------------------------------------------------------------
62# |    4     |       yes           |    yes     |  creation of RUN_DIR and Job_debug  |
63# -------------------------------------------------------------------------------------
64
65
66#=====================================================
67# Global Variables :
68#=====================================================
69# Language : "fr" or "en"
70typeset -r MYLANG="fr"
71
72#=====================================================
73# Host user names project maxCpuTime
74# $hostname ou hostname
75typeset HOST=${HOST:=$( hostname )}
76# $username ou whoami
77typeset LOGIN=${LOGIN:=$( whoami )}
78# $hostname of the MASTER job
79typeset MASTER=irene-amd
80# add default project on irene-amd
81typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=::default_project::} | cut -d@ -f1 )
82# jobWarningDelay in seconds
83typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
84
85#D-
86#D-#==================================================
87#D-Program used in libIGCM
88#D-#==================================================
89
90# Submit command
91typeset SUBMIT=${SUBMIT:=ccc_msub}
92# rsync with path
93typeset -r RSYNC=/usr/bin/rsync
94# RSYNC_opt args to rsync
95typeset -r RSYNC_opt="-va"
96# ie storage filesystem
97typeset -r STOREHOST=${MASTER}
98typeset -r REMOTE_RSYNC=/usr/bin/rsync
99
100#====================================================
101# Set environment tools (ferret, nco, cdo, rebuild, ...)
102#====================================================
103typeset CCCOS=${CCCOS:=$( ccc_os )}
104
105if [ X${TaskType} = Xcomputing ] ; then
106  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
107else
108  module purge > /dev/null 2>&1
109  if [ X${CCCOS} = X"Rhel_8__x86_64" ] ; then
110      . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
111  else
112      . $( ccc_home -u igcmg)/MachineEnvironment/irene-amd_prev/env_irene-amd > /dev/null 2>&1
113      . $( ccc_home -u igcmg)/MachineEnvironment/irene-amd_prev/env_atlas_irene-amd > /dev/null 2>&1
114  fi
115
116  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
117  export UVCDAT_ANONYMOUS_LOG=FALSE
118fi
119
120# FYI
121[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
122[ ! X${TaskType} = Xchecking ] && module list
123
124# For AddNoise and AddPertu3DOCE
125export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/bin
126
127# For rebuild
128export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl/bin
129
130#====================================================
131# Host specific DIRECTORIES
132#====================================================
133
134# ============ CESIUM START ============ #
135
136#====================================================
137#- Mirror libIGCM from titane to cesium if needed
138#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
139#if [ ! ${ROOTSYS} = "home" ] ; then
140#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
141#else
142#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
143#fi
144
145#====================================================
146#- libIGCM_POST
147#if ( ${MirrorlibIGCM} ) ; then
148#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
149#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
150#else
151#  typeset -r libIGCM_POST=${libIGCM}
152#fi
153
154# ============ CESIUM  END  ============ #
155
156#====================================================
157#- MirrorlibIGCM for frontend
158typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
159
160#====================================================
161#- libIGCM_POST for frontend
162typeset -r libIGCM_POST=${libIGCM}
163
164#====================================================
165#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
166typeset -r R_EXE="${MODIPSL}/bin"
167
168#====================================================
169#- SUBMIT_DIR : submission dir
170typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
171
172#====================================================
173#- IN
174typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
175
176#====================================================
177#- RUN_DIR_PATH : Temporary working directory (=> TMP)
178typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
179
180#====================================================
181#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
182typeset -r OUTCOMMAND_PATH=/tmp
183
184#====================================================
185#- HOST_MPIRUN_COMMAND
186typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
187
188#====================================================
189#- Max number of arguments passed to nco operator or demigration command
190UNIX_MAX_LIMIT=360
191
192#====================================================
193#- set PackDefault to true on irene-amd
194PackDefault=true
195
196#====================================================
197#- Default number of MPI task for IPSL coupled model
198#- required for backward compatibility
199#-
200DEFAULT_NUM_PROC_OCE=5
201DEFAULT_NUM_PROC_CPL=1
202(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
203
204#====================================================
205#- Number of computing cores per node
206#-
207NB_CORE_PER_NODE=128
208
209#D-#==================================================
210#D-function IGCM_sys_defineArchives
211#D-* Purpose:
212#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
213#D-* Define ARCHIVE : Dedicated to large files
214#D-* Define STORAGE : Dedicated to small/medium files
215#D-* Define R_OUT   : Output tree located on ARCHIVE
216#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
217#D-* Define R_BUF   : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
218#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
219#D-* Examples:
220#D-
221function IGCM_sys_defineArchives {
222  IGCM_debug_PushStack "IGCM_sys_defineArchives"
223
224  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
225  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = XDEFAULT ]; then
226    # Default option: Change the dfldatadir according to the project used for submission
227    # The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
228    if [ X"$( echo ${PROJECT} | grep cmip6 )" != "X" ] ; then
229      module switch dfldatadir dfldatadir/gencmip6
230      IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
231    else
232      module switch dfldatadir dfldatadir/${PROJECT}
233      IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
234    fi
235  else
236    # Use the project set by the variable DataProject in section UserChoices in config.card
237    module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
238    IGCM_debug_Print 1 "Load dfldatadir for DataProject red in config.card : ${config_UserChoices_DataProject}"
239  fi
240
241 #====================================================
242 #- RUN_DIR_PATH : Temporary working directory (=> TMP)
243 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
244
245  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
246    #====================================================
247    #- ARCHIVE (dedicated to large files)
248    ARCHIVE=${config_UserChoices_ARCHIVE}
249  else
250    #====================================================
251    #- ARCHIVE (dedicated to large files)
252    ARCHIVE=${CCCSTOREDIR}
253  fi
254
255  if [ ! X${config_UserChoices_STORAGE} = X ]; then
256    #====================================================
257    #- STORAGE (dedicated to small/medium files)
258    STORAGE=${config_UserChoices_STORAGE}
259  else
260    #====================================================
261    #- STORAGE (dedicated to small/medium files)
262    STORAGE=${CCCWORKDIR}
263  fi
264
265  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
266    #====================================================
267    #- R_OUT
268    R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
269
270    #====================================================
271    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
272    R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
273
274    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
275
276  else
277    #====================================================
278    #- R_OUT
279    R_OUT=${ARCHIVE}/IGCM_OUT
280
281    #====================================================
282    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
283    R_FIG=${STORAGE}/IGCM_OUT
284  fi
285
286  #====================================================
287  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
288  CMIP6_BUF=${STORAGE}/IGCM_OUT
289
290  #====================================================
291  #- R_BUF (ONLY FOR double copy an scratch)
292  R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
293
294  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
295  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
296  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
297
298  IGCM_debug_PopStack "IGCM_sys_defineArchives"
299}
300
301#D-#==================================================
302#D-function IGCM_sys_RshArchive
303#D-* Purpose: Archive rsh command
304#D-* Examples:
305#D-
306function IGCM_sys_RshArchive {
307  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
308  /bin/ksh <<-EOF
309    ${@}
310EOF
311  status=$?
312  if [ ${status} -gt 0 ] ; then
313    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
314    IGCM_debug_Exit "IGCM_sys_RshArchive"
315  fi
316  IGCM_debug_PopStack "IGCM_sys_RshArchive"
317}
318
319#D-#==================================================
320#D-function IGCM_sys_RshArchive_NoError
321#D-* Purpose: Archive rsh command, without error
322#D-*          used only in monitoring.job
323#D-* Examples:
324#D-
325function IGCM_sys_RshArchive_NoError {
326  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
327  /bin/ksh <<-EOF
328    ${@} 2> /dev/null
329EOF
330  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
331}
332
333#D-#==================================================
334#D-function IGCM_sys_MkdirArchive
335#D-* Purpose: Mkdir on Archive
336#D-* Examples:
337#D-
338function IGCM_sys_MkdirArchive {
339  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
340  if ( $DEBUG_sys ) ; then
341    echo "IGCM_sys_MkdirArchive :" $@
342  fi
343  #- creation de repertoire sur le serveur fichier
344  if [ ! -d ${1} ]; then
345    \mkdir -p $1
346    status=$?
347
348    if [ ${status} -gt 0 ] ; then
349      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
350      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
351    fi
352  fi
353  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
354}
355
356#D-#==================================================
357#D-function IGCM_sys_TestDirArchive
358#D-* Purpose: Test Directory that must exists on Archive
359#D-* Examples:
360#D-
361function IGCM_sys_TestDirArchive {
362  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
363  if ( $DEBUG_sys ) ; then
364    echo "IGCM_sys_TestDirArchive :" $@
365  fi
366  typeset ExistFlag
367  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
368  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
369  return ${ExistFlag}
370}
371
372#D-#==================================================
373#D-function IGCM_sys_IsFileArchived
374#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
375#D-* Examples:
376#D-
377function IGCM_sys_IsFileArchived {
378  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
379  if ( $DEBUG_sys ) ; then
380    echo "IGCM_sys_IsFileArchived :" $@
381  fi
382  typeset IsArchivedFlag
383  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
384  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
385
386  return ${IsArchivedFlag}
387}
388
389#D-#==================================================
390#D-function IGCM_sys_TestFileArchive
391#D-* Purpose: Test file that must NOT EXISTS on Archive
392#D-* Examples:
393#D-
394function IGCM_sys_TestFileArchive {
395  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
396  typeset ExistFlag
397  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
398  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
399
400  return ${ExistFlag}
401}
402
403#D-#==================================================
404#D-function IGCM_sys_CountFileArchive
405#D-* Purpose: Count files on Archive filesystem
406#D-* Examples:
407#D-
408function IGCM_sys_CountFileArchive {
409  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
410  ls ${@} 2>/dev/null | wc -l
411  if [ $? -gt 0 ] ; then
412    echo "IGCM_sys_CountFileArchive : erreur."
413  fi
414  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
415}
416
417#D-#==================================================
418#D-function IGCM_sys_Tree
419#D-* Purpose: Tree directories with files on ${ARCHIVE}
420#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
421#D-
422function IGCM_sys_Tree {
423  IGCM_debug_PushStack "IGCM_sys_Tree" $@
424  if ( $DEBUG_sys ) ; then
425    echo "IGCM_sys_Tree :" $@
426  fi
427
428  \ls -lR ${@}
429
430  IGCM_debug_PopStack "IGCM_sys_Tree"
431}
432
433#D-#==================================================
434#D-function IGCM_sys_Qsub
435#D-* Purpose: Qsub new job
436#D-* Examples:
437#D-
438function IGCM_sys_Qsub {
439  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
440  if ( $DEBUG_sys ) ; then
441    echo "IGCM_sys_Qsub :" $@
442  fi
443  typeset options status
444  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
445
446  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
447  status=$?
448
449  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
450  if [ ${status} -gt 0 ] ; then
451    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
452    IGCM_debug_Exit "IGCM_sys_Qsub"
453  else
454    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
455  fi
456  IGCM_debug_PopStack "IGCM_sys_Qsub"
457}
458
459#D-#==================================================
460#D-function IGCM_sys_QsubPost
461#D-* Purpose: Qsub new job on scalaire
462#D-* Examples:
463#D-
464function IGCM_sys_QsubPost {
465  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
466  if ( $DEBUG_sys ) ; then
467    echo "IGCM_sys_QsubPost :" $@
468  fi
469  typeset options status
470  options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
471
472  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
473  status=$?
474
475  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
476  if [ ${status} -gt 0 ] ; then
477    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
478    IGCM_debug_Exit "IGCM_sys_QsubPost"
479  else
480    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
481  fi
482  IGCM_debug_PopStack "IGCM_sys_QsubPost"
483}
484
485#D-*************************
486#D- File transfer functions
487#D-*************************
488#D-
489
490#D-#==================================================
491#D-function IGCM_sys_RmRunDir
492#D-* Purpose: rm tmpdir (dummy function most of the time batch
493#D-                      scheduler will do the job)
494#D-* Examples:
495#D-
496function IGCM_sys_RmRunDir {
497  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
498  if ( $DEBUG_sys ) ; then
499    echo "IGCM_sys_RmRunDir :" $@
500  fi
501
502  typeset status
503
504  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
505  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
506  status=$?
507
508  if [ ${status} -gt 0 ] ; then
509    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
510    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
511    IGCM_debug_Exit "IGCM_sys_RmRunDir"
512  else
513    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
514  fi
515  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
516}
517
518#D-#==================================================
519#D-function IGCM_sys_Put_Dir
520#D-* Purpose: Copy a complete directory on $(ARCHIVE)
521#D-* Examples:
522#D-
523function IGCM_sys_Put_Dir {
524  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
525  if ( $DEBUG_sys ) ; then
526    echo "IGCM_sys_Put_Dir :" $@
527  fi
528  if [ $DRYRUN = 0 ]; then
529    if [ ! -d ${1} ] ; then
530      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
531      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
532      return
533    fi
534
535    typeset status
536
537    # Only if we use rsync
538    #IGCM_sys_TestDirArchive $( dirname $2 )
539    #
540    #USUAL WAY
541    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
542    status=$?
543
544    if [ ${status} -gt 0 ] ; then
545      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
546      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
547      IGCM_debug_Exit "IGCM_sys_Put_Dir"
548    else
549      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
550    fi
551  fi
552  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
553}
554
555#D-#==================================================
556#D-function IGCM_sys_Get_Dir
557#D-* Purpose: Copy a complete directory from ${ARCHIVE}
558#D-* Examples:
559#D-
560function IGCM_sys_Get_Dir {
561  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
562  if ( $DEBUG_sys ) ; then
563    echo "IGCM_sys_Get_Dir :" $@
564  fi
565  if [ $DRYRUN = 0 ]; then
566    typeset NB_ESSAI DELAI status i
567    # number of tentative
568    NB_ESSAI=3
569    # time delay between tentative
570    DELAI=2
571
572    #
573    # USUAL WAY
574    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
575    ccc_hsm get -r $1
576
577    i=0
578    while [ $i -lt $NB_ESSAI ] ; do
579      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
580      status=$?
581      if [ ${status} -gt 0 ] ; then
582        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
583        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
584        sleep $DELAI
585      else
586        break
587      fi
588      (( i = i + 1 ))
589    done
590
591    if [ ${status} -gt 0 ] ; then
592      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
593      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
594      IGCM_debug_Exit "IGCM_sys_Get_Dir"
595    else
596      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
597    fi
598  fi
599  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
600}
601
602#D-#==================================================
603#D-function IGCM_sys_Put_Rest
604#D-* Purpose: Put computied restarts on ${ARCHIVE}.
605#D-           File and target directory must exist.
606#D-* Examples:
607#D-
608function IGCM_sys_Put_Rest {
609  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
610  if ( $DEBUG_sys ) ; then
611    echo "IGCM_sys_Put_Rest :" $@
612  fi
613  if [ $DRYRUN = 0 ]; then
614    if [ ! -f ${1} ] ; then
615        # The file is missing     
616        if [ X${3} = X'OPTIONAL' ] ; then
617            echo "WARINING : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
618            IGCM_debug_Print 1 "IGCM_sys_Put_Rest: Optional restart file ${1} is missing, now continue for the next"
619        else
620            echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
621            IGCM_debug_Exit "IGCM_sys_Put_Rest"
622        fi
623
624    else
625      # The file exist
626
627    typeset status
628    #
629    # USUAL WAY
630    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
631    status=$?
632
633#       #RSYNC WITH NETWORK SSH CALL
634#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
635#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
636
637#       #RSYNC WITH NFS USE
638#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
639#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
640
641#       status=$?
642#       IGCM_sys_Rsync_out $status
643
644#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
645#       (( status=status+$? ))
646
647    if [ ${status} -gt 0 ] ; then
648      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
649      [ -f ${1} ] && ls -l ${1}
650      [ -f ${2} ] && ls -l ${2}
651      [ -f ${2}/${1} ] && ls -l ${2}/${1}
652      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
653      IGCM_debug_Exit "IGCM_sys_Put_Rest"
654    else
655
656      if [ X${JobType} = XRUN ] ; then
657        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
658        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
659      fi
660
661      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
662    fi
663    fi
664  fi
665  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
666}
667
668#D-#==================================================
669#D-function IGCM_sys_Put_Out
670#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
671#D-* Examples:
672#D-
673function IGCM_sys_Put_Out {
674  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
675  if ( $DEBUG_sys ) ; then
676    echo "IGCM_sys_Put_Out :" $@
677  fi
678
679  typeset NB_ESSAI DELAI status i exist skip
680  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
681
682  # number of tentative
683  NB_ESSAI=3
684  # time delay between tentative
685  DELAI=2
686
687  if [ $DRYRUN = 0 ]; then
688    if [ ! -f ${1} ] ; then
689      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
690      IGCM_debug_PopStack "IGCM_sys_Put_Out"
691      return 1
692    fi
693    #
694    IGCM_sys_MkdirArchive $( dirname $2 )
695    #
696    exist=false
697    skip=false
698    if [ -f $2 ] ; then
699      IGCM_debug_Print 1 "$2 already exist"
700      ccc_hsm get $2
701      exist=true
702      if [ "X$( diff $1 $2 )" = X ] ; then
703        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
704        skip=true
705      else
706        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
707        skip=false
708      fi
709    fi
710    #
711    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
712      IGCM_sys_Chmod u+w $2
713    fi
714
715    if [ X${skip} = Xfalse ] ; then
716      i=0
717      while [ $i -lt $NB_ESSAI ] ; do
718        # Identify file system
719        fileDeviceNumberInHex=$( stat -c %d $1 )
720        status=$?
721        if [ ${status} -gt 0 ] ; then
722          IGCM_debug_Exit "IGCM_sys_Put_Out"
723        fi
724        # Identify file system
725        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
726        status=$?
727        if [ ${status} -gt 0 ] ; then
728          IGCM_debug_Exit "IGCM_sys_Put_Out"
729        fi
730
731        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
732          # They are not on the same device. USUAL WAY
733          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
734          status=$?
735        else
736          # They are on the same device. NOT SO USUAL WAY
737          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
738          status=$?
739        fi
740        if [ ${status} -gt 0 ]; then
741          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
742          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
743          [ -f ${1} ] && ls -l ${1}
744          [ -f ${2} ] && ls -l ${2}
745          [ -f ${2}/${1} ] && ls -l ${2}/${1}
746          sleep $DELAI
747        else
748          break
749        fi
750        (( i = i + 1 ))
751      done
752    fi
753
754#       #RSYNC WITH NETWORK SSH CALL
755#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
756#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
757
758#       #RSYNC WITH NFS USE
759#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
760#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
761
762#       status=$?
763#       IGCM_sys_Rsync_out $status
764
765#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
766#       (( status=status+$? ))
767
768    if [ ${status} -gt 0 ] ; then
769      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
770      [ -f ${1} ] && ls -l ${1}
771      [ -f ${2} ] && ls -l ${2}
772      [ -f ${2}/${1} ] && ls -l ${2}/${1}
773      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
774      IGCM_debug_Exit "IGCM_sys_Put_Out"
775    else
776
777      if [ X${JobType} = XRUN ] ; then
778        if [ X${3} = X ] ; then
779          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
780          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
781        fi
782      fi
783
784      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
785    fi
786  fi
787  IGCM_debug_PopStack "IGCM_sys_Put_Out"
788  return 0
789}
790
791#D-#==================================================
792#D-function IGCM_sys_Get
793#D-* Purpose: Get a file from ${ARCHIVE}
794#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
795#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
796function IGCM_sys_Get {
797  IGCM_debug_PushStack "IGCM_sys_Get" $@
798
799  typeset DEST dm_liste target file_work
800  typeset NB_ESSAI DELAI status i
801
802  if ( $DEBUG_sys ) ; then
803    echo "IGCM_sys_Get :" $@
804  fi
805
806  # number of tentative
807  NB_ESSAI=3
808  # time delay between tentative
809  DELAI=2
810
811  if [ $DRYRUN -le 2 ]; then
812    if [ X${1} = X'/l' ] ; then
813      eval set +A dm_liste \${${2}}
814    else
815      eval set +A dm_liste ${1}
816    fi
817    eval DEST=\${${#}}
818    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
819    status=$?
820    if [ ${status} -gt 0 ] ; then
821      echo "WARNING IGCM_sys_Get : error code ${status}"
822      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
823      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
824    fi
825
826    #if [ ${status} -gt 0 ] ; then
827    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
828    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
829    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
830    #    sleep 30
831    #    echo "We try another time"
832    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
833    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
834    #    status=$?
835    #    if [ ${status} -gt 0 ] ; then
836    #      echo "ERROR IGCM_sys_Get : again demigration error :"
837    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
838    #      IGCM_debug_Exit "IGCM_sys_Get"
839    #    fi
840    #  else
841    #    echo "ERROR IGCM_sys_Get : demigration error :"
842    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
843    #    IGCM_debug_Exit "IGCM_sys_Get"
844    #  fi
845    #fi
846
847    #   #RSYNC WITH NETWORK SSH CALL
848    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
849    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
850
851    #   #RSYNC WITH NFS USE
852    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
853    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
854
855    #   status=$?
856    #   IGCM_sys_Rsync_out $status
857
858    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
859    #   (( status=status+$? ))
860
861    #USUAL WAY
862    if [ X${1} = X'/l' ] ; then
863      for target in ${dm_liste[*]} ; do
864        local_file=$( basename ${target} )
865        # test if the target file is present before the loop
866        IGCM_sys_TestFileArchive ${target}
867        status=$?
868        if [ ${status} -gt 0 ] ; then
869          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
870          IGCM_debug_Exit "IGCM_sys_Get"
871        else
872          i=0
873          while [ $i -lt $NB_ESSAI ] ; do
874            #if [ X${DoLink} = Xtrue ] ; then
875            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
876            #  status=$?
877            #  else
878            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
879            #  status=$?
880            #fi
881            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
882            status=$?
883            if [ ${status} -gt 0 ]; then
884              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
885              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
886              sleep $DELAI
887            else
888              break
889            fi
890            (( i = i + 1 ))
891          done
892          if [ ${status} -gt 0 ] ; then
893            echo "IGCM_sys_Get : error"
894            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
895            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
896            IGCM_debug_Exit "IGCM_sys_Get"
897          else
898            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
899          fi
900        fi
901      done
902    else
903      i=0
904      while [ $i -lt $NB_ESSAI ] ; do
905        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
906        status=$?
907        if [ ${status} -gt 0 ]; then
908          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
909          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
910          sleep $DELAI
911        else
912          break
913        fi
914        (( i = i + 1 ))
915      done
916      if [ ${status} -gt 0 ] ; then
917        echo "IGCM_sys_Get : error"
918        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
919        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
920        IGCM_debug_Exit "IGCM_sys_Get"
921      else
922        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
923      fi
924    fi
925  fi
926  IGCM_debug_PopStack "IGCM_sys_Get"
927}
928
929#D-#==================================================
930#D-function IGCM_sys_GetDate_Monitoring
931#D-* Purpose: get the last year for which the monitoring has been computed
932#D-* Examples:
933#D-
934function IGCM_sys_GetDate_Monitoring {
935  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
936  if ( $DEBUG_sys ) ; then
937    echo "IGCM_sys_GetDate_Monitoring :" $@
938  fi
939
940  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
941
942  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
943}
944
945#D-#==================================================
946#D-function IGCM_sys_Dods_Rm
947#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
948#D-* Examples:
949#D-
950function IGCM_sys_Dods_Rm {
951  if ( $DEBUG_sys ) ; then
952    echo "IGCM_sys_Dods_Rm :" $@
953  fi
954  typeset status
955  if [ $DRYRUN = 0 ]; then
956
957#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
958#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
959#      echo "Nothing has been done."
960#      return
961#    fi
962
963    if [ "$#" -eq 1 ]; then
964      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
965      status=$?
966    else
967      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
968      status=$?
969    fi
970
971#    if [ ${status} -gt 0 ] ; then
972#      echo "IGCM_sys_Dods_Rm : error."
973#      cat out_dods_rm
974#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
975#    else
976#      rm out_dods_rm
977#    fi
978
979  fi
980  return $status
981}
982
983#D-#==================================================
984#D-function IGCM_sys_Dods_Cp
985#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
986#D-* Examples:
987#D-
988function IGCM_sys_Dods_Cp {
989  if ( $DEBUG_sys ) ; then
990    echo "IGCM_sys_Dods_Cp :" $@
991  fi
992  typeset status
993  if [ $DRYRUN = 0 ]; then
994
995#    if [ ! -d ${R_SAVE}/${1} ] ; then
996#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
997#      echo "Nothing has been done."
998#      return
999#    fi
1000
1001    $(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1002    status=$?
1003
1004#       if [ ${status} -gt 0 ] ; then
1005#           echo "IGCM_sys_Dods_Cp : error."
1006#           cat out_dods_cp
1007#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1008#       else
1009#           rm out_dods_cp
1010#       fi
1011
1012  fi
1013  return $status
1014}
1015
1016#D-#==================================================
1017#D-function IGCM_sys_Put_Dods
1018#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1019#D-* Examples:
1020#D-
1021function IGCM_sys_Put_Dods {
1022  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1023  if ( $DEBUG_sys ) ; then
1024    echo "IGCM_sys_Put_Dods :" $@
1025  fi
1026  typeset status
1027  if [ $DRYRUN = 0 ]; then
1028    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1029      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1030      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1031      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1032      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1033      return
1034    fi
1035
1036    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1037      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1038      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1039      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1040      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1041      return
1042    fi
1043    #
1044    if [ -d ${R_SAVE}/${1} ] ; then
1045      cd ${R_SAVE}
1046    elif [ -d ${R_FIGR}/${1} ] ; then
1047      cd ${R_FIGR}
1048    fi
1049
1050    IGCM_sys_Dods_Rm ${1}
1051    IGCM_sys_Dods_Cp ${1}
1052    status=0
1053
1054    if [ ${status} -gt 0 ] ; then
1055      echo "IGCM_sys_Put_Dods : error."
1056      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1057    fi
1058  fi
1059  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1060}
1061
1062##############################################################
1063# REBUILD OPERATOR
1064
1065#D-#==================================================
1066#D-function IGCM_sys_sync
1067#D-* Purpose: flush buffer on disk
1068#D-* Examples:
1069#D-
1070function IGCM_sys_sync {
1071  IGCM_debug_PushStack "IGCM_sys_sync" $@
1072  if ( $DEBUG_sys ) ; then
1073    echo "IGCM_sys_sync :" $@
1074  fi
1075
1076  /bin/sync
1077
1078  IGCM_debug_PopStack "IGCM_sys_sync"
1079}
1080
1081############################################################
1082# Activate Running Environnment Variables
1083
1084#D-#==================================================
1085#D-function IGCM_sys_activ_variables
1086#D-* Purpose: set environement variables prior to execution
1087#D-* Examples:
1088#D-
1089function IGCM_sys_activ_variables {
1090  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1091  if ( $DEBUG_sys ) ; then
1092    echo "IGCM_sys_activ_variables"
1093  fi
1094
1095# --------------------------------------------------------------------
1096#D- MODULE specifications
1097# --------------------------------------------------------------------
1098  # Source the file EnvFile containing all module load needed to run the model.
1099  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1100  # it will be used. Otherwise a default file will be used.
1101  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1102      # EnvFile is set in config.card and the file exists
1103      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1104      EnvFile=${config_UserChoices_EnvFile}
1105  else
1106      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1107      module purge > /dev/null 2>&1
1108      if [ X${compilerVersion} = XV19 ] ; then
1109          IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V19"
1110          EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
1111      else
1112          IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1113      fi
1114  fi
1115  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1116  IGCM_debug_Print 1 ${EnvFile}
1117  . ${EnvFile}
1118
1119  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene-amd. "
1120  module list
1121
1122# --------------------------------------------------------------------
1123#D- MPI specifications
1124# --------------------------------------------------------------------
1125
1126# --------------------------------------------------------------------
1127#D- Other specifications
1128# --------------------------------------------------------------------
1129
1130  ulimit -s unlimited
1131
1132  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1133}
1134
1135############################################################
1136# Desactivate Running Environnment Variables
1137
1138#D-#==================================================
1139#D-function IGCM_sys_desactiv_variables
1140#D-* Purpose: unset environement variables after execution
1141#D-* Examples:
1142#D-
1143function IGCM_sys_desactiv_variables {
1144  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1145  if ( $DEBUG_sys ) ; then
1146    echo "IGCM_sys_desactiv_variables"
1147  fi
1148# --------------------------------------------------------------------
1149#D- MPI specifications
1150# --------------------------------------------------------------------
1151
1152# --------------------------------------------------------------------
1153#D- Other specifications
1154# --------------------------------------------------------------------
1155
1156  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1157}
1158
1159############################################################
1160# Update job headers to be used by the scheduler
1161
1162#D-#==================================================
1163#D-function IGCM_sys_updateHeaders
1164#D-* Purpose: Update job headers to be used by the scheduler
1165#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1166#D-
1167function IGCM_sys_updateHeaders {
1168  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1169  if ( $DEBUG_sys ) ; then
1170    echo "IGCM_sys_updateHeaders"
1171  fi
1172  typeset file
1173  file=$1
1174
1175   # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1176   # Compute the number of core to add to force the use of a new node for xios servers
1177
1178   if [ "X${IOS_OK_DEDICATED}" == "X" ] ; then
1179       (( IOS_OK_DEDICATED = 0 ))
1180   fi
1181
1182
1183   if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1184      (( current_core_noxios = coreNumber - IOS_PROC_MPI * IOS_PROC_OMP * IOS_PROC_DEP ))
1185      (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1186      (( second_comp_proc_mpi_loc = IOS_PROC_MPI - 1 ))
1187      (( coreNumber_final = current_core_noxios - 1 + first_comp_proc_dep_loc * 2 + second_comp_proc_mpi_loc * IOS_PROC_OMP * IOS_PROC_DEP ))
1188   else
1189      (( coreNumber_final = coreNumber ))
1190   fi
1191
1192  if [ ${executionType} -eq 1 ] ; then
1193    # MPMD + MPI
1194    sed -e "/::openMPthreads::/d"                  \
1195        -e "s/::JobNumProcTot::/${coreNumber_final}/"    \
1196      ${file} > ${file}.tmp
1197
1198  elif [ ${executionType} -eq 2 ] ; then
1199    # MPMD + MPI + OMP :ccc_mprun
1200    sed -e "/::openMPthreads::/d"                  \
1201        -e "s/::JobNumProcTot::/${coreNumber_final}/"    \
1202      ${file} > ${file}.tmp
1203
1204  elif [ ${executionType} -eq 3 ] ; then
1205    # SPMD + MPI/OMP
1206    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1207        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1208        -e "/#MSUB -x/d"                           \
1209      ${file} > ${file}.tmp
1210
1211  elif [ ${executionType} -eq 4 ] ; then
1212    # SPMD + MPI only
1213    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1214        -e "/::openMPthreads::/d"                  \
1215        -e "/#MSUB -x/d"                           \
1216      ${file} > ${file}.tmp
1217
1218  elif [ ${executionType} -eq 5 ] ; then
1219    # SPMD + OMP only
1220    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1221        -e "/::JobNumProcTot::/d"                  \
1222        -e "/#MSUB -x/d"                           \
1223      ${file} > ${file}.tmp
1224
1225  elif [ ${executionType} -eq 6 ] ; then
1226    # SEQUENTIAL THEN
1227    sed -e "s/::JobNumProcTot::/1/"                \
1228        -e "/::openMPthreads::/d"                  \
1229        -e "/#MSUB -x/d"                           \
1230      ${file} > ${file}.tmp
1231
1232  fi
1233
1234  IGCM_sys_Mv ${file}.tmp ${file}
1235
1236  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1237}
1238
1239############################################################
1240# Build MPI/OMP scripts run file (dummy function)
1241
1242#D-#==================================================
1243#D-function IGCM_sys_build_run_file
1244#D-* Purpose: build run file (deprecated)
1245#D-* Examples:
1246#D-
1247function IGCM_sys_build_run_file {
1248  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1249}
1250
1251############################################################
1252# Build MPI/OMP scripts
1253
1254#D-#==================================================
1255#D-function IGCM_sys_build_execution_scripts
1256#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1257#D-* Examples:
1258#D-
1259function IGCM_sys_build_execution_scripts
1260{
1261  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1262  if ( $DEBUG_sys ) ; then
1263    echo "IGCM_sys_build_execution_scripts " $@
1264  fi
1265
1266  EXECUTION=${HOST_MPIRUN_COMMAND}
1267
1268  if ( ${OK_PARA_MPMD} ) ; then
1269
1270    # MPMD mode
1271    # 1 MPI only
1272    # 2 MPI/OpenMP mpirun method
1273    # 3 MPI/OpenMP ccc_mprun method
1274
1275    if [ -f run_file ] ; then
1276      IGCM_sys_Rm -f run_file
1277    fi
1278    touch run_file
1279
1280    # case 1 : Only MPI (MPMD)
1281    if  ( ! ${OK_PARA_OMP} ) ; then
1282
1283      # Build run_file
1284
1285      # First loop on the components for the coupler ie oasis (only if oasis3)
1286      # the coupler ie oasis3 must be the first one
1287      for comp in ${config_ListOfComponents[*]} ; do
1288
1289        eval ExeNameIn=\${config_Executable_${comp}[0]}
1290        eval ExeNameOut=\${config_Executable_${comp}[1]}
1291
1292        # for CPL component only
1293        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1294
1295          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1296          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1297          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1298        fi
1299      done
1300
1301      # Then second loop on the components
1302      for comp in ${config_ListOfComponents[*]} ; do
1303
1304        eval ExeNameIn=\${config_Executable_${comp}[0]}
1305        eval ExeNameOut=\${config_Executable_${comp}[1]}
1306
1307        # Only if we really have an executable for the component and not the coupler ie oasis:
1308        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1309
1310          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1311          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1312          eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1313
1314          # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1315          # Compute the number of core to add to force the use of a new node for xios servers
1316         if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XIOS" ] ) ; then
1317            if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1318              (( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_dep_loc ))
1319              (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1320              (( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1321              if [ ${comp_proc_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1322                 echo "1-${first_comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1323                 if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1324                    echo "${second_comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1325                 fi
1326              else
1327                 echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1328              fi
1329           else
1330              echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1331           fi
1332         else
1333          echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1334         fi
1335
1336
1337        fi
1338      done
1339
1340      ## module advised by TGCC (instead of 2 variables)
1341      module load feature/bridge/heterogenous_mpmd
1342
1343      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1344
1345      IGCM_sys_Chmod u+x run_file
1346      if ( $DEBUG_sys ) ; then
1347        echo "run_file contains : "
1348        cat run_file
1349      fi
1350
1351    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1352
1353    # 2 MPI/OpenMP mpirun method
1354
1355      # Use of mpirun instead of ccc_mprun
1356      EXECUTION="time mpirun"
1357
1358      #  Hosts treatment
1359      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1360
1361      i=0
1362      rm -f hosts rankfile
1363      IGCM_debug_Print 1 "sys Irene-amd, Hosts available :"
1364      for nodes in `cat hosts.tmp`
1365      do
1366        host[$i]=$nodes
1367        echo "${host[$i]}" >> hosts
1368        IGCM_debug_Print 1 ${host[$i]}
1369        i=$((i+1))
1370      done
1371      rm -f hosts.tmp
1372
1373      listnodes=${host[*]}
1374
1375      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1376
1377      # Initialisation
1378      rank=0
1379      current_core=0
1380      init_exec=n
1381
1382      # Loop on the components
1383      for comp in ${config_ListOfComponents[*]} ; do
1384
1385        eval ExeNameIn=\${config_Executable_${comp}[0]}
1386        eval ExeNameOut=\${config_Executable_${comp}[1]}
1387
1388        # Not possible if oasis has an executable (i.e old version of oasis3)
1389        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1390          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1391          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1392          IGCM_debug_Verif_Exit
1393        fi
1394
1395        # Only if we really have an executable for the component :
1396        if [ "X${ExeNameOut}" != X\"\" ] ; then
1397
1398          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1399          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1400          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1401
1402          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1403          echo ""  >> script_${ExeNameOut}.ksh
1404          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1405
1406            # Check if the number of threads is correct
1407            case ${comp_proc_omp_loc} in
1408            2|4|8|16)
1409              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1410              ;;
1411            *)
1412              IGCM_debug_Exit "ERROR with OMP parameters !"
1413              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1414              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1415              IGCM_debug_Verif_Exit
1416              ;;
1417            esac
1418            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1419            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1420            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1421            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1422          fi
1423
1424          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1425          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1426          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1427          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1428          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1429
1430          if [ ${init_exec} = y ] ; then
1431            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1432          else
1433            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1434            init_exec=y
1435          fi
1436
1437          # Build rankfile : method used to assign cores and nodes for the MPI process
1438          # Ex :
1439          #rank 0=curie5296 slot=0,1,2,3
1440          #rank 1=curie5296 slot=4,5,6,7
1441          # Example of final command :
1442          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1443          # with script_lmdz.x.ksh :
1444          # #!/bin/ksh
1445          #export KMP_STACKSIZE=3g
1446          #export KMP_LIBRARY=turnaround
1447          #export MKL_SERIAL=YES
1448          #OMP_NUM_THREADS=4
1449          #./lmdz.x
1450          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1451              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1452          else
1453              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1454          fi
1455          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1456             (( index_host = current_core / NB_CORE_PER_NODE ))
1457             host_value=${host[${index_host}]}
1458             (( slot =  current_core % NB_CORE_PER_NODE ))
1459             virg=","
1460             string_final=""
1461             for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1462                 string=$index$virg
1463                 string_final=$string_final$string
1464             done
1465             string_final=$( echo $string_final | sed "s/.$//" )
1466             echo "rank $rank=$host_value slot=$string_final" >> rankfile
1467             (( rank = rank + 1 ))
1468             (( current_core = current_core + offset_comp_proc_loc ))
1469          done
1470        fi
1471      done
1472
1473    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1474
1475    # 3 MPI/OpenMP ccc_mprun method
1476
1477    # MPI-OpenMP (MPMD)
1478    # example of run_file
1479    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1480    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1481    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1482
1483      # Loop on the components
1484      for comp in ${config_ListOfComponents[*]} ; do
1485
1486        eval ExeNameIn=\${config_Executable_${comp}[0]}
1487        eval ExeNameOut=\${config_Executable_${comp}[1]}
1488
1489        # Not possible if oasis has an executable (i.e old version of oasis3)
1490        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1491          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1492          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1493          IGCM_debug_Verif_Exit
1494        fi
1495
1496        # Only if we really have an executable for the component :
1497        if [ "X${ExeNameOut}" != X\"\" ] ; then
1498
1499          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1500          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1501          eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1502
1503          # Compute number of cores per process
1504          (( comp_proc_omp_dep_loc = comp_proc_omp_loc * comp_proc_dep_loc )) 
1505
1506          # Build script files
1507
1508          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1509          echo ""  >> script_${ExeNameOut}.ksh
1510          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1511
1512            # Check if the number of threads is correct
1513            case ${comp_proc_omp_loc} in
1514            2|4|8|16)
1515              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1516              ;;
1517            *)
1518              IGCM_debug_Exit "ERROR with OMP parameters !"
1519              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1520              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1521              IGCM_debug_Verif_Exit
1522              ;;
1523            esac
1524            ### only for LMDZ?
1525            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1526            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1527            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1528            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1529
1530          fi
1531
1532          # to have out/err per process on different files
1533          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1534          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1535
1536          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1537
1538          # Complete run_file
1539
1540          # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1541          # Compute the number of core to add to force the use of a new node for xios servers
1542         if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XIOS" ] ) ; then
1543            if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1544              (( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_omp_dep_loc )) 
1545              (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1546              (( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1547              if [ ${comp_proc_omp_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1548                 echo "1-${first_comp_proc_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1549                  if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1550                    echo "${second_comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1551                  fi
1552              else
1553                 echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1554              fi
1555           else
1556              echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1557           fi
1558         else
1559          echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1560         fi
1561
1562        fi
1563      done
1564
1565      ## variable added to stop after 60s instead of 600s by default.
1566      ## This is used when no error comes from executables and when something stopped an executable without notice.
1567      export SLURM_WAIT=60
1568
1569      ## module advised by TGCC (instead of 2 variables)
1570      module load feature/bridge/heterogenous_mpmd
1571
1572      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1573
1574      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1575
1576      IGCM_sys_Chmod u+x run_file
1577      if ( $DEBUG_sys ) ; then
1578        echo "run_file contains : "
1579        cat run_file
1580      fi
1581
1582    else
1583
1584      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1585
1586    fi # 1 2 or 3 if ${OK_PARA_MPMD}
1587
1588  else
1589  # Only one executable (SPMD mode).
1590
1591    for comp in ${config_ListOfComponents[*]} ; do
1592
1593      # Only if we really have an executable for the component :
1594      eval ExeNameOut=\${config_Executable_${comp}[1]}
1595      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1596
1597        # Build script files
1598
1599        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1600        echo ""  >> script_${ExeNameOut}.ksh
1601        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1602
1603        if ( ${OK_PARA_OMP} ) ; then
1604          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1605          echo ""  >> script_${ExeNameOut}.ksh
1606          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1607          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1608          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1609          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1610        fi
1611
1612        if  ( ${OK_PARA_MPI} ) ; then
1613          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1614          # Default : ccc_mprun used if nb_proc gt 1
1615          # to have out/err per process on different files
1616          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1617          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1618          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1619        else
1620          # Default : ccc_mprun is NOT used if nb_proc eq 1
1621          # to have out/err per process on different files
1622          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1623          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1624          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1625        fi
1626
1627        IGCM_debug_Print 1 "sys Irene-amd : script_${ExeNameOut}.ksh contains"
1628        cat script_${ExeNameOut}.ksh
1629
1630      fi
1631
1632    done
1633
1634  fi # ${OK_PARA_MPMD}
1635
1636  IGCM_debug_Print 1 "sys Irene-amd : execution command is "
1637  IGCM_debug_Print 1 "$EXECUTION"
1638
1639  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1640}
1641
1642#D-#==================================================
1643#D-function IGCM_sys_check_path
1644#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1645#D-* do not point to an important use directory. Stop immediately in that case.
1646#D-* Examples:
1647#D-
1648function IGCM_sys_check_path {
1649  IGCM_debug_PushStack "IGCM_sys_check_path"
1650  if ( $DEBUG_sys ) ; then
1651    echo "IGCM_sys_check_path"
1652  fi
1653
1654  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1655    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1656    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1657    IGCM_debug_Exit "This will stop the job"
1658  fi
1659  IGCM_debug_PopStack "IGCM_sys_check_path"
1660}
1661
1662#D-#==================================================
1663#D-function IGCM_sys_check_quota
1664#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1665#D-* Examples:
1666#D-
1667function IGCM_sys_check_quota {
1668  IGCM_debug_PushStack "IGCM_sys_check_quota"
1669  if ( $DEBUG_sys ) ; then
1670    echo "IGCM_sys_check_quota"
1671  fi
1672  # Limit of quota (in %)
1673  limit_quota=90
1674
1675  # Check of the volume
1676  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1677  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1678
1679  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1680
1681    unit_avail=${volume_avail: -1}
1682    unit_quota=${volume_quota: -1}
1683
1684    if [ "${unit_quota}" = "*" ] ; then
1685      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1686      IGCM_debug_Print 1 "More than 100% of your quota is used"
1687      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1688      IGCM_debug_Print 1 "You must have more than 10% available to run"
1689      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1690      IGCM_debug_Verif_Exit
1691    fi
1692
1693    temp_avail=${volume_avail%%${volume_avail: -1}*}
1694    temp_quota=${volume_quota%%${volume_quota: -1}*}
1695
1696    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1697
1698    # Convertion
1699      if [ ${volume_avail: -1} = "T" ] ; then
1700        (( temp_avail = temp_avail * 1000000000000 ))
1701      elif [ ${volume_avail: -1} = "G" ] ; then
1702        (( temp_avail = temp_avail * 1000000000 ))
1703      elif [ ${volume_avail: -1} = "M" ] ; then
1704        (( temp_avail = temp_avail * 1000000 ))
1705      elif [ ${volume_avail: -1} = "k" ] ; then
1706        (( temp_avail = temp_avail * 1000 ))
1707      else
1708        (( temp_avail = volume_avail ))
1709      fi
1710      if [ ${volume_quota: -1} = "T" ] ; then
1711        (( temp_quota = temp_quota * 1000000000000 ))
1712      elif [ ${volume_quota: -1} = "G" ] ; then
1713        (( temp_quota = temp_quota * 1000000000 ))
1714      elif [ ${volume_quota: -1} = "M" ] ; then
1715        (( temp_quota = temp_quota * 1000000 ))
1716      elif [ ${volume_quota: -1} = "k" ] ; then
1717        (( temp_quota = temp_quota * 1000 ))
1718      else
1719        (( temp_quota = volume_quota ))
1720      fi
1721    fi
1722
1723    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1724#    echo "volume ratio is " $quota_volume
1725
1726    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1727      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1728      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1729      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1730      IGCM_debug_Print 1 "You must have more than 10% available to run"
1731      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1732      IGCM_debug_Verif_Exit
1733    fi
1734
1735  fi
1736
1737# Check of the number of inodes
1738
1739  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1740  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1741
1742  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1743
1744    unit_avail=${inode_avail: -1}
1745    unit_quota=${inode_quota: -1}
1746
1747    if [ "${unit_quota}" = "*" ] ; then
1748      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1749      IGCM_debug_Print 1 "More than 100% of your quota is used"
1750      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1751      IGCM_debug_Print 1 "You must have more than 10% available to run"
1752      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1753      IGCM_debug_Verif_Exit
1754    fi
1755
1756    temp_avail=${inode_avail%%${inode_avail: -1}*}
1757    temp_quota=${inode_quota%%${inode_quota: -1}*}
1758
1759    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1760
1761    # Convertion
1762      if [ ${inode_avail: -1} = "T" ] ; then
1763        (( temp_avail = temp_avail * 1000000000000 ))
1764      elif [ ${inode_avail: -1} = "G" ] ; then
1765        (( temp_avail = temp_avail * 1000000000 ))
1766      elif [ ${inode_avail: -1} = "M" ] ; then
1767        (( temp_avail = temp_avail * 1000000 ))
1768      elif [ ${inode_avail: -1} = "k" ] ; then
1769        (( temp_avail = temp_avail * 1000 ))
1770      else
1771        (( temp_avail = inode_avail ))
1772      fi
1773
1774      if [ ${inode_quota: -1} = "T" ] ; then
1775        (( temp_quota = temp_quota * 1000000000000 ))
1776      elif [ ${inode_quota: -1} = "G" ] ; then
1777        (( temp_quota = temp_quota * 1000000000 ))
1778      elif [ ${inode_quota: -1} = "M" ] ; then
1779        (( temp_quota = temp_quota * 1000000 ))
1780      elif [ ${inode_quota: -1} = "k" ] ; then
1781        (( temp_quota = temp_quota * 1000 ))
1782      else
1783        (( temp_quota = inode_quota ))
1784      fi
1785    fi
1786    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1787#    echo "inode ratio is " $quota_inode
1788
1789    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1790      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1791      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1792      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1793      IGCM_debug_Print 1 "You must have more than 10% available to run"
1794      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1795      IGCM_debug_Verif_Exit
1796    fi
1797  fi
1798  IGCM_debug_PopStack "IGCM_sys_check_quota"
1799}
1800
1801#D-#==================================================
1802#D-function IGCM_sys_projectAccounting
1803#D-* Purpose: store project accounting information in a file
1804#D-* Examples:
1805#D-
1806function IGCM_sys_projectAccounting {
1807  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1808  if ( $DEBUG_sys ) ; then
1809    echo "IGCM_sys_check_quota"
1810  fi
1811
1812  ssh irene194 /usr/bin/ccc_myproject > $1
1813
1814  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1815}
1816
1817#D-#==================================================
1818#D-function IGCM_sys_getJobSchedulerID
1819#D-* Purpose: Get the job ID during execution
1820#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1821#D-
1822function IGCM_sys_getJobSchedulerID {
1823  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1824  if ( $DEBUG_sys ) ; then
1825    echo "IGCM_sys_getJobSchedulerID"
1826  fi
1827  eval ${1}=${BRIDGE_MSUB_JOBID}
1828
1829  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1830}
1831
1832#D-#==================================================
1833#D-function IGCM_sys_GetJobID
1834#D-* Purpose: Get the job ID from the JobName
1835#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1836#D-
1837function IGCM_sys_GetJobID {
1838  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1839  if ( $DEBUG_sys ) ; then
1840    echo "IGCM_sys_GetJobID"
1841  fi
1842
1843  # With -f option, the full job name is given in the last column
1844  ID=$( ccc_mstat -f -u $2 | \
1845        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1846
1847  eval ${3}=${ID}
1848  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1849}
1850
1851#D-#==================================================
1852#D-function IGCM_sys_CountJobInQueue
1853#D-* Purpose: Count number of users job
1854#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1855#D-
1856function IGCM_sys_CountJobInQueue {
1857  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1858  if ( $DEBUG_sys ) ; then
1859    echo "IGCM_sys_CountJobInQueue"
1860  fi
1861
1862  # With -f option, the full job name is given in the last column
1863  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1864
1865  eval ${2}=${NbRun}
1866
1867  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1868}
1869
1870#D-#==================================================
1871#D-function IGCM_sys_ListJobInQueue
1872#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1873#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1874#D-
1875function IGCM_sys_ListJobInQueue {
1876  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1877  if ( $DEBUG_sys ) ; then
1878    echo "IGCM_sys_ListJobInQueue"
1879  fi
1880
1881  # With -f option, the full job name is given in the last column
1882  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1883                                        '( $2  == User      && \
1884                                           $NF != /TS/      && \
1885                                           $NF !~ /PACK/    && \
1886                                           $NF !~ /REBUILD/ && \
1887                                           $NF !~ /pack/ )     \
1888                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1889
1890  eval set -A ${2} ${JobList[*]}
1891
1892  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1893}
1894
1895#D-#==================================================
1896#D-function IGCM_sys_atlas
1897#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1898#D-* Examples:
1899#D-
1900function IGCM_sys_atlas {
1901  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1902  if ( $DEBUG_sys ) ; then
1903    echo "IGCM_sys_atlas :" $@
1904  fi
1905
1906  typeset status
1907
1908  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1909  status=$?
1910  if [ ${status} -gt 0 ] ; then
1911    echo "IGCM_sys_atlas : error code ${status}"
1912    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1913    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1914    IGCM_debug_PopStack "IGCM_sys_atlas"
1915    return 1
1916  else
1917    IGCM_debug_PopStack "IGCM_sys_atlas"
1918    return 0
1919  fi
1920
1921  IGCM_debug_PopStack "IGCM_sys_atlas"
1922}
1923
1924#D-#==================================================
1925#D-function IGCM_sys_rebuild_nemo
1926#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1927#D-* Examples:
1928#D-
1929
1930function IGCM_sys_rebuild_nemo {
1931  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1932  if ( $DEBUG_sys ) ; then
1933    echo "IGCM_sys_rebuild_nemo :" $@
1934  fi
1935
1936  $( ccc_home -u igcmg)/Tools/irene/bin/rebuild_nemo ${1} ${2}
1937
1938  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1939}
1940
Note: See TracBrowser for help on using the repository browser.