source: tags/libIGCM_v2.8.4/libIGCM_sys/libIGCM_sys_irene.ksh @ 1458

Last change on this file since 1458 was 1458, checked in by jgipsl, 6 years ago

Changed paths for irene as done in the trunk in rev [1457]
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 62.1 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Irene
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host user names project maxCpuTime
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=irene
71# add default project on irene
72typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} | cut -d@ -f1 )
73# jobWarningDelay in seconds
74typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=ccc_msub}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Set environment tools (ferret, nco, cdo, rebuild, ...)
93#====================================================
94if [ X${TaskType} = Xcomputing ] ; then
95  IGCM_debug_Print 1 "Module will be loaded once we know the compiler version"
96else
97  module purge > /dev/null 2>&1
98  ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_irene > /dev/null 2>&1
99  . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_irene > /dev/null 2>&1
100  ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
101  . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
102
103  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
104  export UVCDAT_ANONYMOUS_LOG=FALSE
105fi
106
107# Use CMIP6 storage space when using project id gch0316
108[[ "X${PROJECT}" = "Xgch0316" && ! $(module list --terse 2>&1 | grep dfldatadir/gencmip6) ]] && module switch dfldatadir dfldatadir/gencmip6
109
110# FYI
111[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
112[ ! X${TaskType} = Xchecking ] && module list
113
114# For AddNoise and AddPertu3DOCE
115##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/curie/bin
116export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/curie/bin
117
118# For rebuild
119##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
120export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
121
122#====================================================
123# Host specific DIRECTORIES
124#====================================================
125
126# ============ CESIUM START ============ #
127
128#====================================================
129#- Mirror libIGCM from titane to cesium if needed
130#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
131#if [ ! ${ROOTSYS} = "home" ] ; then
132#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
133#else
134#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
135#fi
136
137#====================================================
138#- libIGCM_POST
139#if ( ${MirrorlibIGCM} ) ; then
140#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
141#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
142#else
143#  typeset -r libIGCM_POST=${libIGCM}
144#fi
145
146# ============ CESIUM  END  ============ #
147
148#====================================================
149#- MirrorlibIGCM for frontend
150typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
151
152#====================================================
153#- libIGCM_POST for frontend
154typeset -r libIGCM_POST=${libIGCM}
155
156#====================================================
157#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
158typeset -r R_EXE="${MODIPSL}/bin"
159
160#====================================================
161#- SUBMIT_DIR : submission dir
162typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
163
164#====================================================
165#- IN
166typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
167typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
168
169#====================================================
170#- RUN_DIR_PATH : Temporary working directory (=> TMP)
171typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
172
173#====================================================
174#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
175typeset -r OUTCOMMAND_PATH=/tmp
176
177#====================================================
178#- HOST_MPIRUN_COMMAND
179typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
180
181#====================================================
182#- Max number of arguments passed to nco operator or demigration command
183UNIX_MAX_LIMIT=360
184
185#====================================================
186#- set PackDefault to true on irene
187PackDefault=true
188
189#====================================================
190#- Default number of MPI task for IPSL coupled model
191#- required for backward compatibility
192#-
193DEFAULT_NUM_PROC_OCE=5
194DEFAULT_NUM_PROC_CPL=1
195(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
196
197#====================================================
198#- Number of computing cores per node
199#-
200NB_CORE_PER_NODE=16
201
202#D-#==================================================
203#D-function IGCM_sys_defineArchives
204#D-* Purpose:
205#D-* Define ARCHIVE : Dedicated to large files
206#D-* Define STORAGE : Dedicated to small/medium files
207#D-* Define R_OUT   : Output tree located on ARCHIVE
208#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
209#D-* Define R_BUF   : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
210#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
211#D-* Examples:
212#D-
213function IGCM_sys_defineArchives {
214  IGCM_debug_PushStack "IGCM_sys_defineArchives"
215
216  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
217    #====================================================
218    #- ARCHIVE (dedicated to large files)
219    ARCHIVE=${config_UserChoices_ARCHIVE}
220  else
221    #====================================================
222    #- ARCHIVE (dedicated to large files)
223    ARCHIVE=${CCCSTOREDIR}
224  fi
225
226  if [ ! X${config_UserChoices_STORAGE} = X ]; then
227    #====================================================
228    #- STORAGE (dedicated to small/medium files)
229    STORAGE=${config_UserChoices_STORAGE}
230  else
231    #====================================================
232    #- STORAGE (dedicated to small/medium files)
233    STORAGE=${CCCWORKDIR}
234  fi
235
236  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
237    #====================================================
238    #- R_OUT
239    R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
240
241    #====================================================
242    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
243    R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
244
245    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
246
247  else
248    #====================================================
249    #- R_OUT
250    R_OUT=${ARCHIVE}/IGCM_OUT
251
252    #====================================================
253    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
254    R_FIG=${STORAGE}/IGCM_OUT
255  fi
256
257  #====================================================
258  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
259  CMIP6_BUF=${STORAGE}/IGCM_OUT
260
261  #====================================================
262  #- R_BUF (ONLY FOR double copy an scratch)
263  R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
264
265  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
266  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
267  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
268
269  IGCM_debug_PopStack "IGCM_sys_defineArchives"
270}
271
272#D-#==================================================
273#D-function IGCM_sys_RshArchive
274#D-* Purpose: Archive rsh command
275#D-* Examples:
276#D-
277function IGCM_sys_RshArchive {
278  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
279  /bin/ksh <<-EOF
280    ${@}
281EOF
282  status=$?
283  if [ ${status} -gt 0 ] ; then
284    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
285    IGCM_debug_Exit "IGCM_sys_RshArchive"
286  fi
287  IGCM_debug_PopStack "IGCM_sys_RshArchive"
288}
289
290#D-#==================================================
291#D-function IGCM_sys_RshArchive_NoError
292#D-* Purpose: Archive rsh command, without error
293#D-*          used only in monitoring.job
294#D-* Examples:
295#D-
296function IGCM_sys_RshArchive_NoError {
297  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
298  /bin/ksh <<-EOF
299    ${@} 2> /dev/null
300EOF
301  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
302}
303
304#D-#==================================================
305#D-function IGCM_sys_MkdirArchive
306#D-* Purpose: Mkdir on Archive
307#D-* Examples:
308#D-
309function IGCM_sys_MkdirArchive {
310  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
311  if ( $DEBUG_sys ) ; then
312    echo "IGCM_sys_MkdirArchive :" $@
313  fi
314  #- creation de repertoire sur le serveur fichier
315  if [ ! -d ${1} ]; then
316    \mkdir -p $1
317    status=$?
318
319    if [ ${status} -gt 0 ] ; then
320      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
321      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
322    fi
323  fi
324  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
325}
326
327#D-#==================================================
328#D-function IGCM_sys_TestDirArchive
329#D-* Purpose: Test Directory that must exists on Archive
330#D-* Examples:
331#D-
332function IGCM_sys_TestDirArchive {
333  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
334  if ( $DEBUG_sys ) ; then
335    echo "IGCM_sys_TestDirArchive :" $@
336  fi
337  typeset ExistFlag
338  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
339  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
340  return ${ExistFlag}
341}
342
343#D-#==================================================
344#D-function IGCM_sys_IsFileArchived
345#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
346#D-* Examples:
347#D-
348function IGCM_sys_IsFileArchived {
349  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
350  if ( $DEBUG_sys ) ; then
351    echo "IGCM_sys_IsFileArchived :" $@
352  fi
353  typeset IsArchivedFlag
354  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
355  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
356
357  return ${IsArchivedFlag}
358}
359
360#D-#==================================================
361#D-function IGCM_sys_TestFileArchive
362#D-* Purpose: Test file that must NOT EXISTS on Archive
363#D-* Examples:
364#D-
365function IGCM_sys_TestFileArchive {
366  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
367  typeset ExistFlag
368  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
369  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
370
371  return ${ExistFlag}
372}
373
374#D-#==================================================
375#D-function IGCM_sys_CountFileArchive
376#D-* Purpose: Count files on Archive filesystem
377#D-* Examples:
378#D-
379function IGCM_sys_CountFileArchive {
380  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
381  ls ${@} 2>/dev/null | wc -l
382  if [ $? -gt 0 ] ; then
383    echo "IGCM_sys_CountFileArchive : erreur."
384  fi
385  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
386}
387
388#D-#==================================================
389#D-function IGCM_sys_Tree
390#D-* Purpose: Tree directories with files on ${ARCHIVE}
391#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
392#D-
393function IGCM_sys_Tree {
394  IGCM_debug_PushStack "IGCM_sys_Tree" $@
395  if ( $DEBUG_sys ) ; then
396    echo "IGCM_sys_Tree :" $@
397  fi
398
399  \ls -lR ${@}
400
401  IGCM_debug_PopStack "IGCM_sys_Tree"
402}
403
404#D-#==================================================
405#D-function IGCM_sys_Qsub
406#D-* Purpose: Qsub new job
407#D-* Examples:
408#D-
409function IGCM_sys_Qsub {
410  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
411  if ( $DEBUG_sys ) ; then
412    echo "IGCM_sys_Qsub :" $@
413  fi
414  typeset options status
415  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
416
417  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
418  status=$?
419
420  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
421  if [ ${status} -gt 0 ] ; then
422    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
423    IGCM_debug_Exit "IGCM_sys_Qsub"
424  else
425    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
426  fi
427  IGCM_debug_PopStack "IGCM_sys_Qsub"
428}
429
430#D-#==================================================
431#D-function IGCM_sys_QsubPost
432#D-* Purpose: Qsub new job on scalaire
433#D-* Examples:
434#D-
435function IGCM_sys_QsubPost {
436  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
437  if ( $DEBUG_sys ) ; then
438    echo "IGCM_sys_QsubPost :" $@
439  fi
440  typeset options status
441  options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
442
443  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
444  status=$?
445
446  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
447  if [ ${status} -gt 0 ] ; then
448    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
449    IGCM_debug_Exit "IGCM_sys_QsubPost"
450  else
451    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
452  fi
453  IGCM_debug_PopStack "IGCM_sys_QsubPost"
454}
455
456#D-*************************
457#D- File transfer functions
458#D-*************************
459#D-
460
461#D-#==================================================
462#D-function IGCM_sys_RmRunDir
463#D-* Purpose: rm tmpdir (dummy function most of the time batch
464#D-                      scheduler will do the job)
465#D-* Examples:
466#D-
467function IGCM_sys_RmRunDir {
468  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
469  if ( $DEBUG_sys ) ; then
470    echo "IGCM_sys_RmRunDir :" $@
471  fi
472
473  typeset status
474
475  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
476  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
477  status=$?
478
479  if [ ${status} -gt 0 ] ; then
480    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
481    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
482    IGCM_debug_Exit "IGCM_sys_RmRunDir"
483  else
484    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
485  fi
486  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
487}
488
489#D-#==================================================
490#D-function IGCM_sys_Put_Dir
491#D-* Purpose: Copy a complete directory on $(ARCHIVE)
492#D-* Examples:
493#D-
494function IGCM_sys_Put_Dir {
495  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
496  if ( $DEBUG_sys ) ; then
497    echo "IGCM_sys_Put_Dir :" $@
498  fi
499  if [ $DRYRUN = 0 ]; then
500    if [ ! -d ${1} ] ; then
501      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
502      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
503      return
504    fi
505
506    typeset status
507
508    # Only if we use rsync
509    #IGCM_sys_TestDirArchive $( dirname $2 )
510    #
511    #USUAL WAY
512    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
513    status=$?
514
515    if [ ${status} -gt 0 ] ; then
516      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
517      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
518      IGCM_debug_Exit "IGCM_sys_Put_Dir"
519    else
520      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
521    fi
522  fi
523  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
524}
525
526#D-#==================================================
527#D-function IGCM_sys_Get_Dir
528#D-* Purpose: Copy a complete directory from ${ARCHIVE}
529#D-* Examples:
530#D-
531function IGCM_sys_Get_Dir {
532  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
533  if ( $DEBUG_sys ) ; then
534    echo "IGCM_sys_Get_Dir :" $@
535  fi
536  if [ $DRYRUN = 0 ]; then
537    typeset NB_ESSAI DELAI status i
538    # number of tentative
539    NB_ESSAI=3
540    # time delay between tentative
541    DELAI=2
542
543    #
544    # USUAL WAY
545    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
546    ccc_hsm get -r $1
547
548    i=0
549    while [ $i -lt $NB_ESSAI ] ; do
550      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
551      status=$?
552      if [ ${status} -gt 0 ] ; then
553        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
554        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
555        sleep $DELAI
556      else
557        break
558      fi
559      (( i = i + 1 ))
560    done
561
562    if [ ${status} -gt 0 ] ; then
563      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
564      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
565      IGCM_debug_Exit "IGCM_sys_Get_Dir"
566    else
567      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
568    fi
569  fi
570  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
571}
572
573#D-#==================================================
574#D-function IGCM_sys_Put_Rest
575#D-* Purpose: Put computied restarts on ${ARCHIVE}.
576#D-           File and target directory must exist.
577#D-* Examples:
578#D-
579function IGCM_sys_Put_Rest {
580  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
581  if ( $DEBUG_sys ) ; then
582    echo "IGCM_sys_Put_Rest :" $@
583  fi
584  if [ $DRYRUN = 0 ]; then
585    if [ ! -f ${1} ] ; then
586      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
587      IGCM_debug_Exit "IGCM_sys_Put_Rest"
588    fi
589
590    typeset status
591    #
592    # USUAL WAY
593    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
594    status=$?
595
596#       #RSYNC WITH NETWORK SSH CALL
597#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
598#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
599
600#       #RSYNC WITH NFS USE
601#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
602#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
603
604#       status=$?
605#       IGCM_sys_Rsync_out $status
606
607#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
608#       (( status=status+$? ))
609
610    if [ ${status} -gt 0 ] ; then
611      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
612      [ -f ${1} ] && ls -l ${1}
613      [ -f ${2} ] && ls -l ${2}
614      [ -f ${2}/${1} ] && ls -l ${2}/${1}
615      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
616      IGCM_debug_Exit "IGCM_sys_Put_Rest"
617    else
618
619      if [ X${JobType} = XRUN ] ; then
620        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
621        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
622      fi
623
624      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
625    fi
626  fi
627  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
628}
629
630#D-#==================================================
631#D-function IGCM_sys_Put_Out
632#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
633#D-* Examples:
634#D-
635function IGCM_sys_Put_Out {
636  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
637  if ( $DEBUG_sys ) ; then
638    echo "IGCM_sys_Put_Out :" $@
639  fi
640
641  typeset NB_ESSAI DELAI status i exist skip
642  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
643
644  # number of tentative
645  NB_ESSAI=3
646  # time delay between tentative
647  DELAI=2
648
649  if [ $DRYRUN = 0 ]; then
650    if [ ! -f ${1} ] ; then
651      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
652      IGCM_debug_PopStack "IGCM_sys_Put_Out"
653      return 1
654    fi
655    #
656    IGCM_sys_MkdirArchive $( dirname $2 )
657    #
658    exist=false
659    skip=false
660    if [ -f $2 ] ; then
661      IGCM_debug_Print 1 "$2 already exist"
662      ccc_hsm get $2
663      exist=true
664      if [ "X$( diff $1 $2 )" = X ] ; then
665        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
666        skip=true
667      else
668        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
669        skip=false
670      fi
671    fi
672    #
673    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
674      IGCM_sys_Chmod u+w $2
675    fi
676
677    if [ X${skip} = Xfalse ] ; then
678      i=0
679      while [ $i -lt $NB_ESSAI ] ; do
680        # Identify file system
681        fileDeviceNumberInHex=$( stat -c %d $1 )
682        status=$?
683        if [ ${status} -gt 0 ] ; then
684          IGCM_debug_Exit "IGCM_sys_Put_Out"
685        fi
686        # Identify file system
687        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
688        status=$?
689        if [ ${status} -gt 0 ] ; then
690          IGCM_debug_Exit "IGCM_sys_Put_Out"
691        fi
692
693        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
694          # They are not on the same device. USUAL WAY
695          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
696          status=$?
697        else
698          # They are on the same device. NOT SO USUAL WAY
699          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
700          status=$?
701        fi
702        if [ ${status} -gt 0 ]; then
703          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
704          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
705          [ -f ${1} ] && ls -l ${1}
706          [ -f ${2} ] && ls -l ${2}
707          [ -f ${2}/${1} ] && ls -l ${2}/${1}
708          sleep $DELAI
709        else
710          break
711        fi
712        (( i = i + 1 ))
713      done
714    fi
715
716#       #RSYNC WITH NETWORK SSH CALL
717#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
718#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
719
720#       #RSYNC WITH NFS USE
721#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
722#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
723
724#       status=$?
725#       IGCM_sys_Rsync_out $status
726
727#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
728#       (( status=status+$? ))
729
730    if [ ${status} -gt 0 ] ; then
731      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
732      [ -f ${1} ] && ls -l ${1}
733      [ -f ${2} ] && ls -l ${2}
734      [ -f ${2}/${1} ] && ls -l ${2}/${1}
735      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
736      IGCM_debug_Exit "IGCM_sys_Put_Out"
737    else
738
739      if [ X${JobType} = XRUN ] ; then
740        if [ X${3} = X ] ; then
741          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
742          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
743        fi
744      fi
745
746      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
747    fi
748  fi
749  IGCM_debug_PopStack "IGCM_sys_Put_Out"
750  return 0
751}
752
753#D-#==================================================
754#D-function IGCM_sys_Get
755#D-* Purpose: Get a file from ${ARCHIVE}
756#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
757#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
758function IGCM_sys_Get {
759  IGCM_debug_PushStack "IGCM_sys_Get" $@
760
761  typeset DEST dm_liste target file_work
762  typeset NB_ESSAI DELAI status i
763
764  if ( $DEBUG_sys ) ; then
765    echo "IGCM_sys_Get :" $@
766  fi
767
768  # number of tentative
769  NB_ESSAI=3
770  # time delay between tentative
771  DELAI=2
772
773  if [ $DRYRUN -le 2 ]; then
774    if [ X${1} = X'/l' ] ; then
775      eval set +A dm_liste \${${2}}
776    else
777      eval set +A dm_liste ${1}
778    fi
779    eval DEST=\${${#}}
780    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
781    status=$?
782    if [ ${status} -gt 0 ] ; then
783      echo "WARNING IGCM_sys_Get : error code ${status}"
784      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
785      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
786    fi
787
788    #if [ ${status} -gt 0 ] ; then
789    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
790    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
791    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
792    #    sleep 30
793    #    echo "We try another time"
794    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
795    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
796    #    status=$?
797    #    if [ ${status} -gt 0 ] ; then
798    #      echo "ERROR IGCM_sys_Get : again demigration error :"
799    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
800    #      IGCM_debug_Exit "IGCM_sys_Get"
801    #    fi
802    #  else
803    #    echo "ERROR IGCM_sys_Get : demigration error :"
804    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
805    #    IGCM_debug_Exit "IGCM_sys_Get"
806    #  fi
807    #fi
808
809    #   #RSYNC WITH NETWORK SSH CALL
810    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
811    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
812
813    #   #RSYNC WITH NFS USE
814    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
815    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
816
817    #   status=$?
818    #   IGCM_sys_Rsync_out $status
819
820    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
821    #   (( status=status+$? ))
822
823    #USUAL WAY
824    if [ X${1} = X'/l' ] ; then
825      for target in ${dm_liste[*]} ; do
826        local_file=$( basename ${target} )
827        # test if the target file is present before the loop
828        IGCM_sys_TestFileArchive ${target}
829        status=$?
830        if [ ${status} -gt 0 ] ; then
831          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
832          IGCM_debug_Exit "IGCM_sys_Get"
833        else
834          i=0
835          while [ $i -lt $NB_ESSAI ] ; do
836            #if [ X${DoLink} = Xtrue ] ; then
837            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
838            #  status=$?
839            #  else
840            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
841            #  status=$?
842            #fi
843            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
844            status=$?
845            if [ ${status} -gt 0 ]; then
846              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
847              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
848              sleep $DELAI
849            else
850              break
851            fi
852            (( i = i + 1 ))
853          done
854          if [ ${status} -gt 0 ] ; then
855            echo "IGCM_sys_Get : error"
856            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
857            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
858            IGCM_debug_Exit "IGCM_sys_Get"
859          else
860            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
861          fi
862        fi
863      done
864    else
865      i=0
866      while [ $i -lt $NB_ESSAI ] ; do
867        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
868        status=$?
869        if [ ${status} -gt 0 ]; then
870          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
871          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
872          sleep $DELAI
873        else
874          break
875        fi
876        (( i = i + 1 ))
877      done
878      if [ ${status} -gt 0 ] ; then
879        echo "IGCM_sys_Get : error"
880        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
881        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
882        IGCM_debug_Exit "IGCM_sys_Get"
883      else
884        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
885      fi
886    fi
887  fi
888  IGCM_debug_PopStack "IGCM_sys_Get"
889}
890
891#D-#==================================================
892#D-function IGCM_sys_GetDate_Monitoring
893#D-* Purpose: get the last year for which the monitoring has been computed
894#D-* Examples:
895#D-
896function IGCM_sys_GetDate_Monitoring {
897  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
898  if ( $DEBUG_sys ) ; then
899    echo "IGCM_sys_GetDate_Monitoring :" $@
900  fi
901
902  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
903
904  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
905}
906
907#D-#==================================================
908#D-function IGCM_sys_Dods_Rm
909#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
910#D-* Examples:
911#D-
912function IGCM_sys_Dods_Rm {
913  if ( $DEBUG_sys ) ; then
914    echo "IGCM_sys_Dods_Rm :" $@
915  fi
916  typeset status
917  if [ $DRYRUN = 0 ]; then
918
919#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
920#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
921#      echo "Nothing has been done."
922#      return
923#    fi
924
925    if [ "$#" -eq 1 ]; then
926      ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
927      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
928      status=$?
929    else
930      ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
931      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
932      status=$?
933    fi
934
935#    if [ ${status} -gt 0 ] ; then
936#      echo "IGCM_sys_Dods_Rm : error."
937#      cat out_dods_rm
938#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
939#    else
940#      rm out_dods_rm
941#    fi
942
943  fi
944  return $status
945}
946
947#D-#==================================================
948#D-function IGCM_sys_Dods_Cp
949#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
950#D-* Examples:
951#D-
952function IGCM_sys_Dods_Cp {
953  if ( $DEBUG_sys ) ; then
954    echo "IGCM_sys_Dods_Cp :" $@
955  fi
956  typeset status
957  if [ $DRYRUN = 0 ]; then
958
959#    if [ ! -d ${R_SAVE}/${1} ] ; then
960#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
961#      echo "Nothing has been done."
962#      return
963#    fi
964
965    ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/bin/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
966    $(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
967    status=$?
968
969#       if [ ${status} -gt 0 ] ; then
970#           echo "IGCM_sys_Dods_Cp : error."
971#           cat out_dods_cp
972#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
973#       else
974#           rm out_dods_cp
975#       fi
976
977  fi
978  return $status
979}
980
981#D-#==================================================
982#D-function IGCM_sys_Put_Dods
983#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
984#D-* Examples:
985#D-
986function IGCM_sys_Put_Dods {
987  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
988  if ( $DEBUG_sys ) ; then
989    echo "IGCM_sys_Put_Dods :" $@
990  fi
991  typeset status
992  if [ $DRYRUN = 0 ]; then
993    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
994      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
995      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
996      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
997      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
998      return
999    fi
1000
1001    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1002      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1003      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1004      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1005      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1006      return
1007    fi
1008    #
1009    if [ -d ${R_SAVE}/${1} ] ; then
1010      cd ${R_SAVE}
1011    elif [ -d ${R_FIGR}/${1} ] ; then
1012      cd ${R_FIGR}
1013    fi
1014
1015    IGCM_sys_Dods_Rm ${1}
1016    IGCM_sys_Dods_Cp ${1}
1017    status=0
1018
1019    if [ ${status} -gt 0 ] ; then
1020      echo "IGCM_sys_Put_Dods : error."
1021      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1022    fi
1023  fi
1024  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1025}
1026
1027##############################################################
1028# REBUILD OPERATOR
1029
1030#D-#==================================================
1031#D-function IGCM_sys_sync
1032#D-* Purpose: flush buffer on disk
1033#D-* Examples:
1034#D-
1035function IGCM_sys_sync {
1036  IGCM_debug_PushStack "IGCM_sys_sync" $@
1037  if ( $DEBUG_sys ) ; then
1038    echo "IGCM_sys_sync :" $@
1039  fi
1040
1041  /bin/sync
1042
1043  IGCM_debug_PopStack "IGCM_sys_sync"
1044}
1045
1046############################################################
1047# Activate Running Environnment Variables
1048
1049#D-#==================================================
1050#D-function IGCM_sys_activ_variables
1051#D-* Purpose: set environement variables prior to execution
1052#D-* Examples:
1053#D-
1054function IGCM_sys_activ_variables {
1055  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1056  if ( $DEBUG_sys ) ; then
1057    echo "IGCM_sys_activ_variables"
1058  fi
1059
1060# --------------------------------------------------------------------
1061#D- MODULE specifications
1062# --------------------------------------------------------------------
1063  module purge > /dev/null 2>&1
1064  if [ X${compilerVersion} = XV17 ] ; then
1065    IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V17"
1066  else
1067    IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1068  fi
1069  ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_irene > /dev/null 2>&1
1070  . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_irene > /dev/null 2>&1
1071  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene. "
1072  module list
1073
1074  # Keep track of this
1075  [ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
1076  [ ! X${TaskType} = Xchecking ] && module list
1077
1078# --------------------------------------------------------------------
1079#D- MPI specifications
1080# --------------------------------------------------------------------
1081
1082# --------------------------------------------------------------------
1083#D- Other specifications
1084# --------------------------------------------------------------------
1085
1086  ulimit -s unlimited
1087
1088  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1089}
1090
1091############################################################
1092# Desactivate Running Environnment Variables
1093
1094#D-#==================================================
1095#D-function IGCM_sys_desactiv_variables
1096#D-* Purpose: unset environement variables after execution
1097#D-* Examples:
1098#D-
1099function IGCM_sys_desactiv_variables {
1100  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1101  if ( $DEBUG_sys ) ; then
1102    echo "IGCM_sys_desactiv_variables"
1103  fi
1104# --------------------------------------------------------------------
1105#D- MPI specifications
1106# --------------------------------------------------------------------
1107
1108# --------------------------------------------------------------------
1109#D- Other specifications
1110# --------------------------------------------------------------------
1111
1112  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1113}
1114
1115############################################################
1116# Update job headers to be used by the scheduler
1117
1118#D-#==================================================
1119#D-function IGCM_sys_updateHeaders
1120#D-* Purpose: Update job headers to be used by the scheduler
1121#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1122#D-
1123function IGCM_sys_updateHeaders {
1124  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1125  if ( $DEBUG_sys ) ; then
1126    echo "IGCM_sys_updateHeaders"
1127  fi
1128  typeset file
1129  file=$1
1130
1131  if [ ${executionType} -eq 1 ] ; then
1132    # MPMD + MPI
1133    sed -e "/::openMPthreads::/d"                  \
1134        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1135        -e "/#MSUB -x/d"                           \
1136        -e "/--cpu_bind=none/d"                    \
1137      ${file} > ${file}.tmp
1138
1139  elif [ ${executionType} -eq 2 ] ; then
1140    # MPMD + MPI + OMP : mpirun/ccc_mprun/error
1141    if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1142      sed -e "/::openMPthreads::/d"                  \
1143          -e "s/::JobNumProcTot::/${coreNumber}/"    \
1144        ${file} > ${file}.tmp
1145    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1146      sed -e "/::openMPthreads::/d"                  \
1147          -e "s/::JobNumProcTot::/${coreNumber}/"    \
1148          -e "/--cpu_bind=none/d"                    \
1149        ${file} > ${file}.tmp
1150    else
1151     IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
1152     IGCM_debug_Exit "IGCM_sys_updateHeaders"
1153    fi
1154  elif [ ${executionType} -eq 3 ] ; then
1155    # SPMD + MPI/OMP
1156    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1157        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1158        -e "/#MSUB -x/d"                           \
1159        -e "/--cpu_bind=none/d"                    \
1160      ${file} > ${file}.tmp
1161
1162  elif [ ${executionType} -eq 4 ] ; then
1163    # SPMD + MPI only
1164    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1165        -e "/::openMPthreads::/d"                  \
1166        -e "/#MSUB -x/d"                           \
1167        -e "/--cpu_bind=none/d"                    \
1168      ${file} > ${file}.tmp
1169
1170  elif [ ${executionType} -eq 5 ] ; then
1171    # SPMD + OMP only
1172    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1173        -e "/::JobNumProcTot::/d"                  \
1174        -e "/#MSUB -x/d"                           \
1175        -e "/--cpu_bind=none/d"                    \
1176      ${file} > ${file}.tmp
1177
1178  elif [ ${executionType} -eq 6 ] ; then
1179    # SEQUENTIAL THEN
1180    sed -e "s/::JobNumProcTot::/1/"                \
1181        -e "/::openMPthreads::/d"                  \
1182        -e "/#MSUB -x/d"                           \
1183        -e "/--cpu_bind=none/d"                    \
1184      ${file} > ${file}.tmp
1185
1186  fi
1187
1188  IGCM_sys_Mv ${file}.tmp ${file}
1189
1190  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1191}
1192
1193############################################################
1194# Build MPI/OMP scripts run file (dummy function)
1195
1196#D-#==================================================
1197#D-function IGCM_sys_build_run_file
1198#D-* Purpose: build run file (deprecated)
1199#D-* Examples:
1200#D-
1201function IGCM_sys_build_run_file {
1202  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1203}
1204
1205############################################################
1206# Build MPI/OMP scripts
1207
1208#D-#==================================================
1209#D-function IGCM_sys_build_execution_scripts
1210#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1211#D-* Examples:
1212#D-
1213function IGCM_sys_build_execution_scripts
1214{
1215  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1216  if ( $DEBUG_sys ) ; then
1217    echo "IGCM_sys_build_execution_scripts " $@
1218  fi
1219
1220  EXECUTION=${HOST_MPIRUN_COMMAND}
1221
1222  if ( ${OK_PARA_MPMD} ) ; then
1223
1224    # MPMD mode
1225    # 1 MPI only
1226    # 2 MPI/OpenMP mpirun method
1227    # 3 MPI/OpenMP ccc_mprun method
1228
1229    if [ -f run_file ] ; then
1230      IGCM_sys_Rm -f run_file
1231    fi
1232    touch run_file
1233
1234    # case 1 : Only MPI (MPMD)
1235    if  ( ! ${OK_PARA_OMP} ) ; then
1236
1237      # Build run_file
1238
1239      # First loop on the components for the coupler ie oasis (only if oasis3)
1240      # the coupler ie oasis3 must be the first one
1241      for comp in ${config_ListOfComponents[*]} ; do
1242
1243        eval ExeNameIn=\${config_Executable_${comp}[0]}
1244        eval ExeNameOut=\${config_Executable_${comp}[1]}
1245
1246        # for CPL component only
1247        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1248
1249          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1250          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1251          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1252        fi
1253      done
1254
1255      # Then second loop on the components
1256      for comp in ${config_ListOfComponents[*]} ; do
1257
1258        eval ExeNameIn=\${config_Executable_${comp}[0]}
1259        eval ExeNameOut=\${config_Executable_${comp}[1]}
1260
1261        # Only if we really have an executable for the component and not the coupler ie oasis:
1262        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1263
1264          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1265          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1266          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1267        fi
1268      done
1269
1270      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1271
1272      IGCM_sys_Chmod u+x run_file
1273      if ( $DEBUG_sys ) ; then
1274        echo "run_file contains : "
1275        cat run_file
1276      fi
1277
1278    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1279
1280    # 2 MPI/OpenMP mpirun method
1281
1282      # Use of mpirun instead of ccc_mprun
1283      EXECUTION="time mpirun"
1284
1285      #  Hosts treatment
1286      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1287
1288      i=0
1289      rm -f hosts rankfile
1290      IGCM_debug_Print 1 "sys Curie, Hosts available :"
1291      for nodes in `cat hosts.tmp`
1292      do
1293        host[$i]=$nodes
1294        echo "${host[$i]}" >> hosts
1295        IGCM_debug_Print 1 ${host[$i]}
1296        i=$((i+1))
1297      done
1298      rm -f hosts.tmp
1299
1300      listnodes=${host[*]}
1301
1302      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1303
1304      # Initialisation
1305      rank=0
1306      current_core=0
1307      init_exec=n
1308
1309      # Loop on the components
1310      for comp in ${config_ListOfComponents[*]} ; do
1311
1312        eval ExeNameIn=\${config_Executable_${comp}[0]}
1313        eval ExeNameOut=\${config_Executable_${comp}[1]}
1314
1315        # Not possible if oasis has an executable (i.e old version of oasis3)
1316        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1317          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1318          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1319          IGCM_debug_Verif_Exit
1320        fi
1321
1322        # Only if we really have an executable for the component :
1323        if [ "X${ExeNameOut}" != X\"\" ] ; then
1324
1325          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1326          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1327          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1328
1329          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1330          echo ""  >> script_${ExeNameOut}.ksh
1331          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1332
1333            # Check if the number of threads is correct
1334            case ${comp_proc_omp_loc} in
1335            2|4|8|16)
1336              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1337              ;;
1338            *)
1339              IGCM_debug_Exit "ERROR with OMP parameters !"
1340              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1341              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1342              IGCM_debug_Verif_Exit
1343              ;;
1344            esac
1345            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1346            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1347            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1348            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1349          fi
1350
1351          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1352          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1353          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1354          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1355          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1356
1357          if [ ${init_exec} = y ] ; then
1358            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1359          else
1360            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1361            init_exec=y
1362          fi
1363
1364          # Build rankfile : method used to assign cores and nodes for the MPI process
1365          # Ex :
1366          #rank 0=curie5296 slot=0,1,2,3
1367          #rank 1=curie5296 slot=4,5,6,7
1368          # Example of final command :
1369          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1370          # with script_lmdz.x.ksh :
1371          # #!/bin/ksh
1372          #export KMP_STACKSIZE=3g
1373          #export KMP_LIBRARY=turnaround
1374          #export MKL_SERIAL=YES
1375          #OMP_NUM_THREADS=4
1376          #./lmdz.x
1377          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1378              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1379          else
1380              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1381          fi
1382          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1383             (( index_host = current_core / NB_CORE_PER_NODE ))
1384             host_value=${host[${index_host}]}
1385             (( slot =  current_core % NB_CORE_PER_NODE ))
1386             virg=","
1387             string_final=""
1388             for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1389                 string=$index$virg
1390                 string_final=$string_final$string
1391             done
1392             string_final=$( echo $string_final | sed "s/.$//" )
1393             echo "rank $rank=$host_value slot=$string_final" >> rankfile
1394             (( rank = rank + 1 ))
1395             (( current_core = current_core + offset_comp_proc_loc ))
1396          done
1397        fi
1398      done
1399
1400    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1401
1402    # 3 MPI/OpenMP ccc_mprun method
1403
1404    # MPI-OpenMP (MPMD)
1405    # example of run_file
1406    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1407    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1408    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1409
1410      # Loop on the components
1411      for comp in ${config_ListOfComponents[*]} ; do
1412
1413        eval ExeNameIn=\${config_Executable_${comp}[0]}
1414        eval ExeNameOut=\${config_Executable_${comp}[1]}
1415
1416        # Not possible if oasis has an executable (i.e old version of oasis3)
1417        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1418          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1419          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1420          IGCM_debug_Verif_Exit
1421        fi
1422
1423        # Only if we really have an executable for the component :
1424        if [ "X${ExeNameOut}" != X\"\" ] ; then
1425
1426          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1427          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1428
1429          # Build script files
1430
1431          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1432          echo ""  >> script_${ExeNameOut}.ksh
1433          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1434
1435            # Check if the number of threads is correct
1436            case ${comp_proc_omp_loc} in
1437            2|4|8|12|16|24)
1438              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1439              ;;
1440            *)
1441              IGCM_debug_Exit "ERROR with OMP parameters !"
1442              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1443              IGCM_debug_Print 2 "Only 2,4,8,12,16,24 as number of OMP threads are possible "
1444              IGCM_debug_Verif_Exit
1445              ;;
1446            esac
1447            ### only for LMDZ?
1448            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1449            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1450            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1451            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1452
1453          fi
1454
1455          # to have out/err per process on different files
1456          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1457          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1458
1459          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1460
1461          # Complete run_file
1462
1463          echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1464
1465        fi
1466      done
1467
1468      ## variable added to stop after 60s instead of 600s by default.
1469      ## This is used when no error comes from executables and when something stopped an executable without notice.
1470      export SLURM_WAIT=60
1471
1472      ## module advised by TGCC (instead of 2 variables)
1473      module load feature/bridge/heterogenous_mpmd
1474
1475      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1476
1477      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1478
1479      IGCM_sys_Chmod u+x run_file
1480      if ( $DEBUG_sys ) ; then
1481        echo "run_file contains : "
1482        cat run_file
1483      fi
1484
1485    else
1486
1487      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1488
1489    fi # 1 2 or 3 if ${OK_PARA_MPMD}
1490
1491  else
1492  # Only one executable (SPMD mode).
1493
1494    for comp in ${config_ListOfComponents[*]} ; do
1495
1496      # Only if we really have an executable for the component :
1497      eval ExeNameOut=\${config_Executable_${comp}[1]}
1498      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1499
1500        # Build script files
1501
1502        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1503        echo ""  >> script_${ExeNameOut}.ksh
1504        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1505
1506        if ( ${OK_PARA_OMP} ) ; then
1507          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1508          echo ""  >> script_${ExeNameOut}.ksh
1509          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1510          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1511          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1512          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1513        fi
1514
1515        if  ( ${OK_PARA_MPI} ) ; then
1516          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1517          # Default : ccc_mprun used if nb_proc gt 1
1518          # to have out/err per process on different files
1519          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1520          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1521          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1522        else
1523          # Default : ccc_mprun is NOT used if nb_proc eq 1
1524          # to have out/err per process on different files
1525          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1526          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1527          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1528        fi
1529
1530        IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
1531        cat script_${ExeNameOut}.ksh
1532
1533      fi
1534
1535    done
1536
1537  fi # ${OK_PARA_MPMD}
1538
1539  IGCM_debug_Print 1 "sys Curie : execution command is "
1540  IGCM_debug_Print 1 "$EXECUTION"
1541
1542  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1543}
1544
1545#D-#==================================================
1546#D-function IGCM_sys_check_path
1547#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1548#D-* do not point to an important use directory. Stop immediately in that case.
1549#D-* Examples:
1550#D-
1551function IGCM_sys_check_path {
1552  IGCM_debug_PushStack "IGCM_sys_check_path"
1553  if ( $DEBUG_sys ) ; then
1554    echo "IGCM_sys_check_path"
1555  fi
1556
1557  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1558    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1559    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1560    IGCM_debug_Exit "This will stop the job"
1561  fi
1562  IGCM_debug_PopStack "IGCM_sys_check_path"
1563}
1564
1565#D-#==================================================
1566#D-function IGCM_sys_check_quota
1567#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1568#D-* Examples:
1569#D-
1570function IGCM_sys_check_quota {
1571  IGCM_debug_PushStack "IGCM_sys_check_quota"
1572  if ( $DEBUG_sys ) ; then
1573    echo "IGCM_sys_check_quota"
1574  fi
1575  # Limit of quota (in %)
1576  limit_quota=90
1577
1578  # Check of the volume
1579  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1580  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1581
1582  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1583
1584    unit_avail=${volume_avail: -1}
1585    unit_quota=${volume_quota: -1}
1586
1587    if [ "${unit_quota}" = "*" ] ; then
1588      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1589      IGCM_debug_Print 1 "More than 100% of your quota is used"
1590      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1591      IGCM_debug_Print 1 "You must have more than 10% available to run"
1592      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1593      IGCM_debug_Verif_Exit
1594    fi
1595
1596    temp_avail=${volume_avail%%${volume_avail: -1}*}
1597    temp_quota=${volume_quota%%${volume_quota: -1}*}
1598
1599    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1600
1601    # Convertion
1602      if [ ${volume_avail: -1} = "T" ] ; then
1603        (( temp_avail = temp_avail * 1000000000000 ))
1604      elif [ ${volume_avail: -1} = "G" ] ; then
1605        (( temp_avail = temp_avail * 1000000000 ))
1606      elif [ ${volume_avail: -1} = "M" ] ; then
1607        (( temp_avail = temp_avail * 1000000 ))
1608      elif [ ${volume_avail: -1} = "k" ] ; then
1609        (( temp_avail = temp_avail * 1000 ))
1610      else
1611        (( temp_avail = volume_avail ))
1612      fi
1613      if [ ${volume_quota: -1} = "T" ] ; then
1614        (( temp_quota = temp_quota * 1000000000000 ))
1615      elif [ ${volume_quota: -1} = "G" ] ; then
1616        (( temp_quota = temp_quota * 1000000000 ))
1617      elif [ ${volume_quota: -1} = "M" ] ; then
1618        (( temp_quota = temp_quota * 1000000 ))
1619      elif [ ${volume_quota: -1} = "k" ] ; then
1620        (( temp_quota = temp_quota * 1000 ))
1621      else
1622        (( temp_quota = volume_quota ))
1623      fi
1624    fi
1625
1626    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1627#    echo "volume ratio is " $quota_volume
1628
1629    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1630      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1631      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1632      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1633      IGCM_debug_Print 1 "You must have more than 10% available to run"
1634      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1635      IGCM_debug_Verif_Exit
1636    fi
1637
1638  fi
1639
1640# Check of the number of inodes
1641
1642  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1643  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1644
1645  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1646
1647    unit_avail=${inode_avail: -1}
1648    unit_quota=${inode_quota: -1}
1649
1650    if [ "${unit_quota}" = "*" ] ; then
1651      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1652      IGCM_debug_Print 1 "More than 100% of your quota is used"
1653      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1654      IGCM_debug_Print 1 "You must have more than 10% available to run"
1655      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1656      IGCM_debug_Verif_Exit
1657    fi
1658
1659    temp_avail=${inode_avail%%${inode_avail: -1}*}
1660    temp_quota=${inode_quota%%${inode_quota: -1}*}
1661
1662    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1663
1664    # Convertion
1665      if [ ${inode_avail: -1} = "T" ] ; then
1666        (( temp_avail = temp_avail * 1000000000000 ))
1667      elif [ ${inode_avail: -1} = "G" ] ; then
1668        (( temp_avail = temp_avail * 1000000000 ))
1669      elif [ ${inode_avail: -1} = "M" ] ; then
1670        (( temp_avail = temp_avail * 1000000 ))
1671      elif [ ${inode_avail: -1} = "k" ] ; then
1672        (( temp_avail = temp_avail * 1000 ))
1673      else
1674        (( temp_avail = inode_avail ))
1675      fi
1676
1677      if [ ${inode_quota: -1} = "T" ] ; then
1678        (( temp_quota = temp_quota * 1000000000000 ))
1679      elif [ ${inode_quota: -1} = "G" ] ; then
1680        (( temp_quota = temp_quota * 1000000000 ))
1681      elif [ ${inode_quota: -1} = "M" ] ; then
1682        (( temp_quota = temp_quota * 1000000 ))
1683      elif [ ${inode_quota: -1} = "k" ] ; then
1684        (( temp_quota = temp_quota * 1000 ))
1685      else
1686        (( temp_quota = inode_quota ))
1687      fi
1688    fi
1689    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1690#    echo "inode ratio is " $quota_inode
1691
1692    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1693      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1694      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1695      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1696      IGCM_debug_Print 1 "You must have more than 10% available to run"
1697      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1698      IGCM_debug_Verif_Exit
1699    fi
1700  fi
1701  IGCM_debug_PopStack "IGCM_sys_check_quota"
1702}
1703
1704#D-#==================================================
1705#D-function IGCM_sys_projectAccounting
1706#D-* Purpose: store project accounting information in a file
1707#D-* Examples:
1708#D-
1709function IGCM_sys_projectAccounting {
1710  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1711  if ( $DEBUG_sys ) ; then
1712    echo "IGCM_sys_check_quota"
1713  fi
1714
1715  ssh curie91 /usr/bin/ccc_myproject > $1
1716
1717  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1718}
1719
1720#D-#==================================================
1721#D-function IGCM_sys_getJobSchedulerID
1722#D-* Purpose: Get the job ID during execution
1723#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1724#D-
1725function IGCM_sys_getJobSchedulerID {
1726  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1727  if ( $DEBUG_sys ) ; then
1728    echo "IGCM_sys_getJobSchedulerID"
1729  fi
1730
1731  eval ${1}=${BRIDGE_MSUB_JOBID}
1732
1733  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1734}
1735
1736#D-#==================================================
1737#D-function IGCM_sys_GetJobID
1738#D-* Purpose: Get the job ID from the JobName
1739#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1740#D-
1741function IGCM_sys_GetJobID {
1742  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1743  if ( $DEBUG_sys ) ; then
1744    echo "IGCM_sys_GetJobID"
1745  fi
1746
1747  # With -f option, the full job name is given in the last column
1748  ID=$( ccc_mstat -f -u $2 | \
1749        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1750
1751  eval ${3}=${ID}
1752  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1753}
1754
1755#D-#==================================================
1756#D-function IGCM_sys_CountJobInQueue
1757#D-* Purpose: Count number of users job
1758#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1759#D-
1760function IGCM_sys_CountJobInQueue {
1761  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1762  if ( $DEBUG_sys ) ; then
1763    echo "IGCM_sys_CountJobInQueue"
1764  fi
1765
1766  # With -f option, the full job name is given in the last column
1767  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1768
1769  eval ${2}=${NbRun}
1770
1771  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1772}
1773
1774#D-#==================================================
1775#D-function IGCM_sys_ListJobInQueue
1776#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1777#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1778#D-
1779function IGCM_sys_ListJobInQueue {
1780  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1781  if ( $DEBUG_sys ) ; then
1782    echo "IGCM_sys_ListJobInQueue"
1783  fi
1784
1785  # With -f option, the full job name is given in the last column
1786  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1787                                        '( $2  == User      && \
1788                                           $NF != /TS/      && \
1789                                           $NF !~ /PACK/    && \
1790                                           $NF !~ /REBUILD/ && \
1791                                           $NF !~ /pack/ )     \
1792                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1793
1794  eval set -A ${2} ${JobList[*]}
1795
1796  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1797}
1798
1799#D-#==================================================
1800#D-function IGCM_sys_atlas
1801#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1802#D-* Examples:
1803#D-
1804function IGCM_sys_atlas {
1805  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1806  if ( $DEBUG_sys ) ; then
1807    echo "IGCM_sys_atlas :" $@
1808  fi
1809
1810  typeset status
1811
1812  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1813  status=$?
1814  if [ ${status} -gt 0 ] ; then
1815    echo "IGCM_sys_atlas : error code ${status}"
1816    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1817    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1818    IGCM_debug_PopStack "IGCM_sys_atlas"
1819    return 1
1820  else
1821    IGCM_debug_PopStack "IGCM_sys_atlas"
1822    return 0
1823  fi
1824
1825  IGCM_debug_PopStack "IGCM_sys_atlas"
1826}
1827
1828#D-#==================================================
1829#D-function IGCM_sys_rebuild_nemo
1830#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1831#D-* Examples:
1832#D-
1833
1834function IGCM_sys_rebuild_nemo {
1835  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1836  if ( $DEBUG_sys ) ; then
1837    echo "IGCM_sys_rebuild_nemo :" $@
1838  fi
1839
1840  $( ccc_home -u igcmg)/Tools/curie/bin/rebuild_nemo ${1} ${2}
1841
1842  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1843}
1844
Note: See TracBrowser for help on using the repository browser.