source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_jeanzay.ksh @ 1534

Last change on this file since 1534 was 1534, checked in by acosce, 4 years ago

commands dods_cp and dods_rm on jeanzay were renamed thredds_cp and thredds_rm

  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 59.5 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Jean-Zay
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host user names project maxCpuTime
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=jeanzay
71# add default project set by ins_job
72typeset PROJECT=$( echo ${SLURM_JOB_ACCOUNT:=::default_project::} | cut -c "1-3" ) 
73### jobWarningDelay in seconds
74##typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75 
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=sbatch}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Access to module command
93#====================================================
94# See ticket #348
95# Following path should be used according to assist IDRIS
96. /gpfslocalsup/spack_soft/environment-modules/current/init/ksh
97# Following old path gives error messages while loading modules but it still works
98#. /usr/share/Modules/init/ksh
99
100#====================================================
101# Set environment tools (ferret, nco, cdo, rebuild, ...)
102#====================================================
103if [ X${TaskType} = Xcomputing ] ; then
104  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
105else
106  module purge > /dev/null 2>&1
107  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay > /dev/null 2>&1
108
109  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_atlas_jeanzay > /dev/null 2>&1
110
111#  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
112  export UVCDAT_ANONYMOUS_LOG=FALSE
113fi
114
115# FYI
116[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
117[ ! X${TaskType} = Xchecking ] && module list
118
119# For AddNoise (not yet AddPertu3DOCE)
120export PATH=${PATH}:/gpfswork/rech/psl/commun/Tools/AddNoise/bin
121
122# For rebuild
123export PATH=${PATH}:/gpfswork/rech/psl/commun/Tools/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
124
125#====================================================
126# Host specific DIRECTORIES
127#====================================================
128
129#====================================================
130#- MirrorlibIGCM for frontend
131typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
132
133#====================================================
134#- libIGCM_POST for frontend
135typeset -r libIGCM_POST=${libIGCM}
136
137#====================================================
138#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
139typeset -r R_EXE="${MODIPSL}/bin"
140
141#====================================================
142#- SUBMIT_DIR : submission dir
143typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${SLURM_SUBMIT_DIR}}
144
145#====================================================
146#- IN
147typeset -r R_IN=${R_IN:=/gpfswork/rech/psl/commun/IGCM}
148
149#====================================================
150#- RUN_DIR_PATH : Temporary working directory (=> TMP)
151typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCH}/RUN_DIR/${SLURM_JOBID}_${$}}
152
153#====================================================
154#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
155typeset -r OUTCOMMAND_PATH=/tmp
156
157#====================================================
158#- HOST_MPIRUN_COMMAND
159typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time srun "}
160
161#====================================================
162#- Max number of arguments passed to nco operator or demigration command
163UNIX_MAX_LIMIT=360
164
165#====================================================
166#- set PackDefault to true
167PackDefault=true
168
169#====================================================
170#- Default number of MPI task for IPSL coupled model
171#- required for backward compatibility
172#-
173DEFAULT_NUM_PROC_OCE=5
174DEFAULT_NUM_PROC_CPL=1
175(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
176
177#====================================================
178#- Number of computing cores per node
179#-
180NB_CORE_PER_NODE=40
181
182#D-#==================================================
183#D-function IGCM_sys_defineArchives
184#D-* Purpose:
185#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
186#D-* Define ARCHIVE : Dedicated to large files
187#D-* Define STORAGE : Dedicated to small/medium files
188#D-* Define R_OUT   : Output tree located on ARCHIVE
189#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monsitoring and atlas, and/or small files)
190#D-* Define R_BUF   : Output tree located on SCRATCH hosting files waiting for rebuild or pack processes
191#D-* if SpaceName=TEST everything is stored on SCRATCH
192#D-* Examples:
193#D-
194function IGCM_sys_defineArchives {
195  IGCM_debug_PushStack "IGCM_sys_defineArchives"
196
197  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
198  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = DEFAULT ]; then
199    # The variable DataProject is not set in in section UserChoices in config.card or it is set to DEFAULT
200    # Default option: Set DataProject according to the project used for submission
201    DataProject=${PROJECT}
202  else
203    # Use the project set by the variable DataProject in section UserChoices in config.card
204    IGCM_debug_Print 1 "Use DataProject as set in config.card : ${config_UserChoices_DataProject}"
205    DataProject=${config_UserChoices_DataProject}
206  fi
207  IGCM_debug_Print 1 "Data project for output to be used: ${DataProject}"
208
209  #====================================================
210  #- RUN_DIR_PATH : Temporary working directory (=> TMP)
211  #- DataProject = for is the dataproject for Training account at Idris
212  #- they are not store in rech but in another partition
213  if [ X${DataProject} = Xfor  ]; then
214      typeset -r RUN_DIR_PATH=$SCRATCH/RUN_DIR/${SLURM_JOBID}_${$}
215  else
216      typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/gpfsscratch/rech/${DataProject}/${LOGIN}/RUN_DIR/${SLURM_JOBID}_${$}}
217  fi
218
219  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
220    #====================================================
221    #- ARCHIVE (dedicated to large files)
222    ARCHIVE=${config_UserChoices_ARCHIVE}
223  else
224      if [ X${DataProject} = Xfor  ]; then
225          ARCHIVE=$STORE
226      else
227          #====================================================
228          #- ARCHIVE (dedicated to large files)
229          ARCHIVE=/gpfsstore/rech/${DataProject}/${LOGIN}
230      fi
231  fi
232
233  if [ ! X${config_UserChoices_STORAGE} = X ]; then
234    #====================================================
235    #- STORAGE (dedicated to small/medium files)
236    STORAGE=${config_UserChoices_STORAGE}
237  else
238    #====================================================
239    #- STORAGE (dedicated to small/medium files)
240      if [ X${DataProject} = Xfor  ]; then
241          STORAGE=$WORK
242      else
243          STORAGE=/gpfswork/rech/${DataProject}/${LOGIN}
244      fi
245  fi
246
247  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
248    #====================================================
249    #- R_OUT
250    if [ X${DataProject} = Xfor  ]; then
251        R_OUT=$SCRATCH/IGCM_OUT
252    else
253        R_OUT=/gpfsscratch/rech/${DataProject}/${LOGIN}/IGCM_OUT
254    fi
255    #====================================================
256    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
257    R_FIG=${R_OUT}
258
259    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
260
261  else
262    #====================================================
263    #- R_OUT
264    R_OUT=${ARCHIVE}/IGCM_OUT
265
266    #====================================================
267    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
268    R_FIG=${STORAGE}/IGCM_OUT
269  fi
270
271  #====================================================
272  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
273  CMIP6_BUF=${STORAGE}/IGCM_OUT
274
275  #====================================================
276  #- R_BUF (ONLY FOR double copy an scratch)
277 
278  if [ X${DataProject} = Xfor  ]; then
279      R_BUF=$SCRATCH/IGCM_OUT
280  else
281      R_BUF=/gpfsscratch/rech/${DataProject}/${LOGIN}/IGCM_OUT
282  fi
283
284  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
285  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
286  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
287
288  IGCM_debug_PopStack "IGCM_sys_defineArchives"
289}
290
291#D-#==================================================
292#D-function IGCM_sys_RshArchive
293#D-* Purpose: Archive rsh command
294#D-* Examples:
295#D-
296function IGCM_sys_RshArchive {
297  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
298  /bin/ksh <<-EOF
299    ${@}
300EOF
301  status=$?
302  if [ ${status} -gt 0 ] ; then
303    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
304    IGCM_debug_Exit "IGCM_sys_RshArchive"
305  fi
306  IGCM_debug_PopStack "IGCM_sys_RshArchive"
307}
308
309#D-#==================================================
310#D-function IGCM_sys_RshArchive_NoError
311#D-* Purpose: Archive rsh command, without error
312#D-*          used only in monitoring.job
313#D-* Examples:
314#D-
315function IGCM_sys_RshArchive_NoError {
316  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
317  /bin/ksh <<-EOF
318    ${@} 2> /dev/null
319EOF
320  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
321}
322
323#D-#==================================================
324#D-function IGCM_sys_MkdirArchive
325#D-* Purpose: Mkdir on Archive
326#D-* Examples:
327#D-
328function IGCM_sys_MkdirArchive {
329  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
330  if ( $DEBUG_sys ) ; then
331    echo "IGCM_sys_MkdirArchive :" $@
332  fi
333  #- creation de repertoire sur le serveur fichier
334  if [ ! -d ${1} ]; then
335    \mkdir -p $1
336    status=$?
337
338    if [ ${status} -gt 0 ] ; then
339      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
340      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
341    fi
342  fi
343  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
344}
345
346#D-#==================================================
347#D-function IGCM_sys_TestDirArchive
348#D-* Purpose: Test Directory that must exists on Archive
349#D-* Examples:
350#D-
351function IGCM_sys_TestDirArchive {
352  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
353  if ( $DEBUG_sys ) ; then
354    echo "IGCM_sys_TestDirArchive :" $@
355  fi
356  typeset ExistFlag
357  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
358  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
359  return ${ExistFlag}
360}
361
362#D-#==================================================
363#D-function IGCM_sys_IsFileArchived
364#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
365#D-* Examples:
366#D-
367function IGCM_sys_IsFileArchived {
368  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
369  if ( $DEBUG_sys ) ; then
370    echo "IGCM_sys_IsFileArchived :" $@
371  fi
372  typeset IsArchivedFlag
373# To be implemented on Jean Zay
374#  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
375  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
376
377  return ${IsArchivedFlag}
378}
379
380#D-#==================================================
381#D-function IGCM_sys_TestFileArchive
382#D-* Purpose: Test file that must NOT EXISTS on Archive
383#D-* Examples:
384#D-
385function IGCM_sys_TestFileArchive {
386  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
387  typeset ExistFlag
388  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
389  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
390
391  return ${ExistFlag}
392}
393
394#D-#==================================================
395#D-function IGCM_sys_CountFileArchive
396#D-* Purpose: Count files on Archive filesystem
397#D-* Examples:
398#D-
399function IGCM_sys_CountFileArchive {
400  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
401  ls ${@} 2>/dev/null | wc -l
402  if [ $? -gt 0 ] ; then
403    echo "IGCM_sys_CountFileArchive : erreur."
404  fi
405  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
406}
407
408#D-#==================================================
409#D-function IGCM_sys_Tree
410#D-* Purpose: Tree directories with files on ${ARCHIVE}
411#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
412#D-
413function IGCM_sys_Tree {
414  IGCM_debug_PushStack "IGCM_sys_Tree" $@
415  if ( $DEBUG_sys ) ; then
416    echo "IGCM_sys_Tree :" $@
417  fi
418
419  \ls -lR ${@}
420
421  IGCM_debug_PopStack "IGCM_sys_Tree"
422}
423
424#D-#==================================================
425#D-function IGCM_sys_Qsub
426#D-* Purpose: Qsub new job
427#D-* Examples:
428#D-
429function IGCM_sys_Qsub {
430  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
431  if ( $DEBUG_sys ) ; then
432    echo "IGCM_sys_Qsub :" $@
433  fi
434  typeset options status
435  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
436
437  /usr/bin/time sbatch ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
438  status=$?
439
440  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
441  if [ ${status} -gt 0 ] ; then
442    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
443    IGCM_debug_Exit "IGCM_sys_Qsub"
444  else
445    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
446  fi
447  IGCM_debug_PopStack "IGCM_sys_Qsub"
448}
449
450#D-#==================================================
451#D-function IGCM_sys_QsubPost
452#D-* Purpose: Qsub new job on scalaire
453#D-* Examples:
454#D-
455function IGCM_sys_QsubPost {
456  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
457  if ( $DEBUG_sys ) ; then
458    echo "IGCM_sys_QsubPost :" $@
459  fi
460  typeset options status
461  options="-o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
462
463  sbatch ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
464  status=$?
465
466  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
467  if [ ${status} -gt 0 ] ; then
468    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
469    IGCM_debug_Exit "IGCM_sys_QsubPost"
470  else
471    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
472  fi
473  IGCM_debug_PopStack "IGCM_sys_QsubPost"
474}
475
476#D-*************************
477#D- File transfer functions
478#D-*************************
479#D-
480
481#D-#==================================================
482#D-function IGCM_sys_RmRunDir
483#D-* Purpose: rm tmpdir (dummy function most of the time batch
484#D-                      scheduler will do the job)
485#D-* Examples:
486#D-
487function IGCM_sys_RmRunDir {
488  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
489  if ( $DEBUG_sys ) ; then
490    echo "IGCM_sys_RmRunDir :" $@
491  fi
492
493  typeset status
494
495  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
496  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
497  status=$?
498
499  if [ ${status} -gt 0 ] ; then
500    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
501    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
502    IGCM_debug_Exit "IGCM_sys_RmRunDir"
503  else
504    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
505  fi
506  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
507}
508
509#D-#==================================================
510#D-function IGCM_sys_Put_Dir
511#D-* Purpose: Copy a complete directory on $(ARCHIVE)
512#D-* Examples:
513#D-
514function IGCM_sys_Put_Dir {
515  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
516  if ( $DEBUG_sys ) ; then
517    echo "IGCM_sys_Put_Dir :" $@
518  fi
519  if [ $DRYRUN = 0 ]; then
520    if [ ! -d ${1} ] ; then
521      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
522      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
523      return
524    fi
525
526    typeset status
527
528    # Only if we use rsync
529    #IGCM_sys_TestDirArchive $( dirname $2 )
530    #
531    #USUAL WAY
532    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
533    status=$?
534
535    if [ ${status} -gt 0 ] ; then
536      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
537      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
538      IGCM_debug_Exit "IGCM_sys_Put_Dir"
539    else
540      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
541    fi
542  fi
543  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
544}
545
546#D-#==================================================
547#D-function IGCM_sys_Get_Dir
548#D-* Purpose: Copy a complete directory from ${ARCHIVE}
549#D-* Examples:
550#D-
551function IGCM_sys_Get_Dir {
552  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
553  if ( $DEBUG_sys ) ; then
554    echo "IGCM_sys_Get_Dir :" $@
555  fi
556  if [ $DRYRUN = 0 ]; then
557    typeset NB_ESSAI DELAI status i
558    # number of tentative
559    NB_ESSAI=3
560    # time delay between tentative
561    DELAI=2
562
563    #
564    # USUAL WAY
565    # add here command to demigrate all offline files to reduce time of this command
566    # actually on Jean Zay files are not archive on bands
567
568    i=0
569    while [ $i -lt $NB_ESSAI ] ; do
570      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
571      status=$?
572      if [ ${status} -gt 0 ] ; then
573        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
574        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
575        sleep $DELAI
576      else
577        break
578      fi
579      (( i = i + 1 ))
580    done
581
582    if [ ${status} -gt 0 ] ; then
583      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
584      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
585      IGCM_debug_Exit "IGCM_sys_Get_Dir"
586    else
587      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
588    fi
589  fi
590  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
591}
592
593#D-#==================================================
594#D-function IGCM_sys_Put_Rest
595#D-* Purpose: Put computied restarts on ${ARCHIVE}.
596#D-           File and target directory must exist.
597#D-* Examples:
598#D-
599function IGCM_sys_Put_Rest {
600  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
601  if ( $DEBUG_sys ) ; then
602    echo "IGCM_sys_Put_Rest :" $@
603  fi
604  if [ $DRYRUN = 0 ]; then
605    if [ ! -f ${1} ] ; then
606      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
607      IGCM_debug_Exit "IGCM_sys_Put_Rest"
608    fi
609
610    typeset status
611    #
612    # USUAL WAY
613    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
614    status=$?
615
616#       #RSYNC WITH NETWORK SSH CALL
617#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
618#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
619
620#       #RSYNC WITH NFS USE
621#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
622#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
623
624#       status=$?
625#       IGCM_sys_Rsync_out $status
626
627#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
628#       (( status=status+$? ))
629
630    if [ ${status} -gt 0 ] ; then
631      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
632      [ -f ${1} ] && ls -l ${1}
633      [ -f ${2} ] && ls -l ${2}
634      [ -f ${2}/${1} ] && ls -l ${2}/${1}
635      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
636      IGCM_debug_Exit "IGCM_sys_Put_Rest"
637    else
638
639      if [ X${JobType} = XRUN ] ; then
640        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
641        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
642      fi
643
644      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
645    fi
646  fi
647  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
648}
649
650#D-#==================================================
651#D-function IGCM_sys_Put_Out
652#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
653#D-* Examples:
654#D-
655function IGCM_sys_Put_Out {
656  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
657  if ( $DEBUG_sys ) ; then
658    echo "IGCM_sys_Put_Out :" $@
659  fi
660
661  typeset NB_ESSAI DELAI status i exist skip
662  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
663
664  # number of tentative
665  NB_ESSAI=3
666  # time delay between tentative
667  DELAI=2
668
669  if [ $DRYRUN = 0 ]; then
670    if [ ! -f ${1} ] ; then
671      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
672      IGCM_debug_PopStack "IGCM_sys_Put_Out"
673      return 1
674    fi
675    #
676    IGCM_sys_MkdirArchive $( dirname $2 )
677    #
678    exist=false
679    skip=false
680    if [ -f $2 ] ; then
681      IGCM_debug_Print 1 "$2 already exist"
682    # add here command to demigrate all offline files to reduce time of this command
683    # actually on Jean Zay files are not archive on bands
684
685      exist=true
686      if [ "X$( diff $1 $2 )" = X ] ; then
687        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
688        skip=true
689      else
690        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
691        skip=false
692      fi
693    fi
694    #
695    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
696      IGCM_sys_Chmod u+w $2
697    fi
698
699    if [ X${skip} = Xfalse ] ; then
700      i=0
701      while [ $i -lt $NB_ESSAI ] ; do
702        # Identify file system
703        fileDeviceNumberInHex=$( stat -c %d $1 )
704        status=$?
705        if [ ${status} -gt 0 ] ; then
706          IGCM_debug_Exit "IGCM_sys_Put_Out"
707        fi
708        # Identify file system
709        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
710        status=$?
711        if [ ${status} -gt 0 ] ; then
712          IGCM_debug_Exit "IGCM_sys_Put_Out"
713        fi
714
715        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
716          # They are not on the same device. USUAL WAY
717          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
718          status=$?
719        else
720          # They are on the same device. NOT SO USUAL WAY
721          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
722          status=$?
723        fi
724        if [ ${status} -gt 0 ]; then
725          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
726          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
727          [ -f ${1} ] && ls -l ${1}
728          [ -f ${2} ] && ls -l ${2}
729          [ -f ${2}/${1} ] && ls -l ${2}/${1}
730          sleep $DELAI
731        else
732          break
733        fi
734        (( i = i + 1 ))
735      done
736    fi
737
738#       #RSYNC WITH NETWORK SSH CALL
739#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
740#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
741
742#       #RSYNC WITH NFS USE
743#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
744#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
745
746#       status=$?
747#       IGCM_sys_Rsync_out $status
748
749#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
750#       (( status=status+$? ))
751
752    if [ ${status} -gt 0 ] ; then
753      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
754      [ -f ${1} ] && ls -l ${1}
755      [ -f ${2} ] && ls -l ${2}
756      [ -f ${2}/${1} ] && ls -l ${2}/${1}
757      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
758      IGCM_debug_Exit "IGCM_sys_Put_Out"
759    else
760
761      if [ X${JobType} = XRUN ] ; then
762        if [ X${3} = X ] ; then
763          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
764          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
765        fi
766      fi
767
768      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
769    fi
770  fi
771  IGCM_debug_PopStack "IGCM_sys_Put_Out"
772  return 0
773}
774
775#D-#==================================================
776#D-function IGCM_sys_Get
777#D-* Purpose: Get a file from ${ARCHIVE}
778#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
779#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
780function IGCM_sys_Get {
781  IGCM_debug_PushStack "IGCM_sys_Get" $@
782
783  typeset DEST dm_liste target file_work
784  typeset NB_ESSAI DELAI status i
785
786  if ( $DEBUG_sys ) ; then
787    echo "IGCM_sys_Get :" $@
788  fi
789
790  # number of tentative
791  NB_ESSAI=3
792  # time delay between tentative
793  DELAI=2
794
795  if [ $DRYRUN -le 2 ]; then
796    if [ X${1} = X'/l' ] ; then
797      eval set +A dm_liste \${${2}}
798    else
799      eval set +A dm_liste ${1}
800    fi
801    eval DEST=\${${#}}
802    # add here command to demigrate all offline files to reduce time of this command
803    # actually on Jean Zay files are not archive on bands
804
805    status=$?
806    if [ ${status} -gt 0 ] ; then
807      echo "WARNING IGCM_sys_Get : error code ${status}"
808      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
809      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
810    fi
811
812    #if [ ${status} -gt 0 ] ; then
813    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
814    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
815    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
816    #    sleep 30
817    #    echo "We try another time"
818    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
819    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
820    #    status=$?
821    #    if [ ${status} -gt 0 ] ; then
822    #      echo "ERROR IGCM_sys_Get : again demigration error :"
823    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
824    #      IGCM_debug_Exit "IGCM_sys_Get"
825    #    fi
826    #  else
827    #    echo "ERROR IGCM_sys_Get : demigration error :"
828    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
829    #    IGCM_debug_Exit "IGCM_sys_Get"
830    #  fi
831    #fi
832
833    #   #RSYNC WITH NETWORK SSH CALL
834    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
835    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
836
837    #   #RSYNC WITH NFS USE
838    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
839    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
840
841    #   status=$?
842    #   IGCM_sys_Rsync_out $status
843
844    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
845    #   (( status=status+$? ))
846
847    #USUAL WAY
848    if [ X${1} = X'/l' ] ; then
849      for target in ${dm_liste[*]} ; do
850        local_file=$( basename ${target} )
851        # test if the target file is present before the loop
852        IGCM_sys_TestFileArchive ${target}
853        status=$?
854        if [ ${status} -gt 0 ] ; then
855          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
856          IGCM_debug_Exit "IGCM_sys_Get"
857        else
858          i=0
859          while [ $i -lt $NB_ESSAI ] ; do
860            #if [ X${DoLink} = Xtrue ] ; then
861            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
862            #  status=$?
863            #  else
864            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
865            #  status=$?
866            #fi
867            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
868            status=$?
869            if [ ${status} -gt 0 ]; then
870              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
871              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
872              sleep $DELAI
873            else
874              break
875            fi
876            (( i = i + 1 ))
877          done
878          if [ ${status} -gt 0 ] ; then
879            echo "IGCM_sys_Get : error"
880            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
881            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
882            IGCM_debug_Exit "IGCM_sys_Get"
883          else
884            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
885          fi
886        fi
887      done
888    else
889      i=0
890      while [ $i -lt $NB_ESSAI ] ; do
891        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
892        status=$?
893        if [ ${status} -gt 0 ]; then
894          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
895          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
896          sleep $DELAI
897        else
898          break
899        fi
900        (( i = i + 1 ))
901      done
902      if [ ${status} -gt 0 ] ; then
903        echo "IGCM_sys_Get : error"
904        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
905        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
906        IGCM_debug_Exit "IGCM_sys_Get"
907      else
908        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
909      fi
910    fi
911  fi
912  IGCM_debug_PopStack "IGCM_sys_Get"
913}
914
915#D-#==================================================
916#D-function IGCM_sys_GetDate_Monitoring
917#D-* Purpose: get the last year for which the monitoring has been computed
918#D-* Examples:
919#D-
920function IGCM_sys_GetDate_Monitoring {
921  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
922  if ( $DEBUG_sys ) ; then
923    echo "IGCM_sys_GetDate_Monitoring :" $@
924  fi
925
926  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
927
928  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
929}
930
931#D-#==================================================
932#D-function IGCM_sys_Dods_Rm
933#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
934#D-* Examples:
935#D-
936function IGCM_sys_Dods_Rm {
937  if ( $DEBUG_sys ) ; then
938    echo "IGCM_sys_Dods_Rm :" $@
939  fi
940  typeset status
941  if [ $DRYRUN = 0 ]; then
942
943#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
944#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
945#      echo "Nothing has been done."
946#      return
947#    fi
948
949    if [ "$#" -eq 1 ]; then
950      thredds_rm ${R_DODS}/${1}
951      status=$?
952    else
953        thredds_rm ${R_DODS}
954      status=$?
955    fi
956
957#    if [ ${status} -gt 0 ] ; then
958#      echo "IGCM_sys_Dods_Rm : error."
959#      cat out_dods_rm
960#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
961#    else
962#      rm out_dods_rm
963#    fi
964
965  fi
966  return $status
967}
968
969#D-#==================================================
970#D-function IGCM_sys_Dods_Cp
971#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
972#D-* Examples:
973#D-
974function IGCM_sys_Dods_Cp {
975  if ( $DEBUG_sys ) ; then
976    echo "IGCM_sys_Dods_Cp :" $@
977  fi
978  typeset status
979  if [ $DRYRUN = 0 ]; then
980
981#    if [ ! -d ${R_SAVE}/${1} ] ; then
982#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
983#      echo "Nothing has been done."
984#      return
985#    fi
986
987
988      thredds_cp ${1} ./${R_DODS}/.
989
990    status=$?
991
992#       if [ ${status} -gt 0 ] ; then
993#           echo "IGCM_sys_Dods_Cp : error."
994#           cat out_dods_cp
995#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
996#       else
997#           rm out_dods_cp
998#       fi
999
1000  fi
1001  return $status
1002}
1003
1004#D-#==================================================
1005#D-function IGCM_sys_Put_Dods
1006#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1007#D-* Examples:
1008#D-
1009function IGCM_sys_Put_Dods {
1010  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1011  if ( $DEBUG_sys ) ; then
1012    echo "IGCM_sys_Put_Dods :" $@
1013  fi
1014  typeset status
1015  if [ $DRYRUN = 0 ]; then
1016    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1017      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1018      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1019      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1020      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1021      return
1022    fi
1023
1024    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1025      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1026      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1027      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1028      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1029      return
1030    fi
1031    #
1032    if [ -d ${R_SAVE}/${1} ] ; then
1033      cd ${R_SAVE}
1034    elif [ -d ${R_FIGR}/${1} ] ; then
1035      cd ${R_FIGR}
1036    fi
1037
1038    IGCM_sys_Dods_Rm ${1}
1039    IGCM_sys_Dods_Cp ${1}
1040    status=0
1041
1042    if [ ${status} -gt 0 ] ; then
1043      echo "IGCM_sys_Put_Dods : error."
1044      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1045    fi
1046  fi
1047  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1048}
1049
1050##############################################################
1051# REBUILD OPERATOR
1052
1053#D-#==================================================
1054#D-function IGCM_sys_sync
1055#D-* Purpose: flush buffer on disk
1056#D-* Examples:
1057#D-
1058function IGCM_sys_sync {
1059  IGCM_debug_PushStack "IGCM_sys_sync" $@
1060  if ( $DEBUG_sys ) ; then
1061    echo "IGCM_sys_sync :" $@
1062  fi
1063
1064  /bin/sync
1065
1066  IGCM_debug_PopStack "IGCM_sys_sync"
1067}
1068
1069############################################################
1070# Activate Running Environnment Variables
1071
1072#D-#==================================================
1073#D-function IGCM_sys_activ_variables
1074#D-* Purpose: set environement variables prior to execution
1075#D-* Examples:
1076#D-
1077function IGCM_sys_activ_variables {
1078  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1079  if ( $DEBUG_sys ) ; then
1080    echo "IGCM_sys_activ_variables"
1081  fi
1082
1083# --------------------------------------------------------------------
1084#D- MODULE specifications
1085# --------------------------------------------------------------------
1086  # Source the file EnvFile containing all module load needed to run the model.
1087  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1088  # it will be used. Otherwise a default file will be used.
1089  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1090      # EnvFile is set in config.card and the file exists
1091      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1092      EnvFile=${config_UserChoices_EnvFile}
1093  else
1094      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1095      module purge > /dev/null 2>&1
1096      EnvFile=/gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay
1097  fi
1098  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1099  IGCM_debug_Print 1 ${EnvFile}
1100  . ${EnvFile}
1101
1102  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Jean-Zay. "
1103  module list
1104
1105# --------------------------------------------------------------------
1106#D- MPI specifications
1107# --------------------------------------------------------------------
1108
1109# --------------------------------------------------------------------
1110#D- Other specifications
1111# --------------------------------------------------------------------
1112
1113  ulimit -s unlimited
1114  if [ ${executionType} -eq 2 ] ; then
1115    export SLURM_HOSTFILE=./hostlist
1116  fi
1117
1118
1119  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1120}
1121
1122############################################################
1123# Desactivate Running Environnment Variables
1124
1125#D-#==================================================
1126#D-function IGCM_sys_desactiv_variables
1127#D-* Purpose: unset environement variables after execution
1128#D-* Examples:
1129#D-
1130function IGCM_sys_desactiv_variables {
1131  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1132  if ( $DEBUG_sys ) ; then
1133    echo "IGCM_sys_desactiv_variables"
1134  fi
1135# --------------------------------------------------------------------
1136#D- MPI specifications
1137# --------------------------------------------------------------------
1138  if [ ${executionType} -eq 2 ] ; then
1139    unset SLURM_HOSTFILE
1140  fi
1141# --------------------------------------------------------------------
1142#D- Other specifications
1143# --------------------------------------------------------------------
1144
1145  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1146}
1147
1148############################################################
1149# Update job headers to be used by the scheduler
1150
1151#D-#==================================================
1152#D-function IGCM_sys_updateHeaders
1153#D-* Purpose: Update job headers to be used by the scheduler
1154#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1155#D-
1156function IGCM_sys_updateHeaders {
1157  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1158  if ( $DEBUG_sys ) ; then
1159    echo "IGCM_sys_updateHeaders"
1160  fi
1161  typeset file
1162  file=$1
1163
1164  if [ ${executionType} -eq 1 ] ; then
1165    # MPMD + MPI
1166    sed -e "/::openMPthreads::/d"                  \
1167        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1168      ${file} > ${file}.tmp
1169
1170  elif [ ${executionType} -eq 2 ] ; then
1171    # MPMD + MPI + OMP
1172      (( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
1173      [ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
1174      sed -e "/::openMPthreads::/d"                 \
1175        -e "s/::JobNumProcTot::/${mpiTasks}/"       \
1176        -e "/ntasks/i\#SBATCH --nodes=${nodeNumber}"\
1177        -e "/ntasks/i\#SBATCH --exclusive"          \
1178         ${file} > ${file}.tmp
1179  elif [ ${executionType} -eq 3 ] ; then
1180    # SPMD + MPI/OMP
1181    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1182        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1183      ${file} > ${file}.tmp
1184
1185  elif [ ${executionType} -eq 4 ] ; then
1186    # SPMD + MPI only
1187    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1188        -e "/::openMPthreads::/d"                  \
1189      ${file} > ${file}.tmp
1190
1191  elif [ ${executionType} -eq 5 ] ; then
1192    # SPMD + OMP only
1193    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1194        -e "/::JobNumProcTot::/d"                  \
1195      ${file} > ${file}.tmp
1196
1197  elif [ ${executionType} -eq 6 ] ; then
1198    # SEQUENTIAL THEN
1199    sed -e "s/::JobNumProcTot::/1/"                \
1200        -e "/::openMPthreads::/d"                  \
1201      ${file} > ${file}.tmp
1202
1203  fi
1204
1205  IGCM_sys_Mv ${file}.tmp ${file}
1206
1207  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1208}
1209
1210############################################################
1211# Build MPI/OMP scripts run file (dummy function)
1212
1213#D-#==================================================
1214#D-function IGCM_sys_build_run_file
1215#D-* Purpose: build run file (deprecated)
1216#D-* Examples:
1217#D-
1218function IGCM_sys_build_run_file {
1219  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1220}
1221
1222############################################################
1223# Build MPI/OMP scripts
1224
1225#D-#==================================================
1226#D-function IGCM_sys_build_execution_scripts
1227#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1228#D-* Examples:
1229#D-
1230function IGCM_sys_build_execution_scripts
1231{
1232  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1233  if ( $DEBUG_sys ) ; then
1234    echo "IGCM_sys_build_execution_scripts " $@
1235  fi
1236  IGCM_debug_Print 1 "executionType= ${executionType}"
1237  EXECUTION=${HOST_MPIRUN_COMMAND}
1238
1239  if ( ${OK_PARA_MPMD} ) ; then
1240
1241    # MPMD mode
1242    # 1 MPI only : executionType=1
1243    # 2 MPI/OpenMP : executionType=2
1244
1245    if [ -f run_file ] ; then
1246      IGCM_sys_Rm -f run_file
1247    fi
1248    touch run_file
1249
1250    # case 1 : Only MPI (MPMD)
1251    if  ( ! ${OK_PARA_OMP} ) ; then
1252      # Build run_file
1253      current_core=0
1254      # First loop on the components for the coupler ie oasis (only if oasis3)
1255      # the coupler ie oasis3 must be the first one
1256      for comp in ${config_ListOfComponents[*]} ; do
1257
1258        eval ExeNameIn=\${config_Executable_${comp}[0]}
1259        eval ExeNameOut=\${config_Executable_${comp}[1]}
1260
1261        # Only if the component has an executable
1262        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1263
1264          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1265          (( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 ))
1266          echo "${current_core}-${end_core} ./${ExeNameOut}" >> run_file
1267          (( current_core = ${end_core} + 1 ))
1268        fi
1269      done
1270
1271      EXECUTION="${HOST_MPIRUN_COMMAND} --multi-prog ./run_file"
1272
1273      IGCM_sys_Chmod u+x run_file
1274      if ( $DEBUG_sys ) ; then
1275        echo "run_file contains : "
1276        cat run_file
1277      fi
1278
1279    else 
1280
1281    # 2 MPI/OpenMP : executionType=2
1282
1283    # MPI-OpenMP (MPMD)
1284    # export SLURM_HOSTFILE=./hostlist
1285    # srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
1286    # example of  hostlist file :
1287    # r3i3n33
1288    # r3i3n33
1289    # ...
1290    # example of run_file :
1291    # 0-70 ./prog_lmdz.x.sh %o %t
1292    # 71-430 ./prog_opa.xx.sh %o %t
1293    # 431-431 ./prog_xios.x.sh %o %t
1294    # examples of prog_file :
1295    # prog_lmdz.x.sh :
1296    # (( init = 0 + $1 ))
1297    # (( index = init * 10 ))
1298    # (( slot = index % 40 ))
1299    # taskset -c $slot-$((slot + 10 - 1)) ./script_lmdz.x.ksh
1300    # that will become
1301    # taskset -c 0-9 ./script_lmdz.x.ksh
1302    # ...
1303    # with script_lmdz.x.ksh
1304    # export OMP_STACKSIZE=3g
1305    # export OMP_PLACES=cores
1306    # export OMP_NUM_THREADS=10
1307    # ./lmdz.x > out_lmdz.x.out.${SLURM_PROCID} 2>out_lmdz.x.err.${SLURM_PROCID}
1308
1309    #  Hosts treatment
1310    _bkIFS=$IFS;
1311        IFS=$'\n'; set -f
1312        listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
1313        IFS=$_bkIFS; set +f
1314        rm -f hostlist
1315
1316     # Loop on the components to build run_file and script_exec files
1317        rank=0
1318        current_core=0
1319        current_core_mpi=0
1320
1321      for comp in ${config_ListOfComponents[*]} ; do
1322
1323        eval ExeNameIn=\${config_Executable_${comp}[0]}
1324        eval ExeNameOut=\${config_Executable_${comp}[1]}
1325
1326        # Not possible if oasis has an executable (i.e old version of oasis3)
1327        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1328          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1329          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1330          IGCM_debug_Verif_Exit
1331        fi
1332
1333        # Only if we really have an executable for the component :
1334        if [ "X${ExeNameOut}" != X\"\" ] ; then
1335
1336          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1337          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1338          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1339     
1340
1341          # Build script files
1342
1343          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1344          echo ""  >> script_${ExeNameOut}.ksh
1345          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1346          # Check if the number of threads is correct
1347              case ${comp_proc_omp_loc} in
1348                  2|4|5|10|20)
1349                      IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1350                      ;;
1351                  *)
1352                      IGCM_debug_Exit "ERROR with OMP parameters !"
1353                      IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1354                      IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1355                      IGCM_debug_Verif_Exit
1356                      ;;
1357              esac
1358            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1359            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1360            echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}"  >> script_${ExeNameOut}.ksh
1361
1362          fi
1363
1364          # to have out/err per process on different files
1365          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1366
1367          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1368
1369        # Build run_file
1370        # Only if the component has an executable
1371        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1372
1373          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1374          (( end_core = ${current_core_mpi} + ${comp_proc_mpi_loc} - 1 ))
1375          echo "${current_core_mpi}-${end_core} ./prog_${ExeNameOut}.sh %o %t" >> run_file
1376          (( current_core_mpi = ${end_core} + 1 ))
1377        fi
1378
1379          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1380              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1381          else
1382              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1383          fi
1384
1385          # Build configuration file
1386 
1387              echo "#!/bin/sh" > prog_${ExeNameOut}.sh
1388          echo "(( init = $current_core + \$1 ))" >> prog_${ExeNameOut}.sh
1389          echo "(( index = init * $comp_proc_omp_loc ))" >> prog_${ExeNameOut}.sh
1390          echo "(( slot = index % 40 ))" >> prog_${ExeNameOut}.sh
1391              echo "echo ${ExeNameOut} taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1))" >> prog_${ExeNameOut}.sh
1392          echo "taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1)) ./script_${ExeNameOut}.ksh" >> prog_${ExeNameOut}.sh
1393
1394          IGCM_sys_Chmod u+x prog_${ExeNameOut}.sh
1395
1396        # Build hostlist file
1397
1398          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1399             (( index_host = current_core / NB_CORE_PER_NODE ))
1400             host_value=${listnodes[${index_host}]}
1401               echo "$host_value" >> hostlist
1402             (( current_core = current_core + offset_comp_proc_loc ))
1403          done
1404        fi
1405      done
1406
1407      ## variable added to stop after 60s instead of 600s by default.
1408      ## This is used when no error comes from executables and when something stopped an executable without notice.
1409      export SLURM_WAIT=60
1410
1411      EXECUTION="${HOST_MPIRUN_COMMAND} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
1412
1413      IGCM_sys_Chmod u+x run_file
1414      if ( $DEBUG_sys ) ; then
1415        echo "run_file contains : "
1416        cat run_file
1417      fi
1418
1419    fi # if ${OK_PARA_MPMD}
1420
1421  else
1422  # Only one executable (SPMD mode):  executionType=3, 4, 5 and 6
1423
1424    for comp in ${config_ListOfComponents[*]} ; do
1425
1426      # Only if we really have an executable for the component :
1427      eval ExeNameOut=\${config_Executable_${comp}[1]}
1428      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1429
1430        # Build script files
1431
1432        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1433        echo ""  >> script_${ExeNameOut}.ksh
1434        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1435
1436        if ( ${OK_PARA_OMP} ) ; then
1437            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1438            # Check if the number of threads is correct
1439            case ${comp_proc_omp_loc} in
1440                2|4|5|10|20)
1441                    IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1442                    ;;
1443                *)
1444                    IGCM_debug_Exit "ERROR with OMP parameters !"
1445                    IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1446                    IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1447                    IGCM_debug_Verif_Exit
1448                    ;;
1449            esac
1450            echo ""  >> script_${ExeNameOut}.ksh
1451            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1452            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1453            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1454        fi
1455
1456        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1457       
1458        # To have out/err per process on different files
1459        echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1460        EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1461
1462        IGCM_debug_Print 1 "sys Irene : script_${ExeNameOut}.ksh contains"
1463        cat script_${ExeNameOut}.ksh
1464
1465      fi
1466
1467    done
1468
1469  fi # ${OK_PARA_MPMD}
1470
1471  IGCM_debug_Print 1 "sys Irene : execution command is "
1472  IGCM_debug_Print 1 "$EXECUTION"
1473
1474  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1475}
1476
1477#D-#==================================================
1478#D-function IGCM_sys_check_path
1479#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1480#D-* do not point to an important use directory. Stop immediately in that case.
1481#D-* Examples:
1482#D-
1483function IGCM_sys_check_path {
1484  IGCM_debug_PushStack "IGCM_sys_check_path"
1485  if ( $DEBUG_sys ) ; then
1486    echo "IGCM_sys_check_path"
1487  fi
1488
1489  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORK} ] || [ X${RUN_DIR_PATH} = X${SCRATCH} ] || [ X${RUN_DIR_PATH} = X${STORE} ] ) ; then
1490    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1491    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1492    IGCM_debug_Exit "This will stop the job"
1493  elif ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X/gpfswork/rech/${DataProject}/${LOGIN} ] || [ X${RUN_DIR_PATH} = X/gpfsscratch/rech/${DataProject}/${LOGIN} ] || [ X${RUN_DIR_PATH} = X/gpfsstore/rech/${DataProject}/${LOGIN} ] ) ; then
1494    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1495    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1496    IGCM_debug_Exit "This will stop the job"
1497  fi
1498  IGCM_debug_PopStack "IGCM_sys_check_path"
1499}
1500
1501#D-#==================================================
1502#D-function IGCM_sys_check_quota
1503#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1504#D-* Examples:
1505#D-
1506function IGCM_sys_check_quota {
1507  IGCM_debug_PushStack "IGCM_sys_check_quota"
1508  echo "IGCM_sys_check_quota is desactivated on JeanZay because there is actually no way to check scratch quota on this computing center (2019.11.19)"
1509#  if ( $DEBUG_sys ) ; then
1510#    echo "IGCM_sys_check_quota"
1511#  fi
1512#  # Limit of quota (in %)
1513#  limit_quota=90
1514#
1515#  # Check of the volume
1516#  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1517#  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1518#
1519#  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1520#
1521#    unit_avail=${volume_avail: -1}
1522#    unit_quota=${volume_quota: -1}
1523#
1524#    if [ "${unit_quota}" = "*" ] ; then
1525#      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1526#      IGCM_debug_Print 1 "More than 100% of your quota is used"
1527#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1528#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1529#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1530#      IGCM_debug_Verif_Exit
1531#    fi
1532#
1533#    temp_avail=${volume_avail%%${volume_avail: -1}*}
1534#    temp_quota=${volume_quota%%${volume_quota: -1}*}
1535#
1536#    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1537#
1538#    # Convertion
1539#      if [ ${volume_avail: -1} = "T" ] ; then
1540#        (( temp_avail = temp_avail * 1000000000000 ))
1541#      elif [ ${volume_avail: -1} = "G" ] ; then
1542#        (( temp_avail = temp_avail * 1000000000 ))
1543#      elif [ ${volume_avail: -1} = "M" ] ; then
1544#        (( temp_avail = temp_avail * 1000000 ))
1545#      elif [ ${volume_avail: -1} = "k" ] ; then
1546#        (( temp_avail = temp_avail * 1000 ))
1547#      else
1548#        (( temp_avail = volume_avail ))
1549#      fi
1550#      if [ ${volume_quota: -1} = "T" ] ; then
1551#        (( temp_quota = temp_quota * 1000000000000 ))
1552#      elif [ ${volume_quota: -1} = "G" ] ; then
1553#        (( temp_quota = temp_quota * 1000000000 ))
1554#      elif [ ${volume_quota: -1} = "M" ] ; then
1555#        (( temp_quota = temp_quota * 1000000 ))
1556#      elif [ ${volume_quota: -1} = "k" ] ; then
1557#        (( temp_quota = temp_quota * 1000 ))
1558#      else
1559#        (( temp_quota = volume_quota ))
1560#      fi
1561#    fi
1562#
1563#    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1564##    echo "volume ratio is " $quota_volume
1565#
1566#    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1567#      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1568#      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1569#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1570#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1571#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1572#      IGCM_debug_Verif_Exit
1573#    fi
1574#
1575#  fi
1576#
1577## Check of the number of inodes
1578#
1579#  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1580#  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1581#
1582#  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1583#
1584#    unit_avail=${inode_avail: -1}
1585#    unit_quota=${inode_quota: -1}
1586#
1587#    if [ "${unit_quota}" = "*" ] ; then
1588#      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1589#      IGCM_debug_Print 1 "More than 100% of your quota is used"
1590#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1591#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1592#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1593#      IGCM_debug_Verif_Exit
1594#    fi
1595#
1596#    temp_avail=${inode_avail%%${inode_avail: -1}*}
1597#    temp_quota=${inode_quota%%${inode_quota: -1}*}
1598#
1599#    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1600#
1601#    # Convertion
1602#      if [ ${inode_avail: -1} = "T" ] ; then
1603#        (( temp_avail = temp_avail * 1000000000000 ))
1604#      elif [ ${inode_avail: -1} = "G" ] ; then
1605#        (( temp_avail = temp_avail * 1000000000 ))
1606#      elif [ ${inode_avail: -1} = "M" ] ; then
1607#        (( temp_avail = temp_avail * 1000000 ))
1608#      elif [ ${inode_avail: -1} = "k" ] ; then
1609#        (( temp_avail = temp_avail * 1000 ))
1610#      else
1611#        (( temp_avail = inode_avail ))
1612#      fi
1613#
1614#      if [ ${inode_quota: -1} = "T" ] ; then
1615#        (( temp_quota = temp_quota * 1000000000000 ))
1616#      elif [ ${inode_quota: -1} = "G" ] ; then
1617#        (( temp_quota = temp_quota * 1000000000 ))
1618#      elif [ ${inode_quota: -1} = "M" ] ; then
1619#        (( temp_quota = temp_quota * 1000000 ))
1620#      elif [ ${inode_quota: -1} = "k" ] ; then
1621#        (( temp_quota = temp_quota * 1000 ))
1622#      else
1623#        (( temp_quota = inode_quota ))
1624#      fi
1625#    fi
1626#    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1627##    echo "inode ratio is " $quota_inode
1628#
1629#    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1630#      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1631#      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1632#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1633#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1634#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1635#      IGCM_debug_Verif_Exit
1636#    fi
1637#  fi
1638  IGCM_debug_PopStack "IGCM_sys_check_quota"
1639}
1640
1641#D-#==================================================
1642#D-function IGCM_sys_projectAccounting
1643#D-* Purpose: store project accounting information in a file
1644#D-* Examples:
1645#D-
1646function IGCM_sys_projectAccounting {
1647  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1648  if ( $DEBUG_sys ) ; then
1649    echo "IGCM_sys_check_quota"
1650  fi
1651#need to implemented on Jean Zay
1652  touch $1
1653#  ssh irene191 /usr/bin/ccc_myproject > $1
1654
1655  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1656}
1657
1658#D-#==================================================
1659#D-function IGCM_sys_getJobSchedulerID
1660#D-* Purpose: Get the job ID during execution
1661#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1662#D-
1663function IGCM_sys_getJobSchedulerID {
1664  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1665  if ( $DEBUG_sys ) ; then
1666    echo "IGCM_sys_getJobSchedulerID"
1667  fi
1668
1669  eval ${1}=${SLURM_JOBID}
1670
1671  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1672}
1673
1674#D-#==================================================
1675#D-function IGCM_sys_GetJobID
1676#D-* Purpose: Get the job ID from the JobName
1677#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1678#D-
1679function IGCM_sys_GetJobID {
1680  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1681  if ( $DEBUG_sys ) ; then
1682    echo "IGCM_sys_GetJobID"
1683  fi
1684
1685  ID=$( squeue -u $user | grep Job_${config_UserChoices_JobName} | gawk ' { print $1}' )
1686
1687
1688  eval ${3}=${ID}
1689  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1690}
1691
1692#D-#==================================================
1693#D-function IGCM_sys_CountJobInQueue
1694#D-* Purpose: Count number of users job
1695#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1696#D-
1697function IGCM_sys_CountJobInQueue {
1698  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1699  if ( $DEBUG_sys ) ; then
1700    echo "IGCM_sys_CountJobInQueue"
1701  fi
1702
1703  # With -f option, the full job name is given in the last column
1704  NbRun=$(squeue -u $user | \
1705      grep -v JOBID | gawk 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1706
1707  eval ${2}=${NbRun}
1708
1709  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1710}
1711
1712#D-#==================================================
1713#D-function IGCM_sys_ListJobInQueue
1714#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1715#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1716#D-
1717function IGCM_sys_ListJobInQueue {
1718  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1719  if ( $DEBUG_sys ) ; then
1720    echo "IGCM_sys_ListJobInQueue"
1721  fi
1722
1723# to be implemented on Jean Zay
1724 set -A JobList $( squeue -u $user | gawk '{print $3}' | grep -v NAME)
1725#  # With -f option, the full job name is given in the last column
1726#  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1727#                                        '( $2  == User      && \
1728#                                           $NF != /TS/      && \
1729#                                           $NF !~ /PACK/    && \
1730#                                           $NF !~ /REBUILD/ && \
1731#                                           $NF !~ /pack/ )     \
1732#                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1733#
1734  eval set -A ${2} ${JobList[*]}
1735
1736  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1737}
1738
1739#D-#==================================================
1740#D-function IGCM_sys_atlas
1741#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1742#D-* Examples:
1743#D-
1744function IGCM_sys_atlas {
1745  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1746  if ( $DEBUG_sys ) ; then
1747    echo "IGCM_sys_atlas :" $@
1748  fi
1749
1750  typeset status
1751# To be implemented on Jean Zay
1752#
1753#  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1754#  status=$?
1755#  if [ ${status} -gt 0 ] ; then
1756#    echo "IGCM_sys_atlas : error code ${status}"
1757#    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1758#    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1759#    IGCM_debug_PopStack "IGCM_sys_atlas"
1760#    return 1
1761#  else
1762#    IGCM_debug_PopStack "IGCM_sys_atlas"
1763#    return 0
1764#  fi
1765#
1766  IGCM_debug_PopStack "IGCM_sys_atlas"
1767}
1768
1769#D-#==================================================
1770#D-function IGCM_sys_rebuild_nemo
1771#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1772#D-* Examples:
1773#D-
1774
1775function IGCM_sys_rebuild_nemo {
1776  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1777  if ( $DEBUG_sys ) ; then
1778    echo "IGCM_sys_rebuild_nemo :" $@
1779  fi
1780
1781  /gpfswork/rech/psl/commun/Tools/rebuild_nemo/bin/rebuild_nemo ${1} ${2}
1782
1783  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1784}
1785
Note: See TracBrowser for help on using the repository browser.