source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_jeanzay.ksh @ 1632

Last change on this file since 1632 was 1632, checked in by jgipsl, 4 months ago

Set jobWarningDelay so that the accounting mail can be done now also for obelix.

Correction in conversion from min to sec in sys_mesoipsl and sys_jeanzay since previous commit [1631].
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 60.4 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Jean-Zay
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40if ([ X${DRYRUN} != X ] && [ ${DRYRUN} = 4 ]) ; then
41  typeset -r DRYRUN=0
42  typeset -r DRYRUN_DEBUG=4
43else
44  typeset -r DRYRUN=${DRYRUN:=0}
45  typeset -r DRYRUN_DEBUG=0
46fi
47
48# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
49# -------------------------------------------------------------------------------------
50# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
51# |          |  Cp/Exe/param/files |            |         |                           |
52# |          |  Chmod Qsub         |            |         |                           |
53# -------------------------------------------------------------------------------------
54# |    0     |       yes           |    yes     |  yes    |      yes                  |
55# -------------------------------------------------------------------------------------
56# |    1     |       yes           |    yes     |  yes    |      no                   |
57# -------------------------------------------------------------------------------------
58# |    2     |       yes           |    yes     |  no     |      no                   |
59# -------------------------------------------------------------------------------------
60# |    3     |       yes           |    no      |  no     |      no                   |
61# -------------------------------------------------------------------------------------
62# |    4     |       yes           |    yes     |  creation of RUN_DIR and Job_debug  |
63# -------------------------------------------------------------------------------------
64
65
66#=====================================================
67# Global Variables :
68#=====================================================
69# Language : "fr" or "en"
70typeset -r MYLANG="fr"
71
72#=====================================================
73# Host user names project maxCpuTime
74# $hostname ou hostname
75typeset HOST=${HOST:=$( hostname )}
76# $username ou whoami
77typeset LOGIN=${LOGIN:=$( whoami )}
78# $hostname of the MASTER job
79typeset MASTER=jeanzay
80# add default project set by ins_job
81typeset PROJECT=$( echo ${SLURM_JOB_ACCOUNT:=::default_project::} | cut -c "1-3" ) 
82### jobWarningDelay in seconds
83typeset jobWarningDelayMin=$(squeue -j $SLURM_JOB_ID -h --Format TimeLimit | awk -F: '{print $1}')
84typeset jobWarningDelay=$( echo "scale=6;${jobWarningDelayMin=}*60" | bc )
85 
86#D-
87#D-#==================================================
88#D-Program used in libIGCM
89#D-#==================================================
90
91# Submit command
92typeset SUBMIT=${SUBMIT:=sbatch}
93# rsync with path
94typeset -r RSYNC=/usr/bin/rsync
95# RSYNC_opt args to rsync
96typeset -r RSYNC_opt="-va"
97# ie storage filesystem
98typeset -r STOREHOST=${MASTER}
99typeset -r REMOTE_RSYNC=/usr/bin/rsync
100
101#====================================================
102# Access to module command
103#====================================================
104# See ticket #348
105# Following path should be used according to assist IDRIS
106. /gpfslocalsup/spack_soft/environment-modules/current/init/ksh
107# Following old path gives error messages while loading modules but it still works
108#. /usr/share/Modules/init/ksh
109
110#====================================================
111# Set environment tools (ferret, nco, cdo, rebuild, ...)
112#====================================================
113if [ X${TaskType} = Xcomputing ] ; then
114  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
115else
116  module purge > /dev/null 2>&1
117  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay > /dev/null 2>&1
118
119  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_atlas_jeanzay > /dev/null 2>&1
120
121#  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
122  export UVCDAT_ANONYMOUS_LOG=FALSE
123fi
124
125# FYI
126[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
127[ ! X${TaskType} = Xchecking ] && module list
128
129# For AddNoise (not yet AddPertu3DOCE)
130export PATH=${PATH}:/gpfswork/rech/psl/commun/Tools/AddNoise/bin
131
132# For rebuild
133export PATH=${PATH}:/gpfswork/rech/psl/commun/Tools/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
134
135#====================================================
136# Host specific DIRECTORIES
137#====================================================
138
139#====================================================
140#- MirrorlibIGCM for frontend
141typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
142
143#====================================================
144#- libIGCM_POST for frontend
145typeset -r libIGCM_POST=${libIGCM}
146
147#====================================================
148#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
149typeset -r R_EXE="${MODIPSL}/bin"
150
151#====================================================
152#- SUBMIT_DIR : submission dir
153typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${SLURM_SUBMIT_DIR}}
154
155#====================================================
156#- IN
157typeset -r R_IN=${R_IN:=/gpfswork/rech/psl/commun/IGCM}
158
159#====================================================
160#- RUN_DIR_PATH : Temporary working directory (=> TMP)
161typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCH}/RUN_DIR/${SLURM_JOBID}_${$}}
162
163#====================================================
164#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
165typeset -r OUTCOMMAND_PATH=/tmp
166
167#====================================================
168#- HOST_MPIRUN_COMMAND
169typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time srun "}
170
171#====================================================
172#- Max number of arguments passed to nco operator or demigration command
173UNIX_MAX_LIMIT=360
174
175#====================================================
176#- set PackDefault to true
177PackDefault=true
178
179#====================================================
180#- Default number of MPI task for IPSL coupled model
181#- required for backward compatibility
182#-
183DEFAULT_NUM_PROC_OCE=5
184DEFAULT_NUM_PROC_CPL=1
185(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
186
187#====================================================
188#- Number of computing cores per node
189#-
190NB_CORE_PER_NODE=40
191
192#D-#==================================================
193#D-function IGCM_sys_defineArchives
194#D-* Purpose:
195#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
196#D-* Define ARCHIVE : Dedicated to large files
197#D-* Define STORAGE : Dedicated to small/medium files
198#D-* Define R_OUT   : Output tree located on ARCHIVE
199#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monsitoring and atlas, and/or small files)
200#D-* Define R_BUF   : Output tree located on SCRATCH hosting files waiting for rebuild or pack processes
201#D-* if SpaceName=TEST everything is stored on SCRATCH
202#D-* Examples:
203#D-
204function IGCM_sys_defineArchives {
205  IGCM_debug_PushStack "IGCM_sys_defineArchives"
206
207  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
208  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = XDEFAULT ]; then
209    # The variable DataProject is not set in in section UserChoices in config.card or it is set to DEFAULT
210    # Default option: Set DataProject according to the project used for submission
211    DataProject=${PROJECT}
212  else
213    # Use the project set by the variable DataProject in section UserChoices in config.card
214    IGCM_debug_Print 1 "Use DataProject as set in config.card : ${config_UserChoices_DataProject}"
215    DataProject=${config_UserChoices_DataProject}
216  fi
217  IGCM_debug_Print 1 "Data project for output to be used: ${DataProject}"
218
219  #====================================================
220  #- RUN_DIR_PATH : Temporary working directory (=> TMP)
221  #- DataProject = for is the dataproject for Training account at Idris
222  #- they are not store in rech but in another partition
223  if [ X${DataProject} = Xfor  ]; then
224      typeset -r RUN_DIR_PATH=$SCRATCH/RUN_DIR/${SLURM_JOBID}_${$}
225  else
226      typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/gpfsscratch/rech/${DataProject}/${LOGIN}/RUN_DIR/${SLURM_JOBID}_${$}}
227  fi
228
229  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
230    #====================================================
231    #- ARCHIVE (dedicated to large files)
232    ARCHIVE=${config_UserChoices_ARCHIVE}
233  else
234      if [ X${DataProject} = Xfor  ]; then
235          ARCHIVE=$STORE
236      else
237          #====================================================
238          #- ARCHIVE (dedicated to large files)
239          ARCHIVE=/gpfsstore/rech/${DataProject}/${LOGIN}
240      fi
241  fi
242
243  if [ ! X${config_UserChoices_STORAGE} = X ]; then
244    #====================================================
245    #- STORAGE (dedicated to small/medium files)
246    STORAGE=${config_UserChoices_STORAGE}
247  else
248    #====================================================
249    #- STORAGE (dedicated to small/medium files)
250      if [ X${DataProject} = Xfor  ]; then
251          STORAGE=$WORK
252      else
253          STORAGE=/gpfswork/rech/${DataProject}/${LOGIN}
254      fi
255  fi
256
257  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
258    #====================================================
259    #- R_OUT
260    if [ X${DataProject} = Xfor  ]; then
261        R_OUT=$SCRATCH/IGCM_OUT
262    else
263        R_OUT=/gpfsscratch/rech/${DataProject}/${LOGIN}/IGCM_OUT
264    fi
265    #====================================================
266    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
267    R_FIG=${R_OUT}
268
269    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
270
271  else
272    #====================================================
273    #- R_OUT
274    R_OUT=${ARCHIVE}/IGCM_OUT
275
276    #====================================================
277    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
278    R_FIG=${STORAGE}/IGCM_OUT
279  fi
280
281  #====================================================
282  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
283  CMIP6_BUF=${STORAGE}/IGCM_OUT
284
285  #====================================================
286  #- R_BUF (ONLY FOR double copy an scratch)
287 
288  if [ X${DataProject} = Xfor  ]; then
289      R_BUF=$SCRATCH/IGCM_OUT
290  else
291      R_BUF=/gpfsscratch/rech/${DataProject}/${LOGIN}/IGCM_OUT
292  fi
293
294  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
295  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
296  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
297
298  IGCM_debug_PopStack "IGCM_sys_defineArchives"
299}
300
301#D-#==================================================
302#D-function IGCM_sys_RshArchive
303#D-* Purpose: Archive rsh command
304#D-* Examples:
305#D-
306function IGCM_sys_RshArchive {
307  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
308  /bin/ksh <<-EOF
309    ${@}
310EOF
311  status=$?
312  if [ ${status} -gt 0 ] ; then
313    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
314    IGCM_debug_Exit "IGCM_sys_RshArchive"
315  fi
316  IGCM_debug_PopStack "IGCM_sys_RshArchive"
317}
318
319#D-#==================================================
320#D-function IGCM_sys_RshArchive_NoError
321#D-* Purpose: Archive rsh command, without error
322#D-*          used only in monitoring.job
323#D-* Examples:
324#D-
325function IGCM_sys_RshArchive_NoError {
326  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
327  /bin/ksh <<-EOF
328    ${@} 2> /dev/null
329EOF
330  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
331}
332
333#D-#==================================================
334#D-function IGCM_sys_MkdirArchive
335#D-* Purpose: Mkdir on Archive
336#D-* Examples:
337#D-
338function IGCM_sys_MkdirArchive {
339  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
340  if ( $DEBUG_sys ) ; then
341    echo "IGCM_sys_MkdirArchive :" $@
342  fi
343  #- creation de repertoire sur le serveur fichier
344  if [ ! -d ${1} ]; then
345    \mkdir -p $1
346    status=$?
347
348    if [ ${status} -gt 0 ] ; then
349      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
350      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
351    fi
352  fi
353  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
354}
355
356#D-#==================================================
357#D-function IGCM_sys_TestDirArchive
358#D-* Purpose: Test Directory that must exists on Archive
359#D-* Examples:
360#D-
361function IGCM_sys_TestDirArchive {
362  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
363  if ( $DEBUG_sys ) ; then
364    echo "IGCM_sys_TestDirArchive :" $@
365  fi
366  typeset ExistFlag
367  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
368  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
369  return ${ExistFlag}
370}
371
372#D-#==================================================
373#D-function IGCM_sys_IsFileArchived
374#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
375#D-* Examples:
376#D-
377function IGCM_sys_IsFileArchived {
378  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
379  if ( $DEBUG_sys ) ; then
380    echo "IGCM_sys_IsFileArchived :" $@
381  fi
382  typeset IsArchivedFlag
383# To be implemented on Jean Zay
384#  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
385  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
386
387  return ${IsArchivedFlag}
388}
389
390#D-#==================================================
391#D-function IGCM_sys_TestFileArchive
392#D-* Purpose: Test file that must NOT EXISTS on Archive
393#D-* Examples:
394#D-
395function IGCM_sys_TestFileArchive {
396  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
397  typeset ExistFlag
398  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
399  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
400
401  return ${ExistFlag}
402}
403
404#D-#==================================================
405#D-function IGCM_sys_CountFileArchive
406#D-* Purpose: Count files on Archive filesystem
407#D-* Examples:
408#D-
409function IGCM_sys_CountFileArchive {
410  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
411  ls ${@} 2>/dev/null | wc -l
412  if [ $? -gt 0 ] ; then
413    echo "IGCM_sys_CountFileArchive : erreur."
414  fi
415  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
416}
417
418#D-#==================================================
419#D-function IGCM_sys_Tree
420#D-* Purpose: Tree directories with files on ${ARCHIVE}
421#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
422#D-
423function IGCM_sys_Tree {
424  IGCM_debug_PushStack "IGCM_sys_Tree" $@
425  if ( $DEBUG_sys ) ; then
426    echo "IGCM_sys_Tree :" $@
427  fi
428
429  \ls -lR ${@}
430
431  IGCM_debug_PopStack "IGCM_sys_Tree"
432}
433
434#D-#==================================================
435#D-function IGCM_sys_Qsub
436#D-* Purpose: Qsub new job
437#D-* Examples:
438#D-
439function IGCM_sys_Qsub {
440  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
441  if ( $DEBUG_sys ) ; then
442    echo "IGCM_sys_Qsub :" $@
443  fi
444  typeset options status
445  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
446
447  /usr/bin/time sbatch ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
448  status=$?
449
450  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
451  if [ ${status} -gt 0 ] ; then
452    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
453    IGCM_debug_Exit "IGCM_sys_Qsub"
454  else
455    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
456  fi
457  IGCM_debug_PopStack "IGCM_sys_Qsub"
458}
459
460#D-#==================================================
461#D-function IGCM_sys_QsubPost
462#D-* Purpose: Qsub new job on scalaire
463#D-* Examples:
464#D-
465function IGCM_sys_QsubPost {
466  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
467
468  BAK_DIR=$PWD 
469  cd ${SUBMIT_DIR}
470
471  if ( $DEBUG_sys ) ; then
472    echo "IGCM_sys_QsubPost :" $@
473  fi
474  typeset options status
475  options="-o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
476
477  sbatch ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
478  status=$?
479
480  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
481  if [ ${status} -gt 0 ] ; then
482    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
483    IGCM_debug_Exit "IGCM_sys_QsubPost"
484  else
485    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
486  fi
487
488  cd ${BAK_DIR}
489
490  IGCM_debug_PopStack "IGCM_sys_QsubPost"
491}
492
493#D-*************************
494#D- File transfer functions
495#D-*************************
496#D-
497
498#D-#==================================================
499#D-function IGCM_sys_RmRunDir
500#D-* Purpose: rm tmpdir (dummy function most of the time batch
501#D-                      scheduler will do the job)
502#D-* Examples:
503#D-
504function IGCM_sys_RmRunDir {
505  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
506  if ( $DEBUG_sys ) ; then
507    echo "IGCM_sys_RmRunDir :" $@
508  fi
509
510  typeset status
511
512  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
513  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
514  status=$?
515
516  if [ ${status} -gt 0 ] ; then
517    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
518    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
519    IGCM_debug_Exit "IGCM_sys_RmRunDir"
520  else
521    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
522  fi
523  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
524}
525
526#D-#==================================================
527#D-function IGCM_sys_Put_Dir
528#D-* Purpose: Copy a complete directory on $(ARCHIVE)
529#D-* Examples:
530#D-
531function IGCM_sys_Put_Dir {
532  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
533  if ( $DEBUG_sys ) ; then
534    echo "IGCM_sys_Put_Dir :" $@
535  fi
536  if [ $DRYRUN = 0 ]; then
537    if [ ! -d ${1} ] ; then
538      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
539      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
540      return
541    fi
542
543    typeset status
544
545    # Only if we use rsync
546    #IGCM_sys_TestDirArchive $( dirname $2 )
547    #
548    #USUAL WAY
549    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
550    status=$?
551
552    if [ ${status} -gt 0 ] ; then
553      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
554      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
555      IGCM_debug_Exit "IGCM_sys_Put_Dir"
556    else
557      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
558    fi
559  fi
560  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
561}
562
563#D-#==================================================
564#D-function IGCM_sys_Get_Dir
565#D-* Purpose: Copy a complete directory from ${ARCHIVE}
566#D-* Examples:
567#D-
568function IGCM_sys_Get_Dir {
569  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
570  if ( $DEBUG_sys ) ; then
571    echo "IGCM_sys_Get_Dir :" $@
572  fi
573  if [ $DRYRUN = 0 ]; then
574    typeset NB_ESSAI DELAI status i
575    # number of tentative
576    NB_ESSAI=3
577    # time delay between tentative
578    DELAI=2
579
580    #
581    # USUAL WAY
582    # add here command to demigrate all offline files to reduce time of this command
583    # actually on Jean Zay files are not archive on bands
584
585    i=0
586    while [ $i -lt $NB_ESSAI ] ; do
587      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
588      status=$?
589      if [ ${status} -gt 0 ] ; then
590        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
591        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
592        sleep $DELAI
593      else
594        break
595      fi
596      (( i = i + 1 ))
597    done
598
599    if [ ${status} -gt 0 ] ; then
600      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
601      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
602      IGCM_debug_Exit "IGCM_sys_Get_Dir"
603    else
604      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
605    fi
606  fi
607  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
608}
609
610#D-#==================================================
611#D-function IGCM_sys_Put_Rest
612#D-* Purpose: Put computied restarts on ${ARCHIVE}.
613#D-           File and target directory must exist.
614#D-* Examples:
615#D-
616function IGCM_sys_Put_Rest {
617  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
618  if ( $DEBUG_sys ) ; then
619    echo "IGCM_sys_Put_Rest :" $@
620  fi
621  if [ $DRYRUN = 0 ]; then
622    if [ ! -f ${1} ] ; then
623        # The file is missing
624        if [ X${3} = X'OPTIONAL' ] ; then
625            echo "WARINING : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
626            IGCM_debug_Print 1 "IGCM_sys_Put_Rest: Optional restart file ${1} is missing, now continue for the next"
627        else
628            echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
629            IGCM_debug_Exit "IGCM_sys_Put_Rest"
630        fi
631
632    else
633      # The file exist
634
635    typeset status
636    #
637    # USUAL WAY
638    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
639    status=$?
640
641#       #RSYNC WITH NETWORK SSH CALL
642#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
643#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
644
645#       #RSYNC WITH NFS USE
646#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
647#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
648
649#       status=$?
650#       IGCM_sys_Rsync_out $status
651
652#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
653#       (( status=status+$? ))
654
655    if [ ${status} -gt 0 ] ; then
656      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
657      [ -f ${1} ] && ls -l ${1}
658      [ -f ${2} ] && ls -l ${2}
659      [ -f ${2}/${1} ] && ls -l ${2}/${1}
660      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
661      IGCM_debug_Exit "IGCM_sys_Put_Rest"
662    else
663
664      if [ X${JobType} = XRUN ] ; then
665        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
666        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
667      fi
668
669      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
670    fi
671    fi
672  fi
673  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
674}
675
676#D-#==================================================
677#D-function IGCM_sys_Put_Out
678#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
679#D-* Examples:
680#D-
681function IGCM_sys_Put_Out {
682  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
683  if ( $DEBUG_sys ) ; then
684    echo "IGCM_sys_Put_Out :" $@
685  fi
686
687  typeset NB_ESSAI DELAI status i exist skip
688  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
689
690  # number of tentative
691  NB_ESSAI=3
692  # time delay between tentative
693  DELAI=2
694
695  if [ $DRYRUN = 0 ]; then
696    if [ ! -f ${1} ] ; then
697      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
698      IGCM_debug_PopStack "IGCM_sys_Put_Out"
699      return 1
700    fi
701    #
702    IGCM_sys_MkdirArchive $( dirname $2 )
703    #
704    exist=false
705    skip=false
706    if [ -f $2 ] ; then
707      IGCM_debug_Print 1 "$2 already exist"
708    # add here command to demigrate all offline files to reduce time of this command
709    # actually on Jean Zay files are not archive on bands
710
711      exist=true
712      if [ "X$( diff $1 $2 )" = X ] ; then
713        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
714        skip=true
715      else
716        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
717        skip=false
718      fi
719    fi
720    #
721    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
722      IGCM_sys_Chmod u+w $2
723    fi
724
725    if [ X${skip} = Xfalse ] ; then
726      i=0
727      while [ $i -lt $NB_ESSAI ] ; do
728        # Identify file system
729        fileDeviceNumberInHex=$( stat -c %d $1 )
730        status=$?
731        if [ ${status} -gt 0 ] ; then
732          IGCM_debug_Exit "IGCM_sys_Put_Out"
733        fi
734        # Identify file system
735        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
736        status=$?
737        if [ ${status} -gt 0 ] ; then
738          IGCM_debug_Exit "IGCM_sys_Put_Out"
739        fi
740
741        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
742          # They are not on the same device. USUAL WAY
743          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
744          status=$?
745        else
746          # They are on the same device. NOT SO USUAL WAY
747          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
748          status=$?
749        fi
750        if [ ${status} -gt 0 ]; then
751          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
752          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
753          [ -f ${1} ] && ls -l ${1}
754          [ -f ${2} ] && ls -l ${2}
755          [ -f ${2}/${1} ] && ls -l ${2}/${1}
756          sleep $DELAI
757        else
758          break
759        fi
760        (( i = i + 1 ))
761      done
762    fi
763
764#       #RSYNC WITH NETWORK SSH CALL
765#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
766#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
767
768#       #RSYNC WITH NFS USE
769#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
770#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
771
772#       status=$?
773#       IGCM_sys_Rsync_out $status
774
775#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
776#       (( status=status+$? ))
777
778    if [ ${status} -gt 0 ] ; then
779      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
780      [ -f ${1} ] && ls -l ${1}
781      [ -f ${2} ] && ls -l ${2}
782      [ -f ${2}/${1} ] && ls -l ${2}/${1}
783      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
784      IGCM_debug_Exit "IGCM_sys_Put_Out"
785    else
786
787      if [ X${JobType} = XRUN ] ; then
788        if [ X${3} = X ] ; then
789          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
790          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
791        fi
792      fi
793
794      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
795    fi
796  fi
797  IGCM_debug_PopStack "IGCM_sys_Put_Out"
798  return 0
799}
800
801#D-#==================================================
802#D-function IGCM_sys_Get
803#D-* Purpose: Get a file from ${ARCHIVE}
804#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
805#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
806function IGCM_sys_Get {
807  IGCM_debug_PushStack "IGCM_sys_Get" $@
808
809  typeset DEST dm_liste target file_work
810  typeset NB_ESSAI DELAI status i
811
812  if ( $DEBUG_sys ) ; then
813    echo "IGCM_sys_Get :" $@
814  fi
815
816  # number of tentative
817  NB_ESSAI=3
818  # time delay between tentative
819  DELAI=2
820
821  if [ $DRYRUN -le 2 ]; then
822    if [ X${1} = X'/l' ] ; then
823      eval set +A dm_liste \${${2}}
824    else
825      eval set +A dm_liste ${1}
826    fi
827    eval DEST=\${${#}}
828    # add here command to demigrate all offline files to reduce time of this command
829    # actually on Jean Zay files are not archive on bands
830
831    status=$?
832    if [ ${status} -gt 0 ] ; then
833      echo "WARNING IGCM_sys_Get : error code ${status}"
834      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
835      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
836    fi
837
838    #if [ ${status} -gt 0 ] ; then
839    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
840    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
841    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
842    #    sleep 30
843    #    echo "We try another time"
844    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
845    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
846    #    status=$?
847    #    if [ ${status} -gt 0 ] ; then
848    #      echo "ERROR IGCM_sys_Get : again demigration error :"
849    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
850    #      IGCM_debug_Exit "IGCM_sys_Get"
851    #    fi
852    #  else
853    #    echo "ERROR IGCM_sys_Get : demigration error :"
854    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
855    #    IGCM_debug_Exit "IGCM_sys_Get"
856    #  fi
857    #fi
858
859    #   #RSYNC WITH NETWORK SSH CALL
860    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
861    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
862
863    #   #RSYNC WITH NFS USE
864    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
865    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
866
867    #   status=$?
868    #   IGCM_sys_Rsync_out $status
869
870    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
871    #   (( status=status+$? ))
872
873    #USUAL WAY
874    if [ X${1} = X'/l' ] ; then
875      for target in ${dm_liste[*]} ; do
876        local_file=$( basename ${target} )
877        # test if the target file is present before the loop
878        IGCM_sys_TestFileArchive ${target}
879        status=$?
880        if [ ${status} -gt 0 ] ; then
881          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
882          IGCM_debug_Exit "IGCM_sys_Get"
883        else
884          i=0
885          while [ $i -lt $NB_ESSAI ] ; do
886            #if [ X${DoLink} = Xtrue ] ; then
887            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
888            #  status=$?
889            #  else
890            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
891            #  status=$?
892            #fi
893            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
894            status=$?
895            if [ ${status} -gt 0 ]; then
896              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
897              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
898              sleep $DELAI
899            else
900              break
901            fi
902            (( i = i + 1 ))
903          done
904          if [ ${status} -gt 0 ] ; then
905            echo "IGCM_sys_Get : error"
906            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
907            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
908            IGCM_debug_Exit "IGCM_sys_Get"
909          else
910            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
911          fi
912        fi
913      done
914    else
915      i=0
916      while [ $i -lt $NB_ESSAI ] ; do
917        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
918        status=$?
919        if [ ${status} -gt 0 ]; then
920          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
921          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
922          sleep $DELAI
923        else
924          break
925        fi
926        (( i = i + 1 ))
927      done
928      if [ ${status} -gt 0 ] ; then
929        echo "IGCM_sys_Get : error"
930        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
931        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
932        IGCM_debug_Exit "IGCM_sys_Get"
933      else
934        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
935      fi
936    fi
937  fi
938  IGCM_debug_PopStack "IGCM_sys_Get"
939}
940
941#D-#==================================================
942#D-function IGCM_sys_GetDate_Monitoring
943#D-* Purpose: get the last year for which the monitoring has been computed
944#D-* Examples:
945#D-
946function IGCM_sys_GetDate_Monitoring {
947  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
948  if ( $DEBUG_sys ) ; then
949    echo "IGCM_sys_GetDate_Monitoring :" $@
950  fi
951
952  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
953
954  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
955}
956
957#D-#==================================================
958#D-function IGCM_sys_Dods_Rm
959#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
960#D-* Examples:
961#D-
962function IGCM_sys_Dods_Rm {
963  if ( $DEBUG_sys ) ; then
964    echo "IGCM_sys_Dods_Rm :" $@
965  fi
966  typeset status
967  if [ $DRYRUN = 0 ]; then
968
969#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
970#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
971#      echo "Nothing has been done."
972#      return
973#    fi
974
975    if [ "$#" -eq 1 ]; then
976      thredds_rm ${R_DODS}/${1}
977      status=$?
978    else
979        thredds_rm ${R_DODS}
980      status=$?
981    fi
982
983#    if [ ${status} -gt 0 ] ; then
984#      echo "IGCM_sys_Dods_Rm : error."
985#      cat out_dods_rm
986#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
987#    else
988#      rm out_dods_rm
989#    fi
990
991  fi
992  return $status
993}
994
995#D-#==================================================
996#D-function IGCM_sys_Dods_Cp
997#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
998#D-* Examples:
999#D-
1000function IGCM_sys_Dods_Cp {
1001  if ( $DEBUG_sys ) ; then
1002    echo "IGCM_sys_Dods_Cp :" $@
1003  fi
1004  typeset status
1005  if [ $DRYRUN = 0 ]; then
1006
1007#    if [ ! -d ${R_SAVE}/${1} ] ; then
1008#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1009#      echo "Nothing has been done."
1010#      return
1011#    fi
1012
1013
1014      thredds_cp ${1} ./${R_DODS}/.
1015
1016    status=$?
1017
1018#       if [ ${status} -gt 0 ] ; then
1019#           echo "IGCM_sys_Dods_Cp : error."
1020#           cat out_dods_cp
1021#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1022#       else
1023#           rm out_dods_cp
1024#       fi
1025
1026  fi
1027  return $status
1028}
1029
1030#D-#==================================================
1031#D-function IGCM_sys_Put_Dods
1032#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1033#D-* Examples:
1034#D-
1035function IGCM_sys_Put_Dods {
1036  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1037  if ( $DEBUG_sys ) ; then
1038    echo "IGCM_sys_Put_Dods :" $@
1039  fi
1040  typeset status
1041  if [ $DRYRUN = 0 ]; then
1042    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1043      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1044      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1045      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1046      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1047      return
1048    fi
1049
1050    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1051      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1052      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1053      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1054      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1055      return
1056    fi
1057    #
1058    if [ -d ${R_SAVE}/${1} ] ; then
1059      cd ${R_SAVE}
1060    elif [ -d ${R_FIGR}/${1} ] ; then
1061      cd ${R_FIGR}
1062    fi
1063
1064    IGCM_sys_Dods_Rm ${1}
1065    IGCM_sys_Dods_Cp ${1}
1066    status=0
1067
1068    if [ ${status} -gt 0 ] ; then
1069      echo "IGCM_sys_Put_Dods : error."
1070      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1071    fi
1072  fi
1073  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1074}
1075
1076##############################################################
1077# REBUILD OPERATOR
1078
1079#D-#==================================================
1080#D-function IGCM_sys_sync
1081#D-* Purpose: flush buffer on disk
1082#D-* Examples:
1083#D-
1084function IGCM_sys_sync {
1085  IGCM_debug_PushStack "IGCM_sys_sync" $@
1086  if ( $DEBUG_sys ) ; then
1087    echo "IGCM_sys_sync :" $@
1088  fi
1089
1090  /bin/sync
1091
1092  IGCM_debug_PopStack "IGCM_sys_sync"
1093}
1094
1095############################################################
1096# Activate Running Environnment Variables
1097
1098#D-#==================================================
1099#D-function IGCM_sys_activ_variables
1100#D-* Purpose: set environement variables prior to execution
1101#D-* Examples:
1102#D-
1103function IGCM_sys_activ_variables {
1104  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1105  if ( $DEBUG_sys ) ; then
1106    echo "IGCM_sys_activ_variables"
1107  fi
1108
1109# --------------------------------------------------------------------
1110#D- MODULE specifications
1111# --------------------------------------------------------------------
1112  # Source the file EnvFile containing all module load needed to run the model.
1113  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1114  # it will be used. Otherwise a default file will be used.
1115  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1116      # EnvFile is set in config.card and the file exists
1117      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1118      EnvFile=${config_UserChoices_EnvFile}
1119  else
1120      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1121      module purge > /dev/null 2>&1
1122      EnvFile=/gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay
1123  fi
1124  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1125  IGCM_debug_Print 1 ${EnvFile}
1126  . ${EnvFile}
1127
1128  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Jean-Zay. "
1129  module list
1130
1131# --------------------------------------------------------------------
1132#D- MPI specifications
1133# --------------------------------------------------------------------
1134
1135# --------------------------------------------------------------------
1136#D- Other specifications
1137# --------------------------------------------------------------------
1138
1139  ulimit -s unlimited
1140  if [ ${executionType} -eq 2 ] ; then
1141    export SLURM_HOSTFILE=./hostlist
1142  fi
1143
1144
1145  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1146}
1147
1148############################################################
1149# Desactivate Running Environnment Variables
1150
1151#D-#==================================================
1152#D-function IGCM_sys_desactiv_variables
1153#D-* Purpose: unset environement variables after execution
1154#D-* Examples:
1155#D-
1156function IGCM_sys_desactiv_variables {
1157  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1158  if ( $DEBUG_sys ) ; then
1159    echo "IGCM_sys_desactiv_variables"
1160  fi
1161# --------------------------------------------------------------------
1162#D- MPI specifications
1163# --------------------------------------------------------------------
1164  if [ ${executionType} -eq 2 ] ; then
1165    unset SLURM_HOSTFILE
1166  fi
1167# --------------------------------------------------------------------
1168#D- Other specifications
1169# --------------------------------------------------------------------
1170
1171  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1172}
1173
1174############################################################
1175# Update job headers to be used by the scheduler
1176
1177#D-#==================================================
1178#D-function IGCM_sys_updateHeaders
1179#D-* Purpose: Update job headers to be used by the scheduler
1180#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1181#D-
1182function IGCM_sys_updateHeaders {
1183  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1184  if ( $DEBUG_sys ) ; then
1185    echo "IGCM_sys_updateHeaders"
1186  fi
1187  typeset file
1188  file=$1
1189
1190  if [ ${executionType} -eq 1 ] ; then
1191    # MPMD + MPI
1192    sed -e "/::openMPthreads::/d"                  \
1193        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1194      ${file} > ${file}.tmp
1195
1196  elif [ ${executionType} -eq 2 ] ; then
1197    # MPMD + MPI + OMP
1198      (( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
1199      [ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
1200      sed -e "/::openMPthreads::/d"                 \
1201        -e "s/::JobNumProcTot::/${mpiTasks}/"       \
1202        -e "/ntasks/i\#SBATCH --nodes=${nodeNumber}"\
1203        -e "/ntasks/i\#SBATCH --exclusive"          \
1204         ${file} > ${file}.tmp
1205  elif [ ${executionType} -eq 3 ] ; then
1206    # SPMD + MPI/OMP
1207    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1208        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1209      ${file} > ${file}.tmp
1210
1211  elif [ ${executionType} -eq 4 ] ; then
1212    # SPMD + MPI only
1213    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1214        -e "/::openMPthreads::/d"                  \
1215      ${file} > ${file}.tmp
1216
1217  elif [ ${executionType} -eq 5 ] ; then
1218    # SPMD + OMP only
1219    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1220        -e "/::JobNumProcTot::/d"                  \
1221      ${file} > ${file}.tmp
1222
1223  elif [ ${executionType} -eq 6 ] ; then
1224    # SEQUENTIAL THEN
1225    sed -e "s/::JobNumProcTot::/1/"                \
1226        -e "/::openMPthreads::/d"                  \
1227      ${file} > ${file}.tmp
1228
1229  fi
1230
1231  IGCM_sys_Mv ${file}.tmp ${file}
1232
1233  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1234}
1235
1236############################################################
1237# Build MPI/OMP scripts run file (dummy function)
1238
1239#D-#==================================================
1240#D-function IGCM_sys_build_run_file
1241#D-* Purpose: build run file (deprecated)
1242#D-* Examples:
1243#D-
1244function IGCM_sys_build_run_file {
1245  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1246}
1247
1248############################################################
1249# Build MPI/OMP scripts
1250
1251#D-#==================================================
1252#D-function IGCM_sys_build_execution_scripts
1253#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1254#D-* Examples:
1255#D-
1256function IGCM_sys_build_execution_scripts
1257{
1258  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1259  if ( $DEBUG_sys ) ; then
1260    echo "IGCM_sys_build_execution_scripts " $@
1261  fi
1262  IGCM_debug_Print 1 "executionType= ${executionType}"
1263  EXECUTION=${HOST_MPIRUN_COMMAND}
1264
1265  if ( ${OK_PARA_MPMD} ) ; then
1266
1267    # MPMD mode
1268    # 1 MPI only : executionType=1
1269    # 2 MPI/OpenMP : executionType=2
1270
1271    if [ -f run_file ] ; then
1272      IGCM_sys_Rm -f run_file
1273    fi
1274    touch run_file
1275
1276    # case 1 : Only MPI (MPMD)
1277    if  ( ! ${OK_PARA_OMP} ) ; then
1278      # Build run_file
1279      current_core=0
1280      # First loop on the components for the coupler ie oasis (only if oasis3)
1281      # the coupler ie oasis3 must be the first one
1282      for comp in ${config_ListOfComponents[*]} ; do
1283
1284        eval ExeNameIn=\${config_Executable_${comp}[0]}
1285        eval ExeNameOut=\${config_Executable_${comp}[1]}
1286
1287        # Only if the component has an executable
1288        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1289
1290          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1291          (( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 ))
1292          echo "${current_core}-${end_core} ./${ExeNameOut}" >> run_file
1293          (( current_core = ${end_core} + 1 ))
1294        fi
1295      done
1296
1297      EXECUTION="${HOST_MPIRUN_COMMAND} --multi-prog ./run_file"
1298
1299      IGCM_sys_Chmod u+x run_file
1300      if ( $DEBUG_sys ) ; then
1301        echo "run_file contains : "
1302        cat run_file
1303      fi
1304
1305    else 
1306
1307    # 2 MPI/OpenMP : executionType=2
1308
1309    # MPI-OpenMP (MPMD)
1310    # export SLURM_HOSTFILE=./hostlist
1311    # srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
1312    # example of  hostlist file :
1313    # r3i3n33
1314    # r3i3n33
1315    # ...
1316    # example of run_file :
1317    # 0-70 ./prog_lmdz.x.sh %o %t
1318    # 71-430 ./prog_opa.xx.sh %o %t
1319    # 431-431 ./prog_xios.x.sh %o %t
1320    # examples of prog_file :
1321    # prog_lmdz.x.sh :
1322    # (( init = 0 + $1 ))
1323    # (( index = init * 10 ))
1324    # (( slot = index % 40 ))
1325    # taskset -c $slot-$((slot + 10 - 1)) ./script_lmdz.x.ksh
1326    # that will become
1327    # taskset -c 0-9 ./script_lmdz.x.ksh
1328    # ...
1329    # with script_lmdz.x.ksh
1330    # export OMP_STACKSIZE=3g
1331    # export OMP_PLACES=cores
1332    # export OMP_NUM_THREADS=10
1333    # ./lmdz.x > out_lmdz.x.out.${SLURM_PROCID} 2>out_lmdz.x.err.${SLURM_PROCID}
1334
1335    #  Hosts treatment
1336    _bkIFS=$IFS;
1337        IFS=$'\n'; set -f
1338        listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
1339        IFS=$_bkIFS; set +f
1340        rm -f hostlist
1341
1342     # Loop on the components to build run_file and script_exec files
1343        rank=0
1344        current_core=0
1345        current_core_mpi=0
1346
1347      for comp in ${config_ListOfComponents[*]} ; do
1348
1349        eval ExeNameIn=\${config_Executable_${comp}[0]}
1350        eval ExeNameOut=\${config_Executable_${comp}[1]}
1351
1352        # Not possible if oasis has an executable (i.e old version of oasis3)
1353        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1354          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1355          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1356          IGCM_debug_Verif_Exit
1357        fi
1358
1359        # Only if we really have an executable for the component :
1360        if [ "X${ExeNameOut}" != X\"\" ] ; then
1361
1362          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1363          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1364          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1365     
1366
1367          # Build script files
1368
1369          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1370          echo ""  >> script_${ExeNameOut}.ksh
1371          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1372          # Check if the number of threads is correct
1373              case ${comp_proc_omp_loc} in
1374                  2|4|5|10|20)
1375                      IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1376                      ;;
1377                  *)
1378                      IGCM_debug_Exit "ERROR with OMP parameters !"
1379                      IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1380                      IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1381                      IGCM_debug_Verif_Exit
1382                      ;;
1383              esac
1384            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1385            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1386            echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}"  >> script_${ExeNameOut}.ksh
1387
1388          fi
1389
1390          # to have out/err per process on different files
1391          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1392
1393          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1394
1395        # Build run_file
1396        # Only if the component has an executable
1397        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1398
1399          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1400          (( end_core = ${current_core_mpi} + ${comp_proc_mpi_loc} - 1 ))
1401          echo "${current_core_mpi}-${end_core} ./prog_${ExeNameOut}.sh %o %t" >> run_file
1402          (( current_core_mpi = ${end_core} + 1 ))
1403        fi
1404
1405          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1406              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1407          else
1408              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1409          fi
1410
1411          # Build configuration file
1412 
1413              echo "#!/bin/sh" > prog_${ExeNameOut}.sh
1414          echo "(( init = $current_core + \$1 ))" >> prog_${ExeNameOut}.sh
1415          echo "(( index = init * $comp_proc_omp_loc ))" >> prog_${ExeNameOut}.sh
1416          echo "(( slot = index % 40 ))" >> prog_${ExeNameOut}.sh
1417              echo "echo ${ExeNameOut} taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1))" >> prog_${ExeNameOut}.sh
1418          echo "taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1)) ./script_${ExeNameOut}.ksh" >> prog_${ExeNameOut}.sh
1419
1420          IGCM_sys_Chmod u+x prog_${ExeNameOut}.sh
1421
1422        # Build hostlist file
1423
1424          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1425             (( index_host = current_core / NB_CORE_PER_NODE ))
1426             host_value=${listnodes[${index_host}]}
1427             echo "$host_value" >> hostlist
1428             if [ ${DRYRUN_DEBUG} = 4 ] ; then
1429                 echo "node_${index_host}_X" >> hostlist_template
1430             fi
1431             (( current_core = current_core + offset_comp_proc_loc ))
1432          done
1433        fi
1434      done
1435
1436      ## variable added to stop after 60s instead of 600s by default.
1437      ## This is used when no error comes from executables and when something stopped an executable without notice.
1438      export SLURM_WAIT=60
1439
1440      EXECUTION="${HOST_MPIRUN_COMMAND} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
1441
1442      IGCM_sys_Chmod u+x run_file
1443      if ( $DEBUG_sys ) ; then
1444        echo "run_file contains : "
1445        cat run_file
1446      fi
1447
1448    fi # if ${OK_PARA_MPMD}
1449
1450  else
1451  # Only one executable (SPMD mode):  executionType=3, 4, 5 and 6
1452
1453    for comp in ${config_ListOfComponents[*]} ; do
1454
1455      # Only if we really have an executable for the component :
1456      eval ExeNameOut=\${config_Executable_${comp}[1]}
1457      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1458
1459        # Build script files
1460
1461        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1462        echo ""  >> script_${ExeNameOut}.ksh
1463        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1464
1465        if ( ${OK_PARA_OMP} ) ; then
1466            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1467            # Check if the number of threads is correct
1468            case ${comp_proc_omp_loc} in
1469                2|4|5|10|20)
1470                    IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1471                    ;;
1472                *)
1473                    IGCM_debug_Exit "ERROR with OMP parameters !"
1474                    IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1475                    IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1476                    IGCM_debug_Verif_Exit
1477                    ;;
1478            esac
1479            echo ""  >> script_${ExeNameOut}.ksh
1480            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1481            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1482            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1483        fi
1484
1485        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1486       
1487        # To have out/err per process on different files
1488        echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1489        EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1490
1491        IGCM_debug_Print 1 "sys Jean-Zay : script_${ExeNameOut}.ksh contains"
1492        cat script_${ExeNameOut}.ksh
1493
1494      fi
1495
1496    done
1497
1498  fi # ${OK_PARA_MPMD}
1499
1500  IGCM_debug_Print 1 "sys Jean-Zay : execution command is "
1501  IGCM_debug_Print 1 "$EXECUTION"
1502
1503  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1504}
1505
1506#D-#==================================================
1507#D-function IGCM_sys_check_path
1508#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1509#D-* do not point to an important use directory. Stop immediately in that case.
1510#D-* Examples:
1511#D-
1512function IGCM_sys_check_path {
1513  IGCM_debug_PushStack "IGCM_sys_check_path"
1514  if ( $DEBUG_sys ) ; then
1515    echo "IGCM_sys_check_path"
1516  fi
1517
1518  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORK} ] || [ X${RUN_DIR_PATH} = X${SCRATCH} ] || [ X${RUN_DIR_PATH} = X${STORE} ] ) ; then
1519    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1520    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1521    IGCM_debug_Exit "This will stop the job"
1522  elif ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X/gpfswork/rech/${DataProject}/${LOGIN} ] || [ X${RUN_DIR_PATH} = X/gpfsscratch/rech/${DataProject}/${LOGIN} ] || [ X${RUN_DIR_PATH} = X/gpfsstore/rech/${DataProject}/${LOGIN} ] ) ; then
1523    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1524    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1525    IGCM_debug_Exit "This will stop the job"
1526  fi
1527  IGCM_debug_PopStack "IGCM_sys_check_path"
1528}
1529
1530#D-#==================================================
1531#D-function IGCM_sys_check_quota
1532#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1533#D-* Examples:
1534#D-
1535function IGCM_sys_check_quota {
1536  IGCM_debug_PushStack "IGCM_sys_check_quota"
1537  echo "IGCM_sys_check_quota is desactivated on JeanZay because there is actually no way to check scratch quota on this computing center (2019.11.19)"
1538#  if ( $DEBUG_sys ) ; then
1539#    echo "IGCM_sys_check_quota"
1540#  fi
1541#  # Limit of quota (in %)
1542#  limit_quota=90
1543#
1544#  # Check of the volume
1545#  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1546#  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1547#
1548#  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1549#
1550#    unit_avail=${volume_avail: -1}
1551#    unit_quota=${volume_quota: -1}
1552#
1553#    if [ "${unit_quota}" = "*" ] ; then
1554#      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1555#      IGCM_debug_Print 1 "More than 100% of your quota is used"
1556#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1557#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1558#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1559#      IGCM_debug_Verif_Exit
1560#    fi
1561#
1562#    temp_avail=${volume_avail%%${volume_avail: -1}*}
1563#    temp_quota=${volume_quota%%${volume_quota: -1}*}
1564#
1565#    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1566#
1567#    # Convertion
1568#      if [ ${volume_avail: -1} = "T" ] ; then
1569#        (( temp_avail = temp_avail * 1000000000000 ))
1570#      elif [ ${volume_avail: -1} = "G" ] ; then
1571#        (( temp_avail = temp_avail * 1000000000 ))
1572#      elif [ ${volume_avail: -1} = "M" ] ; then
1573#        (( temp_avail = temp_avail * 1000000 ))
1574#      elif [ ${volume_avail: -1} = "k" ] ; then
1575#        (( temp_avail = temp_avail * 1000 ))
1576#      else
1577#        (( temp_avail = volume_avail ))
1578#      fi
1579#      if [ ${volume_quota: -1} = "T" ] ; then
1580#        (( temp_quota = temp_quota * 1000000000000 ))
1581#      elif [ ${volume_quota: -1} = "G" ] ; then
1582#        (( temp_quota = temp_quota * 1000000000 ))
1583#      elif [ ${volume_quota: -1} = "M" ] ; then
1584#        (( temp_quota = temp_quota * 1000000 ))
1585#      elif [ ${volume_quota: -1} = "k" ] ; then
1586#        (( temp_quota = temp_quota * 1000 ))
1587#      else
1588#        (( temp_quota = volume_quota ))
1589#      fi
1590#    fi
1591#
1592#    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1593##    echo "volume ratio is " $quota_volume
1594#
1595#    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1596#      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1597#      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1598#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1599#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1600#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1601#      IGCM_debug_Verif_Exit
1602#    fi
1603#
1604#  fi
1605#
1606## Check of the number of inodes
1607#
1608#  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1609#  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1610#
1611#  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1612#
1613#    unit_avail=${inode_avail: -1}
1614#    unit_quota=${inode_quota: -1}
1615#
1616#    if [ "${unit_quota}" = "*" ] ; then
1617#      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1618#      IGCM_debug_Print 1 "More than 100% of your quota is used"
1619#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1620#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1621#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1622#      IGCM_debug_Verif_Exit
1623#    fi
1624#
1625#    temp_avail=${inode_avail%%${inode_avail: -1}*}
1626#    temp_quota=${inode_quota%%${inode_quota: -1}*}
1627#
1628#    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1629#
1630#    # Convertion
1631#      if [ ${inode_avail: -1} = "T" ] ; then
1632#        (( temp_avail = temp_avail * 1000000000000 ))
1633#      elif [ ${inode_avail: -1} = "G" ] ; then
1634#        (( temp_avail = temp_avail * 1000000000 ))
1635#      elif [ ${inode_avail: -1} = "M" ] ; then
1636#        (( temp_avail = temp_avail * 1000000 ))
1637#      elif [ ${inode_avail: -1} = "k" ] ; then
1638#        (( temp_avail = temp_avail * 1000 ))
1639#      else
1640#        (( temp_avail = inode_avail ))
1641#      fi
1642#
1643#      if [ ${inode_quota: -1} = "T" ] ; then
1644#        (( temp_quota = temp_quota * 1000000000000 ))
1645#      elif [ ${inode_quota: -1} = "G" ] ; then
1646#        (( temp_quota = temp_quota * 1000000000 ))
1647#      elif [ ${inode_quota: -1} = "M" ] ; then
1648#        (( temp_quota = temp_quota * 1000000 ))
1649#      elif [ ${inode_quota: -1} = "k" ] ; then
1650#        (( temp_quota = temp_quota * 1000 ))
1651#      else
1652#        (( temp_quota = inode_quota ))
1653#      fi
1654#    fi
1655#    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1656##    echo "inode ratio is " $quota_inode
1657#
1658#    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1659#      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1660#      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1661#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1662#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1663#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1664#      IGCM_debug_Verif_Exit
1665#    fi
1666#  fi
1667  IGCM_debug_PopStack "IGCM_sys_check_quota"
1668}
1669
1670#D-#==================================================
1671#D-function IGCM_sys_projectAccounting
1672#D-* Purpose: store project accounting information in a file
1673#D-* Examples:
1674#D-
1675function IGCM_sys_projectAccounting {
1676  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1677  if ( $DEBUG_sys ) ; then
1678    echo "IGCM_sys_check_quota"
1679  fi
1680#need to implemented on Jean Zay
1681  touch $1
1682#  ssh irene191 /usr/bin/ccc_myproject > $1
1683
1684  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1685}
1686
1687#D-#==================================================
1688#D-function IGCM_sys_getJobSchedulerID
1689#D-* Purpose: Get the job ID during execution
1690#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1691#D-
1692function IGCM_sys_getJobSchedulerID {
1693  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1694  if ( $DEBUG_sys ) ; then
1695    echo "IGCM_sys_getJobSchedulerID"
1696  fi
1697
1698  eval ${1}=${SLURM_JOBID}
1699
1700  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1701}
1702
1703#D-#==================================================
1704#D-function IGCM_sys_GetJobID
1705#D-* Purpose: Get the job ID from the JobName
1706#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1707#D-
1708function IGCM_sys_GetJobID {
1709  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1710  if ( $DEBUG_sys ) ; then
1711    echo "IGCM_sys_GetJobID"
1712  fi
1713
1714  ID=$( squeue -u $user | grep Job_${config_UserChoices_JobName} | gawk ' { print $1}' )
1715
1716
1717  eval ${3}=${ID}
1718  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1719}
1720
1721#D-#==================================================
1722#D-function IGCM_sys_CountJobInQueue
1723#D-* Purpose: Count number of users job
1724#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1725#D-
1726function IGCM_sys_CountJobInQueue {
1727  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1728  if ( $DEBUG_sys ) ; then
1729    echo "IGCM_sys_CountJobInQueue"
1730  fi
1731
1732  # With -f option, the full job name is given in the last column
1733  NbRun=$(squeue -u $user | \
1734      grep -v JOBID | gawk 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1735
1736  eval ${2}=${NbRun}
1737
1738  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1739}
1740
1741#D-#==================================================
1742#D-function IGCM_sys_ListJobInQueue
1743#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1744#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1745#D-
1746function IGCM_sys_ListJobInQueue {
1747  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1748  if ( $DEBUG_sys ) ; then
1749    echo "IGCM_sys_ListJobInQueue"
1750  fi
1751
1752# to be implemented on Jean Zay
1753 set -A JobList $( squeue -u $user | gawk '{print $3}' | grep -v NAME)
1754#  # With -f option, the full job name is given in the last column
1755#  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1756#                                        '( $2  == User      && \
1757#                                           $NF != /TS/      && \
1758#                                           $NF !~ /PACK/    && \
1759#                                           $NF !~ /REBUILD/ && \
1760#                                           $NF !~ /pack/ )     \
1761#                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1762#
1763  eval set -A ${2} ${JobList[*]}
1764
1765  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1766}
1767
1768#D-#==================================================
1769#D-function IGCM_sys_atlas
1770#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1771#D-* Examples:
1772#D-
1773function IGCM_sys_atlas {
1774  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1775  if ( $DEBUG_sys ) ; then
1776    echo "IGCM_sys_atlas :" $@
1777  fi
1778
1779  typeset status
1780# To be implemented on Jean Zay
1781#
1782#  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1783#  status=$?
1784#  if [ ${status} -gt 0 ] ; then
1785#    echo "IGCM_sys_atlas : error code ${status}"
1786#    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1787#    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1788#    IGCM_debug_PopStack "IGCM_sys_atlas"
1789#    return 1
1790#  else
1791#    IGCM_debug_PopStack "IGCM_sys_atlas"
1792#    return 0
1793#  fi
1794#
1795  IGCM_debug_PopStack "IGCM_sys_atlas"
1796}
1797
1798#D-#==================================================
1799#D-function IGCM_sys_rebuild_nemo
1800#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1801#D-* Examples:
1802#D-
1803
1804function IGCM_sys_rebuild_nemo {
1805  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1806  if ( $DEBUG_sys ) ; then
1807    echo "IGCM_sys_rebuild_nemo :" $@
1808  fi
1809
1810  /gpfswork/rech/psl/commun/Tools/rebuild_nemo/bin/rebuild_nemo ${1} ${2}
1811
1812  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1813}
1814
Note: See TracBrowser for help on using the repository browser.