source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_jeanzay.ksh @ 1548

Last change on this file since 1548 was 1548, checked in by rpennel, 3 years ago

jean-zay hack to deal with POST jobs submission
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 59.6 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Jean-Zay
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host user names project maxCpuTime
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=jeanzay
71# add default project set by ins_job
72typeset PROJECT=$( echo ${SLURM_JOB_ACCOUNT:=::default_project::} | cut -c "1-3" ) 
73### jobWarningDelay in seconds
74##typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75 
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=sbatch}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Access to module command
93#====================================================
94# See ticket #348
95# Following path should be used according to assist IDRIS
96. /gpfslocalsup/spack_soft/environment-modules/current/init/ksh
97# Following old path gives error messages while loading modules but it still works
98#. /usr/share/Modules/init/ksh
99
100#====================================================
101# Set environment tools (ferret, nco, cdo, rebuild, ...)
102#====================================================
103if [ X${TaskType} = Xcomputing ] ; then
104  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
105else
106  module purge > /dev/null 2>&1
107  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay > /dev/null 2>&1
108
109  . /gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_atlas_jeanzay > /dev/null 2>&1
110
111#  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
112  export UVCDAT_ANONYMOUS_LOG=FALSE
113fi
114
115# FYI
116[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
117[ ! X${TaskType} = Xchecking ] && module list
118
119# For AddNoise (not yet AddPertu3DOCE)
120export PATH=${PATH}:/gpfswork/rech/psl/commun/Tools/AddNoise/bin
121
122# For rebuild
123export PATH=${PATH}:/gpfswork/rech/psl/commun/Tools/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
124
125#====================================================
126# Host specific DIRECTORIES
127#====================================================
128
129#====================================================
130#- MirrorlibIGCM for frontend
131typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
132
133#====================================================
134#- libIGCM_POST for frontend
135typeset -r libIGCM_POST=${libIGCM}
136
137#====================================================
138#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
139typeset -r R_EXE="${MODIPSL}/bin"
140
141#====================================================
142#- SUBMIT_DIR : submission dir
143typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${SLURM_SUBMIT_DIR}}
144
145#====================================================
146#- IN
147typeset -r R_IN=${R_IN:=/gpfswork/rech/psl/commun/IGCM}
148
149#====================================================
150#- RUN_DIR_PATH : Temporary working directory (=> TMP)
151typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCH}/RUN_DIR/${SLURM_JOBID}_${$}}
152
153#====================================================
154#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
155typeset -r OUTCOMMAND_PATH=/tmp
156
157#====================================================
158#- HOST_MPIRUN_COMMAND
159typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time srun "}
160
161#====================================================
162#- Max number of arguments passed to nco operator or demigration command
163UNIX_MAX_LIMIT=360
164
165#====================================================
166#- set PackDefault to true
167PackDefault=true
168
169#====================================================
170#- Default number of MPI task for IPSL coupled model
171#- required for backward compatibility
172#-
173DEFAULT_NUM_PROC_OCE=5
174DEFAULT_NUM_PROC_CPL=1
175(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
176
177#====================================================
178#- Number of computing cores per node
179#-
180NB_CORE_PER_NODE=40
181
182#D-#==================================================
183#D-function IGCM_sys_defineArchives
184#D-* Purpose:
185#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
186#D-* Define ARCHIVE : Dedicated to large files
187#D-* Define STORAGE : Dedicated to small/medium files
188#D-* Define R_OUT   : Output tree located on ARCHIVE
189#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monsitoring and atlas, and/or small files)
190#D-* Define R_BUF   : Output tree located on SCRATCH hosting files waiting for rebuild or pack processes
191#D-* if SpaceName=TEST everything is stored on SCRATCH
192#D-* Examples:
193#D-
194function IGCM_sys_defineArchives {
195  IGCM_debug_PushStack "IGCM_sys_defineArchives"
196
197  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
198  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = DEFAULT ]; then
199    # The variable DataProject is not set in in section UserChoices in config.card or it is set to DEFAULT
200    # Default option: Set DataProject according to the project used for submission
201    DataProject=${PROJECT}
202  else
203    # Use the project set by the variable DataProject in section UserChoices in config.card
204    IGCM_debug_Print 1 "Use DataProject as set in config.card : ${config_UserChoices_DataProject}"
205    DataProject=${config_UserChoices_DataProject}
206  fi
207  IGCM_debug_Print 1 "Data project for output to be used: ${DataProject}"
208
209  #====================================================
210  #- RUN_DIR_PATH : Temporary working directory (=> TMP)
211  #- DataProject = for is the dataproject for Training account at Idris
212  #- they are not store in rech but in another partition
213  if [ X${DataProject} = Xfor  ]; then
214      typeset -r RUN_DIR_PATH=$SCRATCH/RUN_DIR/${SLURM_JOBID}_${$}
215  else
216      typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/gpfsscratch/rech/${DataProject}/${LOGIN}/RUN_DIR/${SLURM_JOBID}_${$}}
217  fi
218
219  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
220    #====================================================
221    #- ARCHIVE (dedicated to large files)
222    ARCHIVE=${config_UserChoices_ARCHIVE}
223  else
224      if [ X${DataProject} = Xfor  ]; then
225          ARCHIVE=$STORE
226      else
227          #====================================================
228          #- ARCHIVE (dedicated to large files)
229          ARCHIVE=/gpfsstore/rech/${DataProject}/${LOGIN}
230      fi
231  fi
232
233  if [ ! X${config_UserChoices_STORAGE} = X ]; then
234    #====================================================
235    #- STORAGE (dedicated to small/medium files)
236    STORAGE=${config_UserChoices_STORAGE}
237  else
238    #====================================================
239    #- STORAGE (dedicated to small/medium files)
240      if [ X${DataProject} = Xfor  ]; then
241          STORAGE=$WORK
242      else
243          STORAGE=/gpfswork/rech/${DataProject}/${LOGIN}
244      fi
245  fi
246
247  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
248    #====================================================
249    #- R_OUT
250    if [ X${DataProject} = Xfor  ]; then
251        R_OUT=$SCRATCH/IGCM_OUT
252    else
253        R_OUT=/gpfsscratch/rech/${DataProject}/${LOGIN}/IGCM_OUT
254    fi
255    #====================================================
256    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
257    R_FIG=${R_OUT}
258
259    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
260
261  else
262    #====================================================
263    #- R_OUT
264    R_OUT=${ARCHIVE}/IGCM_OUT
265
266    #====================================================
267    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
268    R_FIG=${STORAGE}/IGCM_OUT
269  fi
270
271  #====================================================
272  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
273  CMIP6_BUF=${STORAGE}/IGCM_OUT
274
275  #====================================================
276  #- R_BUF (ONLY FOR double copy an scratch)
277 
278  if [ X${DataProject} = Xfor  ]; then
279      R_BUF=$SCRATCH/IGCM_OUT
280  else
281      R_BUF=/gpfsscratch/rech/${DataProject}/${LOGIN}/IGCM_OUT
282  fi
283
284  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
285  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
286  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
287
288  IGCM_debug_PopStack "IGCM_sys_defineArchives"
289}
290
291#D-#==================================================
292#D-function IGCM_sys_RshArchive
293#D-* Purpose: Archive rsh command
294#D-* Examples:
295#D-
296function IGCM_sys_RshArchive {
297  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
298  /bin/ksh <<-EOF
299    ${@}
300EOF
301  status=$?
302  if [ ${status} -gt 0 ] ; then
303    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
304    IGCM_debug_Exit "IGCM_sys_RshArchive"
305  fi
306  IGCM_debug_PopStack "IGCM_sys_RshArchive"
307}
308
309#D-#==================================================
310#D-function IGCM_sys_RshArchive_NoError
311#D-* Purpose: Archive rsh command, without error
312#D-*          used only in monitoring.job
313#D-* Examples:
314#D-
315function IGCM_sys_RshArchive_NoError {
316  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
317  /bin/ksh <<-EOF
318    ${@} 2> /dev/null
319EOF
320  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
321}
322
323#D-#==================================================
324#D-function IGCM_sys_MkdirArchive
325#D-* Purpose: Mkdir on Archive
326#D-* Examples:
327#D-
328function IGCM_sys_MkdirArchive {
329  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
330  if ( $DEBUG_sys ) ; then
331    echo "IGCM_sys_MkdirArchive :" $@
332  fi
333  #- creation de repertoire sur le serveur fichier
334  if [ ! -d ${1} ]; then
335    \mkdir -p $1
336    status=$?
337
338    if [ ${status} -gt 0 ] ; then
339      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
340      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
341    fi
342  fi
343  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
344}
345
346#D-#==================================================
347#D-function IGCM_sys_TestDirArchive
348#D-* Purpose: Test Directory that must exists on Archive
349#D-* Examples:
350#D-
351function IGCM_sys_TestDirArchive {
352  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
353  if ( $DEBUG_sys ) ; then
354    echo "IGCM_sys_TestDirArchive :" $@
355  fi
356  typeset ExistFlag
357  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
358  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
359  return ${ExistFlag}
360}
361
362#D-#==================================================
363#D-function IGCM_sys_IsFileArchived
364#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
365#D-* Examples:
366#D-
367function IGCM_sys_IsFileArchived {
368  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
369  if ( $DEBUG_sys ) ; then
370    echo "IGCM_sys_IsFileArchived :" $@
371  fi
372  typeset IsArchivedFlag
373# To be implemented on Jean Zay
374#  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
375  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
376
377  return ${IsArchivedFlag}
378}
379
380#D-#==================================================
381#D-function IGCM_sys_TestFileArchive
382#D-* Purpose: Test file that must NOT EXISTS on Archive
383#D-* Examples:
384#D-
385function IGCM_sys_TestFileArchive {
386  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
387  typeset ExistFlag
388  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
389  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
390
391  return ${ExistFlag}
392}
393
394#D-#==================================================
395#D-function IGCM_sys_CountFileArchive
396#D-* Purpose: Count files on Archive filesystem
397#D-* Examples:
398#D-
399function IGCM_sys_CountFileArchive {
400  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
401  ls ${@} 2>/dev/null | wc -l
402  if [ $? -gt 0 ] ; then
403    echo "IGCM_sys_CountFileArchive : erreur."
404  fi
405  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
406}
407
408#D-#==================================================
409#D-function IGCM_sys_Tree
410#D-* Purpose: Tree directories with files on ${ARCHIVE}
411#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
412#D-
413function IGCM_sys_Tree {
414  IGCM_debug_PushStack "IGCM_sys_Tree" $@
415  if ( $DEBUG_sys ) ; then
416    echo "IGCM_sys_Tree :" $@
417  fi
418
419  \ls -lR ${@}
420
421  IGCM_debug_PopStack "IGCM_sys_Tree"
422}
423
424#D-#==================================================
425#D-function IGCM_sys_Qsub
426#D-* Purpose: Qsub new job
427#D-* Examples:
428#D-
429function IGCM_sys_Qsub {
430  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
431  if ( $DEBUG_sys ) ; then
432    echo "IGCM_sys_Qsub :" $@
433  fi
434  typeset options status
435  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
436
437  /usr/bin/time sbatch ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
438  status=$?
439
440  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
441  if [ ${status} -gt 0 ] ; then
442    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
443    IGCM_debug_Exit "IGCM_sys_Qsub"
444  else
445    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
446  fi
447  IGCM_debug_PopStack "IGCM_sys_Qsub"
448}
449
450#D-#==================================================
451#D-function IGCM_sys_QsubPost
452#D-* Purpose: Qsub new job on scalaire
453#D-* Examples:
454#D-
455function IGCM_sys_QsubPost {
456  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
457
458  BAK_DIR=$PWD 
459  cd ${SUBMIT_DIR}
460
461  if ( $DEBUG_sys ) ; then
462    echo "IGCM_sys_QsubPost :" $@
463  fi
464  typeset options status
465  options="-o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
466
467  sbatch ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
468  status=$?
469
470  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
471  if [ ${status} -gt 0 ] ; then
472    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
473    IGCM_debug_Exit "IGCM_sys_QsubPost"
474  else
475    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
476  fi
477
478  cd ${BAK_DIR}
479
480  IGCM_debug_PopStack "IGCM_sys_QsubPost"
481}
482
483#D-*************************
484#D- File transfer functions
485#D-*************************
486#D-
487
488#D-#==================================================
489#D-function IGCM_sys_RmRunDir
490#D-* Purpose: rm tmpdir (dummy function most of the time batch
491#D-                      scheduler will do the job)
492#D-* Examples:
493#D-
494function IGCM_sys_RmRunDir {
495  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
496  if ( $DEBUG_sys ) ; then
497    echo "IGCM_sys_RmRunDir :" $@
498  fi
499
500  typeset status
501
502  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
503  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
504  status=$?
505
506  if [ ${status} -gt 0 ] ; then
507    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
508    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
509    IGCM_debug_Exit "IGCM_sys_RmRunDir"
510  else
511    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
512  fi
513  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
514}
515
516#D-#==================================================
517#D-function IGCM_sys_Put_Dir
518#D-* Purpose: Copy a complete directory on $(ARCHIVE)
519#D-* Examples:
520#D-
521function IGCM_sys_Put_Dir {
522  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
523  if ( $DEBUG_sys ) ; then
524    echo "IGCM_sys_Put_Dir :" $@
525  fi
526  if [ $DRYRUN = 0 ]; then
527    if [ ! -d ${1} ] ; then
528      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
529      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
530      return
531    fi
532
533    typeset status
534
535    # Only if we use rsync
536    #IGCM_sys_TestDirArchive $( dirname $2 )
537    #
538    #USUAL WAY
539    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
540    status=$?
541
542    if [ ${status} -gt 0 ] ; then
543      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
544      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
545      IGCM_debug_Exit "IGCM_sys_Put_Dir"
546    else
547      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
548    fi
549  fi
550  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
551}
552
553#D-#==================================================
554#D-function IGCM_sys_Get_Dir
555#D-* Purpose: Copy a complete directory from ${ARCHIVE}
556#D-* Examples:
557#D-
558function IGCM_sys_Get_Dir {
559  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
560  if ( $DEBUG_sys ) ; then
561    echo "IGCM_sys_Get_Dir :" $@
562  fi
563  if [ $DRYRUN = 0 ]; then
564    typeset NB_ESSAI DELAI status i
565    # number of tentative
566    NB_ESSAI=3
567    # time delay between tentative
568    DELAI=2
569
570    #
571    # USUAL WAY
572    # add here command to demigrate all offline files to reduce time of this command
573    # actually on Jean Zay files are not archive on bands
574
575    i=0
576    while [ $i -lt $NB_ESSAI ] ; do
577      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
578      status=$?
579      if [ ${status} -gt 0 ] ; then
580        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
581        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
582        sleep $DELAI
583      else
584        break
585      fi
586      (( i = i + 1 ))
587    done
588
589    if [ ${status} -gt 0 ] ; then
590      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
591      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
592      IGCM_debug_Exit "IGCM_sys_Get_Dir"
593    else
594      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
595    fi
596  fi
597  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
598}
599
600#D-#==================================================
601#D-function IGCM_sys_Put_Rest
602#D-* Purpose: Put computied restarts on ${ARCHIVE}.
603#D-           File and target directory must exist.
604#D-* Examples:
605#D-
606function IGCM_sys_Put_Rest {
607  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
608  if ( $DEBUG_sys ) ; then
609    echo "IGCM_sys_Put_Rest :" $@
610  fi
611  if [ $DRYRUN = 0 ]; then
612    if [ ! -f ${1} ] ; then
613      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
614      IGCM_debug_Exit "IGCM_sys_Put_Rest"
615    fi
616
617    typeset status
618    #
619    # USUAL WAY
620    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
621    status=$?
622
623#       #RSYNC WITH NETWORK SSH CALL
624#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
625#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
626
627#       #RSYNC WITH NFS USE
628#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
629#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
630
631#       status=$?
632#       IGCM_sys_Rsync_out $status
633
634#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
635#       (( status=status+$? ))
636
637    if [ ${status} -gt 0 ] ; then
638      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
639      [ -f ${1} ] && ls -l ${1}
640      [ -f ${2} ] && ls -l ${2}
641      [ -f ${2}/${1} ] && ls -l ${2}/${1}
642      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
643      IGCM_debug_Exit "IGCM_sys_Put_Rest"
644    else
645
646      if [ X${JobType} = XRUN ] ; then
647        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
648        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
649      fi
650
651      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
652    fi
653  fi
654  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
655}
656
657#D-#==================================================
658#D-function IGCM_sys_Put_Out
659#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
660#D-* Examples:
661#D-
662function IGCM_sys_Put_Out {
663  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
664  if ( $DEBUG_sys ) ; then
665    echo "IGCM_sys_Put_Out :" $@
666  fi
667
668  typeset NB_ESSAI DELAI status i exist skip
669  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
670
671  # number of tentative
672  NB_ESSAI=3
673  # time delay between tentative
674  DELAI=2
675
676  if [ $DRYRUN = 0 ]; then
677    if [ ! -f ${1} ] ; then
678      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
679      IGCM_debug_PopStack "IGCM_sys_Put_Out"
680      return 1
681    fi
682    #
683    IGCM_sys_MkdirArchive $( dirname $2 )
684    #
685    exist=false
686    skip=false
687    if [ -f $2 ] ; then
688      IGCM_debug_Print 1 "$2 already exist"
689    # add here command to demigrate all offline files to reduce time of this command
690    # actually on Jean Zay files are not archive on bands
691
692      exist=true
693      if [ "X$( diff $1 $2 )" = X ] ; then
694        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
695        skip=true
696      else
697        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
698        skip=false
699      fi
700    fi
701    #
702    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
703      IGCM_sys_Chmod u+w $2
704    fi
705
706    if [ X${skip} = Xfalse ] ; then
707      i=0
708      while [ $i -lt $NB_ESSAI ] ; do
709        # Identify file system
710        fileDeviceNumberInHex=$( stat -c %d $1 )
711        status=$?
712        if [ ${status} -gt 0 ] ; then
713          IGCM_debug_Exit "IGCM_sys_Put_Out"
714        fi
715        # Identify file system
716        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
717        status=$?
718        if [ ${status} -gt 0 ] ; then
719          IGCM_debug_Exit "IGCM_sys_Put_Out"
720        fi
721
722        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
723          # They are not on the same device. USUAL WAY
724          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
725          status=$?
726        else
727          # They are on the same device. NOT SO USUAL WAY
728          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
729          status=$?
730        fi
731        if [ ${status} -gt 0 ]; then
732          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
733          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
734          [ -f ${1} ] && ls -l ${1}
735          [ -f ${2} ] && ls -l ${2}
736          [ -f ${2}/${1} ] && ls -l ${2}/${1}
737          sleep $DELAI
738        else
739          break
740        fi
741        (( i = i + 1 ))
742      done
743    fi
744
745#       #RSYNC WITH NETWORK SSH CALL
746#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
747#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
748
749#       #RSYNC WITH NFS USE
750#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
751#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
752
753#       status=$?
754#       IGCM_sys_Rsync_out $status
755
756#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
757#       (( status=status+$? ))
758
759    if [ ${status} -gt 0 ] ; then
760      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
761      [ -f ${1} ] && ls -l ${1}
762      [ -f ${2} ] && ls -l ${2}
763      [ -f ${2}/${1} ] && ls -l ${2}/${1}
764      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
765      IGCM_debug_Exit "IGCM_sys_Put_Out"
766    else
767
768      if [ X${JobType} = XRUN ] ; then
769        if [ X${3} = X ] ; then
770          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
771          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
772        fi
773      fi
774
775      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
776    fi
777  fi
778  IGCM_debug_PopStack "IGCM_sys_Put_Out"
779  return 0
780}
781
782#D-#==================================================
783#D-function IGCM_sys_Get
784#D-* Purpose: Get a file from ${ARCHIVE}
785#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
786#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
787function IGCM_sys_Get {
788  IGCM_debug_PushStack "IGCM_sys_Get" $@
789
790  typeset DEST dm_liste target file_work
791  typeset NB_ESSAI DELAI status i
792
793  if ( $DEBUG_sys ) ; then
794    echo "IGCM_sys_Get :" $@
795  fi
796
797  # number of tentative
798  NB_ESSAI=3
799  # time delay between tentative
800  DELAI=2
801
802  if [ $DRYRUN -le 2 ]; then
803    if [ X${1} = X'/l' ] ; then
804      eval set +A dm_liste \${${2}}
805    else
806      eval set +A dm_liste ${1}
807    fi
808    eval DEST=\${${#}}
809    # add here command to demigrate all offline files to reduce time of this command
810    # actually on Jean Zay files are not archive on bands
811
812    status=$?
813    if [ ${status} -gt 0 ] ; then
814      echo "WARNING IGCM_sys_Get : error code ${status}"
815      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
816      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
817    fi
818
819    #if [ ${status} -gt 0 ] ; then
820    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
821    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
822    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
823    #    sleep 30
824    #    echo "We try another time"
825    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
826    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
827    #    status=$?
828    #    if [ ${status} -gt 0 ] ; then
829    #      echo "ERROR IGCM_sys_Get : again demigration error :"
830    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
831    #      IGCM_debug_Exit "IGCM_sys_Get"
832    #    fi
833    #  else
834    #    echo "ERROR IGCM_sys_Get : demigration error :"
835    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
836    #    IGCM_debug_Exit "IGCM_sys_Get"
837    #  fi
838    #fi
839
840    #   #RSYNC WITH NETWORK SSH CALL
841    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
842    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
843
844    #   #RSYNC WITH NFS USE
845    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
846    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
847
848    #   status=$?
849    #   IGCM_sys_Rsync_out $status
850
851    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
852    #   (( status=status+$? ))
853
854    #USUAL WAY
855    if [ X${1} = X'/l' ] ; then
856      for target in ${dm_liste[*]} ; do
857        local_file=$( basename ${target} )
858        # test if the target file is present before the loop
859        IGCM_sys_TestFileArchive ${target}
860        status=$?
861        if [ ${status} -gt 0 ] ; then
862          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
863          IGCM_debug_Exit "IGCM_sys_Get"
864        else
865          i=0
866          while [ $i -lt $NB_ESSAI ] ; do
867            #if [ X${DoLink} = Xtrue ] ; then
868            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
869            #  status=$?
870            #  else
871            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
872            #  status=$?
873            #fi
874            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
875            status=$?
876            if [ ${status} -gt 0 ]; then
877              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
878              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
879              sleep $DELAI
880            else
881              break
882            fi
883            (( i = i + 1 ))
884          done
885          if [ ${status} -gt 0 ] ; then
886            echo "IGCM_sys_Get : error"
887            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
888            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
889            IGCM_debug_Exit "IGCM_sys_Get"
890          else
891            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
892          fi
893        fi
894      done
895    else
896      i=0
897      while [ $i -lt $NB_ESSAI ] ; do
898        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
899        status=$?
900        if [ ${status} -gt 0 ]; then
901          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
902          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
903          sleep $DELAI
904        else
905          break
906        fi
907        (( i = i + 1 ))
908      done
909      if [ ${status} -gt 0 ] ; then
910        echo "IGCM_sys_Get : error"
911        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
912        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
913        IGCM_debug_Exit "IGCM_sys_Get"
914      else
915        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
916      fi
917    fi
918  fi
919  IGCM_debug_PopStack "IGCM_sys_Get"
920}
921
922#D-#==================================================
923#D-function IGCM_sys_GetDate_Monitoring
924#D-* Purpose: get the last year for which the monitoring has been computed
925#D-* Examples:
926#D-
927function IGCM_sys_GetDate_Monitoring {
928  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
929  if ( $DEBUG_sys ) ; then
930    echo "IGCM_sys_GetDate_Monitoring :" $@
931  fi
932
933  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
934
935  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
936}
937
938#D-#==================================================
939#D-function IGCM_sys_Dods_Rm
940#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
941#D-* Examples:
942#D-
943function IGCM_sys_Dods_Rm {
944  if ( $DEBUG_sys ) ; then
945    echo "IGCM_sys_Dods_Rm :" $@
946  fi
947  typeset status
948  if [ $DRYRUN = 0 ]; then
949
950#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
951#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
952#      echo "Nothing has been done."
953#      return
954#    fi
955
956    if [ "$#" -eq 1 ]; then
957      thredds_rm ${R_DODS}/${1}
958      status=$?
959    else
960        thredds_rm ${R_DODS}
961      status=$?
962    fi
963
964#    if [ ${status} -gt 0 ] ; then
965#      echo "IGCM_sys_Dods_Rm : error."
966#      cat out_dods_rm
967#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
968#    else
969#      rm out_dods_rm
970#    fi
971
972  fi
973  return $status
974}
975
976#D-#==================================================
977#D-function IGCM_sys_Dods_Cp
978#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
979#D-* Examples:
980#D-
981function IGCM_sys_Dods_Cp {
982  if ( $DEBUG_sys ) ; then
983    echo "IGCM_sys_Dods_Cp :" $@
984  fi
985  typeset status
986  if [ $DRYRUN = 0 ]; then
987
988#    if [ ! -d ${R_SAVE}/${1} ] ; then
989#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
990#      echo "Nothing has been done."
991#      return
992#    fi
993
994
995      thredds_cp ${1} ./${R_DODS}/.
996
997    status=$?
998
999#       if [ ${status} -gt 0 ] ; then
1000#           echo "IGCM_sys_Dods_Cp : error."
1001#           cat out_dods_cp
1002#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1003#       else
1004#           rm out_dods_cp
1005#       fi
1006
1007  fi
1008  return $status
1009}
1010
1011#D-#==================================================
1012#D-function IGCM_sys_Put_Dods
1013#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1014#D-* Examples:
1015#D-
1016function IGCM_sys_Put_Dods {
1017  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1018  if ( $DEBUG_sys ) ; then
1019    echo "IGCM_sys_Put_Dods :" $@
1020  fi
1021  typeset status
1022  if [ $DRYRUN = 0 ]; then
1023    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1024      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1025      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1026      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1027      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1028      return
1029    fi
1030
1031    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1032      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1033      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1034      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1035      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1036      return
1037    fi
1038    #
1039    if [ -d ${R_SAVE}/${1} ] ; then
1040      cd ${R_SAVE}
1041    elif [ -d ${R_FIGR}/${1} ] ; then
1042      cd ${R_FIGR}
1043    fi
1044
1045    IGCM_sys_Dods_Rm ${1}
1046    IGCM_sys_Dods_Cp ${1}
1047    status=0
1048
1049    if [ ${status} -gt 0 ] ; then
1050      echo "IGCM_sys_Put_Dods : error."
1051      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1052    fi
1053  fi
1054  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1055}
1056
1057##############################################################
1058# REBUILD OPERATOR
1059
1060#D-#==================================================
1061#D-function IGCM_sys_sync
1062#D-* Purpose: flush buffer on disk
1063#D-* Examples:
1064#D-
1065function IGCM_sys_sync {
1066  IGCM_debug_PushStack "IGCM_sys_sync" $@
1067  if ( $DEBUG_sys ) ; then
1068    echo "IGCM_sys_sync :" $@
1069  fi
1070
1071  /bin/sync
1072
1073  IGCM_debug_PopStack "IGCM_sys_sync"
1074}
1075
1076############################################################
1077# Activate Running Environnment Variables
1078
1079#D-#==================================================
1080#D-function IGCM_sys_activ_variables
1081#D-* Purpose: set environement variables prior to execution
1082#D-* Examples:
1083#D-
1084function IGCM_sys_activ_variables {
1085  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1086  if ( $DEBUG_sys ) ; then
1087    echo "IGCM_sys_activ_variables"
1088  fi
1089
1090# --------------------------------------------------------------------
1091#D- MODULE specifications
1092# --------------------------------------------------------------------
1093  # Source the file EnvFile containing all module load needed to run the model.
1094  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1095  # it will be used. Otherwise a default file will be used.
1096  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1097      # EnvFile is set in config.card and the file exists
1098      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1099      EnvFile=${config_UserChoices_EnvFile}
1100  else
1101      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1102      module purge > /dev/null 2>&1
1103      EnvFile=/gpfswork/rech/psl/commun/MachineEnvironment/jeanzay/env_jeanzay
1104  fi
1105  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1106  IGCM_debug_Print 1 ${EnvFile}
1107  . ${EnvFile}
1108
1109  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Jean-Zay. "
1110  module list
1111
1112# --------------------------------------------------------------------
1113#D- MPI specifications
1114# --------------------------------------------------------------------
1115
1116# --------------------------------------------------------------------
1117#D- Other specifications
1118# --------------------------------------------------------------------
1119
1120  ulimit -s unlimited
1121  if [ ${executionType} -eq 2 ] ; then
1122    export SLURM_HOSTFILE=./hostlist
1123  fi
1124
1125
1126  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1127}
1128
1129############################################################
1130# Desactivate Running Environnment Variables
1131
1132#D-#==================================================
1133#D-function IGCM_sys_desactiv_variables
1134#D-* Purpose: unset environement variables after execution
1135#D-* Examples:
1136#D-
1137function IGCM_sys_desactiv_variables {
1138  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1139  if ( $DEBUG_sys ) ; then
1140    echo "IGCM_sys_desactiv_variables"
1141  fi
1142# --------------------------------------------------------------------
1143#D- MPI specifications
1144# --------------------------------------------------------------------
1145  if [ ${executionType} -eq 2 ] ; then
1146    unset SLURM_HOSTFILE
1147  fi
1148# --------------------------------------------------------------------
1149#D- Other specifications
1150# --------------------------------------------------------------------
1151
1152  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1153}
1154
1155############################################################
1156# Update job headers to be used by the scheduler
1157
1158#D-#==================================================
1159#D-function IGCM_sys_updateHeaders
1160#D-* Purpose: Update job headers to be used by the scheduler
1161#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1162#D-
1163function IGCM_sys_updateHeaders {
1164  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1165  if ( $DEBUG_sys ) ; then
1166    echo "IGCM_sys_updateHeaders"
1167  fi
1168  typeset file
1169  file=$1
1170
1171  if [ ${executionType} -eq 1 ] ; then
1172    # MPMD + MPI
1173    sed -e "/::openMPthreads::/d"                  \
1174        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1175      ${file} > ${file}.tmp
1176
1177  elif [ ${executionType} -eq 2 ] ; then
1178    # MPMD + MPI + OMP
1179      (( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
1180      [ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
1181      sed -e "/::openMPthreads::/d"                 \
1182        -e "s/::JobNumProcTot::/${mpiTasks}/"       \
1183        -e "/ntasks/i\#SBATCH --nodes=${nodeNumber}"\
1184        -e "/ntasks/i\#SBATCH --exclusive"          \
1185         ${file} > ${file}.tmp
1186  elif [ ${executionType} -eq 3 ] ; then
1187    # SPMD + MPI/OMP
1188    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1189        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1190      ${file} > ${file}.tmp
1191
1192  elif [ ${executionType} -eq 4 ] ; then
1193    # SPMD + MPI only
1194    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1195        -e "/::openMPthreads::/d"                  \
1196      ${file} > ${file}.tmp
1197
1198  elif [ ${executionType} -eq 5 ] ; then
1199    # SPMD + OMP only
1200    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1201        -e "/::JobNumProcTot::/d"                  \
1202      ${file} > ${file}.tmp
1203
1204  elif [ ${executionType} -eq 6 ] ; then
1205    # SEQUENTIAL THEN
1206    sed -e "s/::JobNumProcTot::/1/"                \
1207        -e "/::openMPthreads::/d"                  \
1208      ${file} > ${file}.tmp
1209
1210  fi
1211
1212  IGCM_sys_Mv ${file}.tmp ${file}
1213
1214  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1215}
1216
1217############################################################
1218# Build MPI/OMP scripts run file (dummy function)
1219
1220#D-#==================================================
1221#D-function IGCM_sys_build_run_file
1222#D-* Purpose: build run file (deprecated)
1223#D-* Examples:
1224#D-
1225function IGCM_sys_build_run_file {
1226  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1227}
1228
1229############################################################
1230# Build MPI/OMP scripts
1231
1232#D-#==================================================
1233#D-function IGCM_sys_build_execution_scripts
1234#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1235#D-* Examples:
1236#D-
1237function IGCM_sys_build_execution_scripts
1238{
1239  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1240  if ( $DEBUG_sys ) ; then
1241    echo "IGCM_sys_build_execution_scripts " $@
1242  fi
1243  IGCM_debug_Print 1 "executionType= ${executionType}"
1244  EXECUTION=${HOST_MPIRUN_COMMAND}
1245
1246  if ( ${OK_PARA_MPMD} ) ; then
1247
1248    # MPMD mode
1249    # 1 MPI only : executionType=1
1250    # 2 MPI/OpenMP : executionType=2
1251
1252    if [ -f run_file ] ; then
1253      IGCM_sys_Rm -f run_file
1254    fi
1255    touch run_file
1256
1257    # case 1 : Only MPI (MPMD)
1258    if  ( ! ${OK_PARA_OMP} ) ; then
1259      # Build run_file
1260      current_core=0
1261      # First loop on the components for the coupler ie oasis (only if oasis3)
1262      # the coupler ie oasis3 must be the first one
1263      for comp in ${config_ListOfComponents[*]} ; do
1264
1265        eval ExeNameIn=\${config_Executable_${comp}[0]}
1266        eval ExeNameOut=\${config_Executable_${comp}[1]}
1267
1268        # Only if the component has an executable
1269        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1270
1271          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1272          (( end_core = ${current_core} + ${comp_proc_mpi_loc} - 1 ))
1273          echo "${current_core}-${end_core} ./${ExeNameOut}" >> run_file
1274          (( current_core = ${end_core} + 1 ))
1275        fi
1276      done
1277
1278      EXECUTION="${HOST_MPIRUN_COMMAND} --multi-prog ./run_file"
1279
1280      IGCM_sys_Chmod u+x run_file
1281      if ( $DEBUG_sys ) ; then
1282        echo "run_file contains : "
1283        cat run_file
1284      fi
1285
1286    else 
1287
1288    # 2 MPI/OpenMP : executionType=2
1289
1290    # MPI-OpenMP (MPMD)
1291    # export SLURM_HOSTFILE=./hostlist
1292    # srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
1293    # example of  hostlist file :
1294    # r3i3n33
1295    # r3i3n33
1296    # ...
1297    # example of run_file :
1298    # 0-70 ./prog_lmdz.x.sh %o %t
1299    # 71-430 ./prog_opa.xx.sh %o %t
1300    # 431-431 ./prog_xios.x.sh %o %t
1301    # examples of prog_file :
1302    # prog_lmdz.x.sh :
1303    # (( init = 0 + $1 ))
1304    # (( index = init * 10 ))
1305    # (( slot = index % 40 ))
1306    # taskset -c $slot-$((slot + 10 - 1)) ./script_lmdz.x.ksh
1307    # that will become
1308    # taskset -c 0-9 ./script_lmdz.x.ksh
1309    # ...
1310    # with script_lmdz.x.ksh
1311    # export OMP_STACKSIZE=3g
1312    # export OMP_PLACES=cores
1313    # export OMP_NUM_THREADS=10
1314    # ./lmdz.x > out_lmdz.x.out.${SLURM_PROCID} 2>out_lmdz.x.err.${SLURM_PROCID}
1315
1316    #  Hosts treatment
1317    _bkIFS=$IFS;
1318        IFS=$'\n'; set -f
1319        listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
1320        IFS=$_bkIFS; set +f
1321        rm -f hostlist
1322
1323     # Loop on the components to build run_file and script_exec files
1324        rank=0
1325        current_core=0
1326        current_core_mpi=0
1327
1328      for comp in ${config_ListOfComponents[*]} ; do
1329
1330        eval ExeNameIn=\${config_Executable_${comp}[0]}
1331        eval ExeNameOut=\${config_Executable_${comp}[1]}
1332
1333        # Not possible if oasis has an executable (i.e old version of oasis3)
1334        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1335          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1336          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1337          IGCM_debug_Verif_Exit
1338        fi
1339
1340        # Only if we really have an executable for the component :
1341        if [ "X${ExeNameOut}" != X\"\" ] ; then
1342
1343          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1344          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1345          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1346     
1347
1348          # Build script files
1349
1350          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1351          echo ""  >> script_${ExeNameOut}.ksh
1352          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1353          # Check if the number of threads is correct
1354              case ${comp_proc_omp_loc} in
1355                  2|4|5|10|20)
1356                      IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1357                      ;;
1358                  *)
1359                      IGCM_debug_Exit "ERROR with OMP parameters !"
1360                      IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1361                      IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1362                      IGCM_debug_Verif_Exit
1363                      ;;
1364              esac
1365            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1366            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1367            echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}"  >> script_${ExeNameOut}.ksh
1368
1369          fi
1370
1371          # to have out/err per process on different files
1372          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1373
1374          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1375
1376        # Build run_file
1377        # Only if the component has an executable
1378        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1379
1380          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1381          (( end_core = ${current_core_mpi} + ${comp_proc_mpi_loc} - 1 ))
1382          echo "${current_core_mpi}-${end_core} ./prog_${ExeNameOut}.sh %o %t" >> run_file
1383          (( current_core_mpi = ${end_core} + 1 ))
1384        fi
1385
1386          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1387              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1388          else
1389              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1390          fi
1391
1392          # Build configuration file
1393 
1394              echo "#!/bin/sh" > prog_${ExeNameOut}.sh
1395          echo "(( init = $current_core + \$1 ))" >> prog_${ExeNameOut}.sh
1396          echo "(( index = init * $comp_proc_omp_loc ))" >> prog_${ExeNameOut}.sh
1397          echo "(( slot = index % 40 ))" >> prog_${ExeNameOut}.sh
1398              echo "echo ${ExeNameOut} taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1))" >> prog_${ExeNameOut}.sh
1399          echo "taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1)) ./script_${ExeNameOut}.ksh" >> prog_${ExeNameOut}.sh
1400
1401          IGCM_sys_Chmod u+x prog_${ExeNameOut}.sh
1402
1403        # Build hostlist file
1404
1405          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1406             (( index_host = current_core / NB_CORE_PER_NODE ))
1407             host_value=${listnodes[${index_host}]}
1408               echo "$host_value" >> hostlist
1409             (( current_core = current_core + offset_comp_proc_loc ))
1410          done
1411        fi
1412      done
1413
1414      ## variable added to stop after 60s instead of 600s by default.
1415      ## This is used when no error comes from executables and when something stopped an executable without notice.
1416      export SLURM_WAIT=60
1417
1418      EXECUTION="${HOST_MPIRUN_COMMAND} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
1419
1420      IGCM_sys_Chmod u+x run_file
1421      if ( $DEBUG_sys ) ; then
1422        echo "run_file contains : "
1423        cat run_file
1424      fi
1425
1426    fi # if ${OK_PARA_MPMD}
1427
1428  else
1429  # Only one executable (SPMD mode):  executionType=3, 4, 5 and 6
1430
1431    for comp in ${config_ListOfComponents[*]} ; do
1432
1433      # Only if we really have an executable for the component :
1434      eval ExeNameOut=\${config_Executable_${comp}[1]}
1435      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1436
1437        # Build script files
1438
1439        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1440        echo ""  >> script_${ExeNameOut}.ksh
1441        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1442
1443        if ( ${OK_PARA_OMP} ) ; then
1444            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1445            # Check if the number of threads is correct
1446            case ${comp_proc_omp_loc} in
1447                2|4|5|10|20)
1448                    IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1449                    ;;
1450                *)
1451                    IGCM_debug_Exit "ERROR with OMP parameters !"
1452                    IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1453                    IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1454                    IGCM_debug_Verif_Exit
1455                    ;;
1456            esac
1457            echo ""  >> script_${ExeNameOut}.ksh
1458            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1459            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1460            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1461        fi
1462
1463        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1464       
1465        # To have out/err per process on different files
1466        echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1467        EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1468
1469        IGCM_debug_Print 1 "sys Irene : script_${ExeNameOut}.ksh contains"
1470        cat script_${ExeNameOut}.ksh
1471
1472      fi
1473
1474    done
1475
1476  fi # ${OK_PARA_MPMD}
1477
1478  IGCM_debug_Print 1 "sys Irene : execution command is "
1479  IGCM_debug_Print 1 "$EXECUTION"
1480
1481  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1482}
1483
1484#D-#==================================================
1485#D-function IGCM_sys_check_path
1486#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1487#D-* do not point to an important use directory. Stop immediately in that case.
1488#D-* Examples:
1489#D-
1490function IGCM_sys_check_path {
1491  IGCM_debug_PushStack "IGCM_sys_check_path"
1492  if ( $DEBUG_sys ) ; then
1493    echo "IGCM_sys_check_path"
1494  fi
1495
1496  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORK} ] || [ X${RUN_DIR_PATH} = X${SCRATCH} ] || [ X${RUN_DIR_PATH} = X${STORE} ] ) ; then
1497    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1498    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1499    IGCM_debug_Exit "This will stop the job"
1500  elif ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X/gpfswork/rech/${DataProject}/${LOGIN} ] || [ X${RUN_DIR_PATH} = X/gpfsscratch/rech/${DataProject}/${LOGIN} ] || [ X${RUN_DIR_PATH} = X/gpfsstore/rech/${DataProject}/${LOGIN} ] ) ; then
1501    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1502    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1503    IGCM_debug_Exit "This will stop the job"
1504  fi
1505  IGCM_debug_PopStack "IGCM_sys_check_path"
1506}
1507
1508#D-#==================================================
1509#D-function IGCM_sys_check_quota
1510#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1511#D-* Examples:
1512#D-
1513function IGCM_sys_check_quota {
1514  IGCM_debug_PushStack "IGCM_sys_check_quota"
1515  echo "IGCM_sys_check_quota is desactivated on JeanZay because there is actually no way to check scratch quota on this computing center (2019.11.19)"
1516#  if ( $DEBUG_sys ) ; then
1517#    echo "IGCM_sys_check_quota"
1518#  fi
1519#  # Limit of quota (in %)
1520#  limit_quota=90
1521#
1522#  # Check of the volume
1523#  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1524#  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1525#
1526#  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1527#
1528#    unit_avail=${volume_avail: -1}
1529#    unit_quota=${volume_quota: -1}
1530#
1531#    if [ "${unit_quota}" = "*" ] ; then
1532#      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1533#      IGCM_debug_Print 1 "More than 100% of your quota is used"
1534#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1535#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1536#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1537#      IGCM_debug_Verif_Exit
1538#    fi
1539#
1540#    temp_avail=${volume_avail%%${volume_avail: -1}*}
1541#    temp_quota=${volume_quota%%${volume_quota: -1}*}
1542#
1543#    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1544#
1545#    # Convertion
1546#      if [ ${volume_avail: -1} = "T" ] ; then
1547#        (( temp_avail = temp_avail * 1000000000000 ))
1548#      elif [ ${volume_avail: -1} = "G" ] ; then
1549#        (( temp_avail = temp_avail * 1000000000 ))
1550#      elif [ ${volume_avail: -1} = "M" ] ; then
1551#        (( temp_avail = temp_avail * 1000000 ))
1552#      elif [ ${volume_avail: -1} = "k" ] ; then
1553#        (( temp_avail = temp_avail * 1000 ))
1554#      else
1555#        (( temp_avail = volume_avail ))
1556#      fi
1557#      if [ ${volume_quota: -1} = "T" ] ; then
1558#        (( temp_quota = temp_quota * 1000000000000 ))
1559#      elif [ ${volume_quota: -1} = "G" ] ; then
1560#        (( temp_quota = temp_quota * 1000000000 ))
1561#      elif [ ${volume_quota: -1} = "M" ] ; then
1562#        (( temp_quota = temp_quota * 1000000 ))
1563#      elif [ ${volume_quota: -1} = "k" ] ; then
1564#        (( temp_quota = temp_quota * 1000 ))
1565#      else
1566#        (( temp_quota = volume_quota ))
1567#      fi
1568#    fi
1569#
1570#    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1571##    echo "volume ratio is " $quota_volume
1572#
1573#    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1574#      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1575#      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1576#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1577#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1578#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1579#      IGCM_debug_Verif_Exit
1580#    fi
1581#
1582#  fi
1583#
1584## Check of the number of inodes
1585#
1586#  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1587#  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1588#
1589#  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1590#
1591#    unit_avail=${inode_avail: -1}
1592#    unit_quota=${inode_quota: -1}
1593#
1594#    if [ "${unit_quota}" = "*" ] ; then
1595#      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1596#      IGCM_debug_Print 1 "More than 100% of your quota is used"
1597#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1598#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1599#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1600#      IGCM_debug_Verif_Exit
1601#    fi
1602#
1603#    temp_avail=${inode_avail%%${inode_avail: -1}*}
1604#    temp_quota=${inode_quota%%${inode_quota: -1}*}
1605#
1606#    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1607#
1608#    # Convertion
1609#      if [ ${inode_avail: -1} = "T" ] ; then
1610#        (( temp_avail = temp_avail * 1000000000000 ))
1611#      elif [ ${inode_avail: -1} = "G" ] ; then
1612#        (( temp_avail = temp_avail * 1000000000 ))
1613#      elif [ ${inode_avail: -1} = "M" ] ; then
1614#        (( temp_avail = temp_avail * 1000000 ))
1615#      elif [ ${inode_avail: -1} = "k" ] ; then
1616#        (( temp_avail = temp_avail * 1000 ))
1617#      else
1618#        (( temp_avail = inode_avail ))
1619#      fi
1620#
1621#      if [ ${inode_quota: -1} = "T" ] ; then
1622#        (( temp_quota = temp_quota * 1000000000000 ))
1623#      elif [ ${inode_quota: -1} = "G" ] ; then
1624#        (( temp_quota = temp_quota * 1000000000 ))
1625#      elif [ ${inode_quota: -1} = "M" ] ; then
1626#        (( temp_quota = temp_quota * 1000000 ))
1627#      elif [ ${inode_quota: -1} = "k" ] ; then
1628#        (( temp_quota = temp_quota * 1000 ))
1629#      else
1630#        (( temp_quota = inode_quota ))
1631#      fi
1632#    fi
1633#    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1634##    echo "inode ratio is " $quota_inode
1635#
1636#    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1637#      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1638#      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1639#      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1640#      IGCM_debug_Print 1 "You must have more than 10% available to run"
1641#      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1642#      IGCM_debug_Verif_Exit
1643#    fi
1644#  fi
1645  IGCM_debug_PopStack "IGCM_sys_check_quota"
1646}
1647
1648#D-#==================================================
1649#D-function IGCM_sys_projectAccounting
1650#D-* Purpose: store project accounting information in a file
1651#D-* Examples:
1652#D-
1653function IGCM_sys_projectAccounting {
1654  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1655  if ( $DEBUG_sys ) ; then
1656    echo "IGCM_sys_check_quota"
1657  fi
1658#need to implemented on Jean Zay
1659  touch $1
1660#  ssh irene191 /usr/bin/ccc_myproject > $1
1661
1662  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1663}
1664
1665#D-#==================================================
1666#D-function IGCM_sys_getJobSchedulerID
1667#D-* Purpose: Get the job ID during execution
1668#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1669#D-
1670function IGCM_sys_getJobSchedulerID {
1671  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1672  if ( $DEBUG_sys ) ; then
1673    echo "IGCM_sys_getJobSchedulerID"
1674  fi
1675
1676  eval ${1}=${SLURM_JOBID}
1677
1678  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1679}
1680
1681#D-#==================================================
1682#D-function IGCM_sys_GetJobID
1683#D-* Purpose: Get the job ID from the JobName
1684#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1685#D-
1686function IGCM_sys_GetJobID {
1687  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1688  if ( $DEBUG_sys ) ; then
1689    echo "IGCM_sys_GetJobID"
1690  fi
1691
1692  ID=$( squeue -u $user | grep Job_${config_UserChoices_JobName} | gawk ' { print $1}' )
1693
1694
1695  eval ${3}=${ID}
1696  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1697}
1698
1699#D-#==================================================
1700#D-function IGCM_sys_CountJobInQueue
1701#D-* Purpose: Count number of users job
1702#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1703#D-
1704function IGCM_sys_CountJobInQueue {
1705  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1706  if ( $DEBUG_sys ) ; then
1707    echo "IGCM_sys_CountJobInQueue"
1708  fi
1709
1710  # With -f option, the full job name is given in the last column
1711  NbRun=$(squeue -u $user | \
1712      grep -v JOBID | gawk 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1713
1714  eval ${2}=${NbRun}
1715
1716  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1717}
1718
1719#D-#==================================================
1720#D-function IGCM_sys_ListJobInQueue
1721#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1722#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1723#D-
1724function IGCM_sys_ListJobInQueue {
1725  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1726  if ( $DEBUG_sys ) ; then
1727    echo "IGCM_sys_ListJobInQueue"
1728  fi
1729
1730# to be implemented on Jean Zay
1731 set -A JobList $( squeue -u $user | gawk '{print $3}' | grep -v NAME)
1732#  # With -f option, the full job name is given in the last column
1733#  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1734#                                        '( $2  == User      && \
1735#                                           $NF != /TS/      && \
1736#                                           $NF !~ /PACK/    && \
1737#                                           $NF !~ /REBUILD/ && \
1738#                                           $NF !~ /pack/ )     \
1739#                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1740#
1741  eval set -A ${2} ${JobList[*]}
1742
1743  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1744}
1745
1746#D-#==================================================
1747#D-function IGCM_sys_atlas
1748#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1749#D-* Examples:
1750#D-
1751function IGCM_sys_atlas {
1752  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1753  if ( $DEBUG_sys ) ; then
1754    echo "IGCM_sys_atlas :" $@
1755  fi
1756
1757  typeset status
1758# To be implemented on Jean Zay
1759#
1760#  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1761#  status=$?
1762#  if [ ${status} -gt 0 ] ; then
1763#    echo "IGCM_sys_atlas : error code ${status}"
1764#    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1765#    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1766#    IGCM_debug_PopStack "IGCM_sys_atlas"
1767#    return 1
1768#  else
1769#    IGCM_debug_PopStack "IGCM_sys_atlas"
1770#    return 0
1771#  fi
1772#
1773  IGCM_debug_PopStack "IGCM_sys_atlas"
1774}
1775
1776#D-#==================================================
1777#D-function IGCM_sys_rebuild_nemo
1778#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1779#D-* Examples:
1780#D-
1781
1782function IGCM_sys_rebuild_nemo {
1783  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1784  if ( $DEBUG_sys ) ; then
1785    echo "IGCM_sys_rebuild_nemo :" $@
1786  fi
1787
1788  /gpfswork/rech/psl/commun/Tools/rebuild_nemo/bin/rebuild_nemo ${1} ${2}
1789
1790  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1791}
1792
Note: See TracBrowser for help on using the repository browser.