source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_irene.ksh @ 1441

Last change on this file since 1441 was 1441, checked in by jgipsl, 6 years ago

Correction for irene. Now save the text output of post-processing jobs.

  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 61.8 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Curie
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host user names project maxCpuTime
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=irene
71# add default project on curie
72typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} | cut -d@ -f1 )
73# jobWarningDelay in seconds
74typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=ccc_msub}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Set environment tools (ferret, nco, cdo, rebuild, ...)
93#====================================================
94if [ X${TaskType} = Xcomputing ] ; then
95  IGCM_debug_Print 1 "Module will be loaded once we know the compiler version"
96else
97  module purge > /dev/null 2>&1
98  . /ccc/cont003/home/dsm/p86ipsl/.env_irene   > /dev/null 2>&1
99  . /ccc/cont003/home/dsm/p86ipsl/.env_atlas_irene   > /dev/null 2>&1
100
101#  . /ccc/cont003/home/dsm/p86ipsl/.env_intel17.0.2_curie   > /dev/null 2>&1
102#  . /ccc/cont003/home/dsm/p86ipsl/.env_netcdf4.3.3.1_curie > /dev/null 2>&1
103#  . /ccc/cont003/home/dsm/p86ipsl/.env_atlas_curie         > /dev/null 2>&1
104  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
105  export UVCDAT_ANONYMOUS_LOG=FALSE
106fi
107
108# Use CMIP6 storage space when using project id gch0316
109[[ "X${PROJECT}" = "Xgch0316" && ! $(module list --terse 2>&1 | grep dfldatadir/gencmip6) ]] && module switch dfldatadir dfldatadir/gencmip6
110
111# FYI
112[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
113[ ! X${TaskType} = Xchecking ] && module list
114
115export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
116export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
117export PATH=${PATH}:$( ccc_home -u p86ipsl )/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/
118
119#====================================================
120# Specific for ocean additionnal diagnostic
121export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
122export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
123
124#====================================================
125# Host specific DIRECTORIES
126#====================================================
127
128# ============ CESIUM START ============ #
129
130#====================================================
131#- Mirror libIGCM from titane to cesium if needed
132#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
133#if [ ! ${ROOTSYS} = "home" ] ; then
134#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
135#else
136#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
137#fi
138
139#====================================================
140#- libIGCM_POST
141#if ( ${MirrorlibIGCM} ) ; then
142#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
143#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
144#else
145#  typeset -r libIGCM_POST=${libIGCM}
146#fi
147
148# ============ CESIUM  END  ============ #
149
150#====================================================
151#- MirrorlibIGCM for frontend
152typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
153
154#====================================================
155#- libIGCM_POST for frontend
156typeset -r libIGCM_POST=${libIGCM}
157
158#====================================================
159#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
160typeset -r R_EXE="${MODIPSL}/bin"
161
162#====================================================
163#- SUBMIT_DIR : submission dir
164typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
165
166#====================================================
167#- IN
168typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
169typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
170
171#====================================================
172#- RUN_DIR_PATH : Temporary working directory (=> TMP)
173typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
174
175#====================================================
176#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
177typeset -r OUTCOMMAND_PATH=/tmp
178
179#====================================================
180#- HOST_MPIRUN_COMMAND
181typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
182
183#====================================================
184#- Max number of arguments passed to nco operator or demigration command
185UNIX_MAX_LIMIT=360
186
187#====================================================
188#- set PackDefault to true on curie
189PackDefault=true
190
191#====================================================
192#- Default number of MPI task for IPSL coupled model
193#- required for backward compatibility
194#-
195DEFAULT_NUM_PROC_OCE=5
196DEFAULT_NUM_PROC_CPL=1
197(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
198
199#====================================================
200#- Number of computing cores per node
201#-
202NB_CORE_PER_NODE=16
203
204#D-#==================================================
205#D-function IGCM_sys_defineArchives
206#D-* Purpose:
207#D-* Define ARCHIVE : Dedicated to large files
208#D-* Define STORAGE : Dedicated to small/medium files
209#D-* Define R_OUT   : Output tree located on ARCHIVE
210#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
211#D-* Define R_BUF   : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
212#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
213#D-* Examples:
214#D-
215function IGCM_sys_defineArchives {
216  IGCM_debug_PushStack "IGCM_sys_defineArchives"
217
218  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
219    #====================================================
220    #- ARCHIVE (dedicated to large files)
221    ARCHIVE=${config_UserChoices_ARCHIVE}
222  else
223    #====================================================
224    #- ARCHIVE (dedicated to large files)
225    ARCHIVE=${CCCSTOREDIR}
226  fi
227
228  if [ ! X${config_UserChoices_STORAGE} = X ]; then
229    #====================================================
230    #- STORAGE (dedicated to small/medium files)
231    STORAGE=${config_UserChoices_STORAGE}
232  else
233    #====================================================
234    #- STORAGE (dedicated to small/medium files)
235    STORAGE=${CCCWORKDIR}
236  fi
237
238  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
239    #====================================================
240    #- R_OUT
241    R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
242
243    #====================================================
244    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
245    R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
246
247    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
248
249  else
250    #====================================================
251    #- R_OUT
252    R_OUT=${ARCHIVE}/IGCM_OUT
253
254    #====================================================
255    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
256    R_FIG=${STORAGE}/IGCM_OUT
257  fi
258
259  #====================================================
260  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
261  CMIP6_BUF=${STORAGE}/IGCM_OUT
262
263  #====================================================
264  #- R_BUF (ONLY FOR double copy an scratch)
265  R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
266
267  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
268  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
269  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
270
271  IGCM_debug_PopStack "IGCM_sys_defineArchives"
272}
273
274#D-#==================================================
275#D-function IGCM_sys_RshArchive
276#D-* Purpose: Archive rsh command
277#D-* Examples:
278#D-
279function IGCM_sys_RshArchive {
280  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
281  /bin/ksh <<-EOF
282    ${@}
283EOF
284  status=$?
285  if [ ${status} -gt 0 ] ; then
286    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
287    IGCM_debug_Exit "IGCM_sys_RshArchive"
288  fi
289  IGCM_debug_PopStack "IGCM_sys_RshArchive"
290}
291
292#D-#==================================================
293#D-function IGCM_sys_RshArchive_NoError
294#D-* Purpose: Archive rsh command, without error
295#D-*          used only in monitoring.job
296#D-* Examples:
297#D-
298function IGCM_sys_RshArchive_NoError {
299  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
300  /bin/ksh <<-EOF
301    ${@} 2> /dev/null
302EOF
303  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
304}
305
306#D-#==================================================
307#D-function IGCM_sys_MkdirArchive
308#D-* Purpose: Mkdir on Archive
309#D-* Examples:
310#D-
311function IGCM_sys_MkdirArchive {
312  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
313  if ( $DEBUG_sys ) ; then
314    echo "IGCM_sys_MkdirArchive :" $@
315  fi
316  #- creation de repertoire sur le serveur fichier
317  if [ ! -d ${1} ]; then
318    \mkdir -p $1
319    status=$?
320
321    if [ ${status} -gt 0 ] ; then
322      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
323      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
324    fi
325  fi
326  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
327}
328
329#D-#==================================================
330#D-function IGCM_sys_TestDirArchive
331#D-* Purpose: Test Directory that must exists on Archive
332#D-* Examples:
333#D-
334function IGCM_sys_TestDirArchive {
335  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
336  if ( $DEBUG_sys ) ; then
337    echo "IGCM_sys_TestDirArchive :" $@
338  fi
339  typeset ExistFlag
340  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
341  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
342  return ${ExistFlag}
343}
344
345#D-#==================================================
346#D-function IGCM_sys_IsFileArchived
347#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
348#D-* Examples:
349#D-
350function IGCM_sys_IsFileArchived {
351  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
352  if ( $DEBUG_sys ) ; then
353    echo "IGCM_sys_IsFileArchived :" $@
354  fi
355  typeset IsArchivedFlag
356  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
357  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
358
359  return ${IsArchivedFlag}
360}
361
362#D-#==================================================
363#D-function IGCM_sys_TestFileArchive
364#D-* Purpose: Test file that must NOT EXISTS on Archive
365#D-* Examples:
366#D-
367function IGCM_sys_TestFileArchive {
368  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
369  typeset ExistFlag
370  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
371  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
372
373  return ${ExistFlag}
374}
375
376#D-#==================================================
377#D-function IGCM_sys_CountFileArchive
378#D-* Purpose: Count files on Archive filesystem
379#D-* Examples:
380#D-
381function IGCM_sys_CountFileArchive {
382  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
383  ls ${@} 2>/dev/null | wc -l
384  if [ $? -gt 0 ] ; then
385    echo "IGCM_sys_CountFileArchive : erreur."
386  fi
387  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
388}
389
390#D-#==================================================
391#D-function IGCM_sys_Tree
392#D-* Purpose: Tree directories with files on ${ARCHIVE}
393#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
394#D-
395function IGCM_sys_Tree {
396  IGCM_debug_PushStack "IGCM_sys_Tree" $@
397  if ( $DEBUG_sys ) ; then
398    echo "IGCM_sys_Tree :" $@
399  fi
400
401  \ls -lR ${@}
402
403  IGCM_debug_PopStack "IGCM_sys_Tree"
404}
405
406#D-#==================================================
407#D-function IGCM_sys_Qsub
408#D-* Purpose: Qsub new job
409#D-* Examples:
410#D-
411function IGCM_sys_Qsub {
412  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
413  if ( $DEBUG_sys ) ; then
414    echo "IGCM_sys_Qsub :" $@
415  fi
416  typeset options status
417  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
418
419  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
420  status=$?
421
422  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
423  if [ ${status} -gt 0 ] ; then
424    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
425    IGCM_debug_Exit "IGCM_sys_Qsub"
426  else
427    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
428  fi
429  IGCM_debug_PopStack "IGCM_sys_Qsub"
430}
431
432#D-#==================================================
433#D-function IGCM_sys_QsubPost
434#D-* Purpose: Qsub new job on scalaire
435#D-* Examples:
436#D-
437function IGCM_sys_QsubPost {
438  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
439  if ( $DEBUG_sys ) ; then
440    echo "IGCM_sys_QsubPost :" $@
441  fi
442  typeset options status
443  options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
444
445  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
446  status=$?
447
448  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
449  if [ ${status} -gt 0 ] ; then
450    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
451    IGCM_debug_Exit "IGCM_sys_QsubPost"
452  else
453    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
454  fi
455  IGCM_debug_PopStack "IGCM_sys_QsubPost"
456}
457
458#D-*************************
459#D- File transfer functions
460#D-*************************
461#D-
462
463#D-#==================================================
464#D-function IGCM_sys_RmRunDir
465#D-* Purpose: rm tmpdir (dummy function most of the time batch
466#D-                      scheduler will do the job)
467#D-* Examples:
468#D-
469function IGCM_sys_RmRunDir {
470  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
471  if ( $DEBUG_sys ) ; then
472    echo "IGCM_sys_RmRunDir :" $@
473  fi
474
475  typeset status
476
477  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
478  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
479  status=$?
480
481  if [ ${status} -gt 0 ] ; then
482    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
483    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
484    IGCM_debug_Exit "IGCM_sys_RmRunDir"
485  else
486    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
487  fi
488  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
489}
490
491#D-#==================================================
492#D-function IGCM_sys_Put_Dir
493#D-* Purpose: Copy a complete directory on $(ARCHIVE)
494#D-* Examples:
495#D-
496function IGCM_sys_Put_Dir {
497  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
498  if ( $DEBUG_sys ) ; then
499    echo "IGCM_sys_Put_Dir :" $@
500  fi
501  if [ $DRYRUN = 0 ]; then
502    if [ ! -d ${1} ] ; then
503      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
504      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
505      return
506    fi
507
508    typeset status
509
510    # Only if we use rsync
511    #IGCM_sys_TestDirArchive $( dirname $2 )
512    #
513    #USUAL WAY
514    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
515    status=$?
516
517    if [ ${status} -gt 0 ] ; then
518      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
519      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
520      IGCM_debug_Exit "IGCM_sys_Put_Dir"
521    else
522      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
523    fi
524  fi
525  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
526}
527
528#D-#==================================================
529#D-function IGCM_sys_Get_Dir
530#D-* Purpose: Copy a complete directory from ${ARCHIVE}
531#D-* Examples:
532#D-
533function IGCM_sys_Get_Dir {
534  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
535  if ( $DEBUG_sys ) ; then
536    echo "IGCM_sys_Get_Dir :" $@
537  fi
538  if [ $DRYRUN = 0 ]; then
539    typeset NB_ESSAI DELAI status i
540    # number of tentative
541    NB_ESSAI=3
542    # time delay between tentative
543    DELAI=2
544
545    #
546    # USUAL WAY
547    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
548    ccc_hsm get -r $1
549
550    i=0
551    while [ $i -lt $NB_ESSAI ] ; do
552      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
553      status=$?
554      if [ ${status} -gt 0 ] ; then
555        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
556        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
557        sleep $DELAI
558      else
559        break
560      fi
561      (( i = i + 1 ))
562    done
563
564    if [ ${status} -gt 0 ] ; then
565      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
566      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
567      IGCM_debug_Exit "IGCM_sys_Get_Dir"
568    else
569      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
570    fi
571  fi
572  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
573}
574
575#D-#==================================================
576#D-function IGCM_sys_Put_Rest
577#D-* Purpose: Put computied restarts on ${ARCHIVE}.
578#D-           File and target directory must exist.
579#D-* Examples:
580#D-
581function IGCM_sys_Put_Rest {
582  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
583  if ( $DEBUG_sys ) ; then
584    echo "IGCM_sys_Put_Rest :" $@
585  fi
586  if [ $DRYRUN = 0 ]; then
587    if [ ! -f ${1} ] ; then
588      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
589      IGCM_debug_Exit "IGCM_sys_Put_Rest"
590    fi
591
592    typeset status
593    #
594    # USUAL WAY
595    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
596    status=$?
597
598#       #RSYNC WITH NETWORK SSH CALL
599#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
600#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
601
602#       #RSYNC WITH NFS USE
603#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
604#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
605
606#       status=$?
607#       IGCM_sys_Rsync_out $status
608
609#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
610#       (( status=status+$? ))
611
612    if [ ${status} -gt 0 ] ; then
613      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
614      [ -f ${1} ] && ls -l ${1}
615      [ -f ${2} ] && ls -l ${2}
616      [ -f ${2}/${1} ] && ls -l ${2}/${1}
617      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
618      IGCM_debug_Exit "IGCM_sys_Put_Rest"
619    else
620
621      if [ X${JobType} = XRUN ] ; then
622        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
623        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
624      fi
625
626      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
627    fi
628  fi
629  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
630}
631
632#D-#==================================================
633#D-function IGCM_sys_Put_Out
634#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
635#D-* Examples:
636#D-
637function IGCM_sys_Put_Out {
638  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
639  if ( $DEBUG_sys ) ; then
640    echo "IGCM_sys_Put_Out :" $@
641  fi
642
643  typeset NB_ESSAI DELAI status i exist skip
644  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
645
646  # number of tentative
647  NB_ESSAI=3
648  # time delay between tentative
649  DELAI=2
650
651  if [ $DRYRUN = 0 ]; then
652    if [ ! -f ${1} ] ; then
653      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
654      IGCM_debug_PopStack "IGCM_sys_Put_Out"
655      return 1
656    fi
657    #
658    IGCM_sys_MkdirArchive $( dirname $2 )
659    #
660    exist=false
661    skip=false
662    if [ -f $2 ] ; then
663      IGCM_debug_Print 1 "$2 already exist"
664      ccc_hsm get $2
665      exist=true
666      if [ "X$( diff $1 $2 )" = X ] ; then
667        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
668        skip=true
669      else
670        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
671        skip=false
672      fi
673    fi
674    #
675    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
676      IGCM_sys_Chmod u+w $2
677    fi
678
679    if [ X${skip} = Xfalse ] ; then
680      i=0
681      while [ $i -lt $NB_ESSAI ] ; do
682        # Identify file system
683        fileDeviceNumberInHex=$( stat -c %d $1 )
684        status=$?
685        if [ ${status} -gt 0 ] ; then
686          IGCM_debug_Exit "IGCM_sys_Put_Out"
687        fi
688        # Identify file system
689        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
690        status=$?
691        if [ ${status} -gt 0 ] ; then
692          IGCM_debug_Exit "IGCM_sys_Put_Out"
693        fi
694
695        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
696          # They are not on the same device. USUAL WAY
697          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
698          status=$?
699        else
700          # They are on the same device. NOT SO USUAL WAY
701          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
702          status=$?
703        fi
704        if [ ${status} -gt 0 ]; then
705          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
706          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
707          [ -f ${1} ] && ls -l ${1}
708          [ -f ${2} ] && ls -l ${2}
709          [ -f ${2}/${1} ] && ls -l ${2}/${1}
710          sleep $DELAI
711        else
712          break
713        fi
714        (( i = i + 1 ))
715      done
716    fi
717
718#       #RSYNC WITH NETWORK SSH CALL
719#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
720#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
721
722#       #RSYNC WITH NFS USE
723#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
724#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
725
726#       status=$?
727#       IGCM_sys_Rsync_out $status
728
729#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
730#       (( status=status+$? ))
731
732    if [ ${status} -gt 0 ] ; then
733      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
734      [ -f ${1} ] && ls -l ${1}
735      [ -f ${2} ] && ls -l ${2}
736      [ -f ${2}/${1} ] && ls -l ${2}/${1}
737      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
738      IGCM_debug_Exit "IGCM_sys_Put_Out"
739    else
740
741      if [ X${JobType} = XRUN ] ; then
742        if [ X${3} = X ] ; then
743          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
744          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
745        fi
746      fi
747
748      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
749    fi
750  fi
751  IGCM_debug_PopStack "IGCM_sys_Put_Out"
752  return 0
753}
754
755#D-#==================================================
756#D-function IGCM_sys_Get
757#D-* Purpose: Get a file from ${ARCHIVE}
758#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
759#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
760function IGCM_sys_Get {
761  IGCM_debug_PushStack "IGCM_sys_Get" $@
762
763  typeset DEST dm_liste target file_work
764  typeset NB_ESSAI DELAI status i
765
766  if ( $DEBUG_sys ) ; then
767    echo "IGCM_sys_Get :" $@
768  fi
769
770  # number of tentative
771  NB_ESSAI=3
772  # time delay between tentative
773  DELAI=2
774
775  if [ $DRYRUN -le 2 ]; then
776    if [ X${1} = X'/l' ] ; then
777      eval set +A dm_liste \${${2}}
778    else
779      eval set +A dm_liste ${1}
780    fi
781    eval DEST=\${${#}}
782    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
783    status=$?
784    if [ ${status} -gt 0 ] ; then
785      echo "WARNING IGCM_sys_Get : error code ${status}"
786      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
787      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
788    fi
789
790    #if [ ${status} -gt 0 ] ; then
791    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
792    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
793    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
794    #    sleep 30
795    #    echo "We try another time"
796    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
797    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
798    #    status=$?
799    #    if [ ${status} -gt 0 ] ; then
800    #      echo "ERROR IGCM_sys_Get : again demigration error :"
801    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
802    #      IGCM_debug_Exit "IGCM_sys_Get"
803    #    fi
804    #  else
805    #    echo "ERROR IGCM_sys_Get : demigration error :"
806    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
807    #    IGCM_debug_Exit "IGCM_sys_Get"
808    #  fi
809    #fi
810
811    #   #RSYNC WITH NETWORK SSH CALL
812    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
813    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
814
815    #   #RSYNC WITH NFS USE
816    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
817    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
818
819    #   status=$?
820    #   IGCM_sys_Rsync_out $status
821
822    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
823    #   (( status=status+$? ))
824
825    #USUAL WAY
826    if [ X${1} = X'/l' ] ; then
827      for target in ${dm_liste[*]} ; do
828        local_file=$( basename ${target} )
829        # test if the target file is present before the loop
830        IGCM_sys_TestFileArchive ${target}
831        status=$?
832        if [ ${status} -gt 0 ] ; then
833          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
834          IGCM_debug_Exit "IGCM_sys_Get"
835        else
836          i=0
837          while [ $i -lt $NB_ESSAI ] ; do
838            #if [ X${DoLink} = Xtrue ] ; then
839            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
840            #  status=$?
841            #  else
842            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
843            #  status=$?
844            #fi
845            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
846            status=$?
847            if [ ${status} -gt 0 ]; then
848              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
849              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
850              sleep $DELAI
851            else
852              break
853            fi
854            (( i = i + 1 ))
855          done
856          if [ ${status} -gt 0 ] ; then
857            echo "IGCM_sys_Get : error"
858            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
859            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
860            IGCM_debug_Exit "IGCM_sys_Get"
861          else
862            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
863          fi
864        fi
865      done
866    else
867      i=0
868      while [ $i -lt $NB_ESSAI ] ; do
869        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
870        status=$?
871        if [ ${status} -gt 0 ]; then
872          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
873          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
874          sleep $DELAI
875        else
876          break
877        fi
878        (( i = i + 1 ))
879      done
880      if [ ${status} -gt 0 ] ; then
881        echo "IGCM_sys_Get : error"
882        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
883        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
884        IGCM_debug_Exit "IGCM_sys_Get"
885      else
886        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
887      fi
888    fi
889  fi
890  IGCM_debug_PopStack "IGCM_sys_Get"
891}
892
893#D-#==================================================
894#D-function IGCM_sys_GetDate_Monitoring
895#D-* Purpose: get the last year for which the monitoring has been computed
896#D-* Examples:
897#D-
898function IGCM_sys_GetDate_Monitoring {
899  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
900  if ( $DEBUG_sys ) ; then
901    echo "IGCM_sys_GetDate_Monitoring :" $@
902  fi
903
904  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
905
906  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
907}
908
909#D-#==================================================
910#D-function IGCM_sys_Dods_Rm
911#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
912#D-* Examples:
913#D-
914function IGCM_sys_Dods_Rm {
915  if ( $DEBUG_sys ) ; then
916    echo "IGCM_sys_Dods_Rm :" $@
917  fi
918  typeset status
919  if [ $DRYRUN = 0 ]; then
920
921#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
922#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
923#      echo "Nothing has been done."
924#      return
925#    fi
926
927    if [ "$#" -eq 1 ]; then
928      /ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
929      status=$?
930    else
931      /ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
932      status=$?
933    fi
934
935#    if [ ${status} -gt 0 ] ; then
936#      echo "IGCM_sys_Dods_Rm : error."
937#      cat out_dods_rm
938#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
939#    else
940#      rm out_dods_rm
941#    fi
942
943  fi
944  return $status
945}
946
947#D-#==================================================
948#D-function IGCM_sys_Dods_Cp
949#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
950#D-* Examples:
951#D-
952function IGCM_sys_Dods_Cp {
953  if ( $DEBUG_sys ) ; then
954    echo "IGCM_sys_Dods_Cp :" $@
955  fi
956  typeset status
957  if [ $DRYRUN = 0 ]; then
958
959#    if [ ! -d ${R_SAVE}/${1} ] ; then
960#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
961#      echo "Nothing has been done."
962#      return
963#    fi
964
965    /ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
966    status=$?
967
968#       if [ ${status} -gt 0 ] ; then
969#           echo "IGCM_sys_Dods_Cp : error."
970#           cat out_dods_cp
971#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
972#       else
973#           rm out_dods_cp
974#       fi
975
976  fi
977  return $status
978}
979
980#D-#==================================================
981#D-function IGCM_sys_Put_Dods
982#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
983#D-* Examples:
984#D-
985function IGCM_sys_Put_Dods {
986  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
987  if ( $DEBUG_sys ) ; then
988    echo "IGCM_sys_Put_Dods :" $@
989  fi
990  typeset status
991  if [ $DRYRUN = 0 ]; then
992    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
993      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
994      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
995      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
996      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
997      return
998    fi
999
1000    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1001      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1002      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1003      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1004      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1005      return
1006    fi
1007    #
1008    if [ -d ${R_SAVE}/${1} ] ; then
1009      cd ${R_SAVE}
1010    elif [ -d ${R_FIGR}/${1} ] ; then
1011      cd ${R_FIGR}
1012    fi
1013
1014    IGCM_sys_Dods_Rm ${1}
1015    IGCM_sys_Dods_Cp ${1}
1016    status=0
1017
1018    if [ ${status} -gt 0 ] ; then
1019      echo "IGCM_sys_Put_Dods : error."
1020      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1021    fi
1022  fi
1023  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1024}
1025
1026##############################################################
1027# REBUILD OPERATOR
1028
1029#D-#==================================================
1030#D-function IGCM_sys_sync
1031#D-* Purpose: flush buffer on disk
1032#D-* Examples:
1033#D-
1034function IGCM_sys_sync {
1035  IGCM_debug_PushStack "IGCM_sys_sync" $@
1036  if ( $DEBUG_sys ) ; then
1037    echo "IGCM_sys_sync :" $@
1038  fi
1039
1040  /bin/sync
1041
1042  IGCM_debug_PopStack "IGCM_sys_sync"
1043}
1044
1045############################################################
1046# Activate Running Environnment Variables
1047
1048#D-#==================================================
1049#D-function IGCM_sys_activ_variables
1050#D-* Purpose: set environement variables prior to execution
1051#D-* Examples:
1052#D-
1053function IGCM_sys_activ_variables {
1054  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1055  if ( $DEBUG_sys ) ; then
1056    echo "IGCM_sys_activ_variables"
1057  fi
1058
1059# --------------------------------------------------------------------
1060#D- MODULE specifications
1061# --------------------------------------------------------------------
1062  module purge > /dev/null 2>&1
1063  if [ X${compilerVersion} = XV17 ] ; then
1064    IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V17"
1065  else
1066    IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1067  fi
1068  . /ccc/cont003/home/dsm/p86ipsl/.env_irene > /dev/null 2>&1
1069  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded module for Irene. "
1070  module list
1071
1072  # Keep track of this
1073  [ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
1074  [ ! X${TaskType} = Xchecking ] && module list
1075
1076# --------------------------------------------------------------------
1077#D- MPI specifications
1078# --------------------------------------------------------------------
1079
1080# --------------------------------------------------------------------
1081#D- Other specifications
1082# --------------------------------------------------------------------
1083
1084  ulimit -s unlimited
1085
1086  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1087}
1088
1089############################################################
1090# Desactivate Running Environnment Variables
1091
1092#D-#==================================================
1093#D-function IGCM_sys_desactiv_variables
1094#D-* Purpose: unset environement variables after execution
1095#D-* Examples:
1096#D-
1097function IGCM_sys_desactiv_variables {
1098  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1099  if ( $DEBUG_sys ) ; then
1100    echo "IGCM_sys_desactiv_variables"
1101  fi
1102# --------------------------------------------------------------------
1103#D- MPI specifications
1104# --------------------------------------------------------------------
1105
1106# --------------------------------------------------------------------
1107#D- Other specifications
1108# --------------------------------------------------------------------
1109
1110  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1111}
1112
1113############################################################
1114# Update job headers to be used by the scheduler
1115
1116#D-#==================================================
1117#D-function IGCM_sys_updateHeaders
1118#D-* Purpose: Update job headers to be used by the scheduler
1119#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1120#D-
1121function IGCM_sys_updateHeaders {
1122  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1123  if ( $DEBUG_sys ) ; then
1124    echo "IGCM_sys_updateHeaders"
1125  fi
1126  typeset file
1127  file=$1
1128
1129  if [ ${executionType} -eq 1 ] ; then
1130    # MPMD + MPI
1131    sed -e "/::openMPthreads::/d"                  \
1132        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1133        -e "/#MSUB -x/d"                           \
1134        -e "/--cpu_bind=none/d"                    \
1135      ${file} > ${file}.tmp
1136
1137  elif [ ${executionType} -eq 2 ] ; then
1138    # MPMD + MPI + OMP : mpirun/ccc_mprun/error
1139    if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1140      sed -e "/::openMPthreads::/d"                  \
1141          -e "s/::JobNumProcTot::/${coreNumber}/"    \
1142        ${file} > ${file}.tmp
1143    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1144      sed -e "/::openMPthreads::/d"                  \
1145          -e "s/::JobNumProcTot::/${coreNumber}/"    \
1146          -e "/--cpu_bind=none/d"                    \
1147        ${file} > ${file}.tmp
1148    else
1149     IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
1150     IGCM_debug_Exit "IGCM_sys_updateHeaders"
1151    fi
1152  elif [ ${executionType} -eq 3 ] ; then
1153    # SPMD + MPI/OMP
1154    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1155        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1156        -e "/#MSUB -x/d"                           \
1157        -e "/--cpu_bind=none/d"                    \
1158      ${file} > ${file}.tmp
1159
1160  elif [ ${executionType} -eq 4 ] ; then
1161    # SPMD + MPI only
1162    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1163        -e "/::openMPthreads::/d"                  \
1164        -e "/#MSUB -x/d"                           \
1165        -e "/--cpu_bind=none/d"                    \
1166      ${file} > ${file}.tmp
1167
1168  elif [ ${executionType} -eq 5 ] ; then
1169    # SPMD + OMP only
1170    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1171        -e "/::JobNumProcTot::/d"                  \
1172        -e "/#MSUB -x/d"                           \
1173        -e "/--cpu_bind=none/d"                    \
1174      ${file} > ${file}.tmp
1175
1176  elif [ ${executionType} -eq 6 ] ; then
1177    # SEQUENTIAL THEN
1178    sed -e "s/::JobNumProcTot::/1/"                \
1179        -e "/::openMPthreads::/d"                  \
1180        -e "/#MSUB -x/d"                           \
1181        -e "/--cpu_bind=none/d"                    \
1182      ${file} > ${file}.tmp
1183
1184  fi
1185
1186  IGCM_sys_Mv ${file}.tmp ${file}
1187
1188  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1189}
1190
1191############################################################
1192# Build MPI/OMP scripts run file (dummy function)
1193
1194#D-#==================================================
1195#D-function IGCM_sys_build_run_file
1196#D-* Purpose: build run file (deprecated)
1197#D-* Examples:
1198#D-
1199function IGCM_sys_build_run_file {
1200  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1201}
1202
1203############################################################
1204# Build MPI/OMP scripts
1205
1206#D-#==================================================
1207#D-function IGCM_sys_build_execution_scripts
1208#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1209#D-* Examples:
1210#D-
1211function IGCM_sys_build_execution_scripts
1212{
1213  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1214  if ( $DEBUG_sys ) ; then
1215    echo "IGCM_sys_build_execution_scripts " $@
1216  fi
1217
1218  EXECUTION=${HOST_MPIRUN_COMMAND}
1219
1220  if ( ${OK_PARA_MPMD} ) ; then
1221
1222    # MPMD mode
1223    # 1 MPI only
1224    # 2 MPI/OpenMP mpirun method
1225    # 3 MPI/OpenMP ccc_mprun method
1226
1227    if [ -f run_file ] ; then
1228      IGCM_sys_Rm -f run_file
1229    fi
1230    touch run_file
1231
1232    # case 1 : Only MPI (MPMD)
1233    if  ( ! ${OK_PARA_OMP} ) ; then
1234
1235      # Build run_file
1236
1237      # First loop on the components for the coupler ie oasis (only if oasis3)
1238      # the coupler ie oasis3 must be the first one
1239      for comp in ${config_ListOfComponents[*]} ; do
1240
1241        eval ExeNameIn=\${config_Executable_${comp}[0]}
1242        eval ExeNameOut=\${config_Executable_${comp}[1]}
1243
1244        # for CPL component only
1245        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1246
1247          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1248          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1249          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1250        fi
1251      done
1252
1253      # Then second loop on the components
1254      for comp in ${config_ListOfComponents[*]} ; do
1255
1256        eval ExeNameIn=\${config_Executable_${comp}[0]}
1257        eval ExeNameOut=\${config_Executable_${comp}[1]}
1258
1259        # Only if we really have an executable for the component and not the coupler ie oasis:
1260        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1261
1262          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1263          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1264          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1265        fi
1266      done
1267
1268      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1269
1270      IGCM_sys_Chmod u+x run_file
1271      if ( $DEBUG_sys ) ; then
1272        echo "run_file contains : "
1273        cat run_file
1274      fi
1275
1276    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1277
1278    # 2 MPI/OpenMP mpirun method
1279
1280      # Use of mpirun instead of ccc_mprun
1281      EXECUTION="time mpirun"
1282
1283      #  Hosts treatment
1284      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1285
1286      i=0
1287      rm -f hosts rankfile
1288      IGCM_debug_Print 1 "sys Curie, Hosts available :"
1289      for nodes in `cat hosts.tmp`
1290      do
1291        host[$i]=$nodes
1292        echo "${host[$i]}" >> hosts
1293        IGCM_debug_Print 1 ${host[$i]}
1294        i=$((i+1))
1295      done
1296      rm -f hosts.tmp
1297
1298      listnodes=${host[*]}
1299
1300      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1301
1302      # Initialisation
1303      rank=0
1304      current_core=0
1305      init_exec=n
1306
1307      # Loop on the components
1308      for comp in ${config_ListOfComponents[*]} ; do
1309
1310        eval ExeNameIn=\${config_Executable_${comp}[0]}
1311        eval ExeNameOut=\${config_Executable_${comp}[1]}
1312
1313        # Not possible if oasis has an executable (i.e old version of oasis3)
1314        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1315          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1316          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1317          IGCM_debug_Verif_Exit
1318        fi
1319
1320        # Only if we really have an executable for the component :
1321        if [ "X${ExeNameOut}" != X\"\" ] ; then
1322
1323          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1324          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1325          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1326
1327          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1328          echo ""  >> script_${ExeNameOut}.ksh
1329          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1330
1331            # Check if the number of threads is correct
1332            case ${comp_proc_omp_loc} in
1333            2|4|8|16)
1334              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1335              ;;
1336            *)
1337              IGCM_debug_Exit "ERROR with OMP parameters !"
1338              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1339              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1340              IGCM_debug_Verif_Exit
1341              ;;
1342            esac
1343            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1344            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1345            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1346            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1347          fi
1348
1349          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1350          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1351          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1352          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1353          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1354
1355          if [ ${init_exec} = y ] ; then
1356            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1357          else
1358            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1359            init_exec=y
1360          fi
1361
1362          # Build rankfile : method used to assign cores and nodes for the MPI process
1363          # Ex :
1364          #rank 0=curie5296 slot=0,1,2,3
1365          #rank 1=curie5296 slot=4,5,6,7
1366          # Example of final command :
1367          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1368          # with script_lmdz.x.ksh :
1369          # #!/bin/ksh
1370          #export KMP_STACKSIZE=3g
1371          #export KMP_LIBRARY=turnaround
1372          #export MKL_SERIAL=YES
1373          #OMP_NUM_THREADS=4
1374          #./lmdz.x
1375          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1376              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1377          else
1378              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1379          fi
1380          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1381             (( index_host = current_core / NB_CORE_PER_NODE ))
1382             host_value=${host[${index_host}]}
1383             (( slot =  current_core % NB_CORE_PER_NODE ))
1384             virg=","
1385             string_final=""
1386             for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1387                 string=$index$virg
1388                 string_final=$string_final$string
1389             done
1390             string_final=$( echo $string_final | sed "s/.$//" )
1391             echo "rank $rank=$host_value slot=$string_final" >> rankfile
1392             (( rank = rank + 1 ))
1393             (( current_core = current_core + offset_comp_proc_loc ))
1394          done
1395        fi
1396      done
1397
1398    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1399
1400    # 3 MPI/OpenMP ccc_mprun method
1401
1402    # MPI-OpenMP (MPMD)
1403    # example of run_file
1404    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1405    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1406    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1407
1408      # Loop on the components
1409      for comp in ${config_ListOfComponents[*]} ; do
1410
1411        eval ExeNameIn=\${config_Executable_${comp}[0]}
1412        eval ExeNameOut=\${config_Executable_${comp}[1]}
1413
1414        # Not possible if oasis has an executable (i.e old version of oasis3)
1415        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1416          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1417          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1418          IGCM_debug_Verif_Exit
1419        fi
1420
1421        # Only if we really have an executable for the component :
1422        if [ "X${ExeNameOut}" != X\"\" ] ; then
1423
1424          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1425          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1426
1427          # Build script files
1428
1429          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1430          echo ""  >> script_${ExeNameOut}.ksh
1431          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1432
1433            # Check if the number of threads is correct
1434            case ${comp_proc_omp_loc} in
1435            2|4|8|12|16|24)
1436              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1437              ;;
1438            *)
1439              IGCM_debug_Exit "ERROR with OMP parameters !"
1440              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1441              IGCM_debug_Print 2 "Only 2,4,8,12,16,24 as number of OMP threads are possible "
1442              IGCM_debug_Verif_Exit
1443              ;;
1444            esac
1445            ### only for LMDZ?
1446            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1447            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1448            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1449            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1450
1451          fi
1452
1453          # to have out/err per process on different files
1454          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1455          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1456
1457          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1458
1459          # Complete run_file
1460
1461          echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1462
1463        fi
1464      done
1465
1466      ## variable added to stop after 60s instead of 600s by default.
1467      ## This is used when no error comes from executables and when something stopped an executable without notice.
1468      export SLURM_WAIT=60
1469
1470      ## module advised by TGCC (instead of 2 variables)
1471      module load feature/bridge/heterogenous_mpmd
1472
1473      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1474
1475      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1476
1477      IGCM_sys_Chmod u+x run_file
1478      if ( $DEBUG_sys ) ; then
1479        echo "run_file contains : "
1480        cat run_file
1481      fi
1482
1483    else
1484
1485      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1486
1487    fi # 1 2 or 3 if ${OK_PARA_MPMD}
1488
1489  else
1490  # Only one executable (SPMD mode).
1491
1492    for comp in ${config_ListOfComponents[*]} ; do
1493
1494      # Only if we really have an executable for the component :
1495      eval ExeNameOut=\${config_Executable_${comp}[1]}
1496      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1497
1498        # Build script files
1499
1500        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1501        echo ""  >> script_${ExeNameOut}.ksh
1502        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1503
1504        if ( ${OK_PARA_OMP} ) ; then
1505          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1506          echo ""  >> script_${ExeNameOut}.ksh
1507          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1508          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1509          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1510          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1511        fi
1512
1513        if  ( ${OK_PARA_MPI} ) ; then
1514          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1515          # Default : ccc_mprun used if nb_proc gt 1
1516          # to have out/err per process on different files
1517          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1518          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1519          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1520        else
1521          # Default : ccc_mprun is NOT used if nb_proc eq 1
1522          # to have out/err per process on different files
1523          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1524          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1525          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1526        fi
1527
1528        IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
1529        cat script_${ExeNameOut}.ksh
1530
1531      fi
1532
1533    done
1534
1535  fi # ${OK_PARA_MPMD}
1536
1537  IGCM_debug_Print 1 "sys Curie : execution command is "
1538  IGCM_debug_Print 1 "$EXECUTION"
1539
1540  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1541}
1542
1543#D-#==================================================
1544#D-function IGCM_sys_check_path
1545#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1546#D-* do not point to an important use directory. Stop immediately in that case.
1547#D-* Examples:
1548#D-
1549function IGCM_sys_check_path {
1550  IGCM_debug_PushStack "IGCM_sys_check_path"
1551  if ( $DEBUG_sys ) ; then
1552    echo "IGCM_sys_check_path"
1553  fi
1554
1555  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1556    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1557    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1558    IGCM_debug_Exit "This will stop the job"
1559  fi
1560  IGCM_debug_PopStack "IGCM_sys_check_path"
1561}
1562
1563#D-#==================================================
1564#D-function IGCM_sys_check_quota
1565#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1566#D-* Examples:
1567#D-
1568function IGCM_sys_check_quota {
1569  IGCM_debug_PushStack "IGCM_sys_check_quota"
1570  if ( $DEBUG_sys ) ; then
1571    echo "IGCM_sys_check_quota"
1572  fi
1573  # Limit of quota (in %)
1574  limit_quota=90
1575
1576  # Check of the volume
1577  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1578  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1579
1580  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1581
1582    unit_avail=${volume_avail: -1}
1583    unit_quota=${volume_quota: -1}
1584
1585    if [ "${unit_quota}" = "*" ] ; then
1586      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1587      IGCM_debug_Print 1 "More than 100% of your quota is used"
1588      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1589      IGCM_debug_Print 1 "You must have more than 10% available to run"
1590      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1591      IGCM_debug_Verif_Exit
1592    fi
1593
1594    temp_avail=${volume_avail%%${volume_avail: -1}*}
1595    temp_quota=${volume_quota%%${volume_quota: -1}*}
1596
1597    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1598
1599    # Convertion
1600      if [ ${volume_avail: -1} = "T" ] ; then
1601        (( temp_avail = temp_avail * 1000000000000 ))
1602      elif [ ${volume_avail: -1} = "G" ] ; then
1603        (( temp_avail = temp_avail * 1000000000 ))
1604      elif [ ${volume_avail: -1} = "M" ] ; then
1605        (( temp_avail = temp_avail * 1000000 ))
1606      elif [ ${volume_avail: -1} = "k" ] ; then
1607        (( temp_avail = temp_avail * 1000 ))
1608      else
1609        (( temp_avail = volume_avail ))
1610      fi
1611      if [ ${volume_quota: -1} = "T" ] ; then
1612        (( temp_quota = temp_quota * 1000000000000 ))
1613      elif [ ${volume_quota: -1} = "G" ] ; then
1614        (( temp_quota = temp_quota * 1000000000 ))
1615      elif [ ${volume_quota: -1} = "M" ] ; then
1616        (( temp_quota = temp_quota * 1000000 ))
1617      elif [ ${volume_quota: -1} = "k" ] ; then
1618        (( temp_quota = temp_quota * 1000 ))
1619      else
1620        (( temp_quota = volume_quota ))
1621      fi
1622    fi
1623
1624    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1625#    echo "volume ratio is " $quota_volume
1626
1627    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1628      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1629      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1630      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1631      IGCM_debug_Print 1 "You must have more than 10% available to run"
1632      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1633      IGCM_debug_Verif_Exit
1634    fi
1635
1636  fi
1637
1638# Check of the number of inodes
1639
1640  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1641  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1642
1643  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1644
1645    unit_avail=${inode_avail: -1}
1646    unit_quota=${inode_quota: -1}
1647
1648    if [ "${unit_quota}" = "*" ] ; then
1649      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1650      IGCM_debug_Print 1 "More than 100% of your quota is used"
1651      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1652      IGCM_debug_Print 1 "You must have more than 10% available to run"
1653      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1654      IGCM_debug_Verif_Exit
1655    fi
1656
1657    temp_avail=${inode_avail%%${inode_avail: -1}*}
1658    temp_quota=${inode_quota%%${inode_quota: -1}*}
1659
1660    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1661
1662    # Convertion
1663      if [ ${inode_avail: -1} = "T" ] ; then
1664        (( temp_avail = temp_avail * 1000000000000 ))
1665      elif [ ${inode_avail: -1} = "G" ] ; then
1666        (( temp_avail = temp_avail * 1000000000 ))
1667      elif [ ${inode_avail: -1} = "M" ] ; then
1668        (( temp_avail = temp_avail * 1000000 ))
1669      elif [ ${inode_avail: -1} = "k" ] ; then
1670        (( temp_avail = temp_avail * 1000 ))
1671      else
1672        (( temp_avail = inode_avail ))
1673      fi
1674
1675      if [ ${inode_quota: -1} = "T" ] ; then
1676        (( temp_quota = temp_quota * 1000000000000 ))
1677      elif [ ${inode_quota: -1} = "G" ] ; then
1678        (( temp_quota = temp_quota * 1000000000 ))
1679      elif [ ${inode_quota: -1} = "M" ] ; then
1680        (( temp_quota = temp_quota * 1000000 ))
1681      elif [ ${inode_quota: -1} = "k" ] ; then
1682        (( temp_quota = temp_quota * 1000 ))
1683      else
1684        (( temp_quota = inode_quota ))
1685      fi
1686    fi
1687    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1688#    echo "inode ratio is " $quota_inode
1689
1690    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1691      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1692      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1693      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1694      IGCM_debug_Print 1 "You must have more than 10% available to run"
1695      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1696      IGCM_debug_Verif_Exit
1697    fi
1698  fi
1699  IGCM_debug_PopStack "IGCM_sys_check_quota"
1700}
1701
1702#D-#==================================================
1703#D-function IGCM_sys_projectAccounting
1704#D-* Purpose: store project accounting information in a file
1705#D-* Examples:
1706#D-
1707function IGCM_sys_projectAccounting {
1708  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1709  if ( $DEBUG_sys ) ; then
1710    echo "IGCM_sys_check_quota"
1711  fi
1712
1713  ssh curie91 /usr/bin/ccc_myproject > $1
1714
1715  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1716}
1717
1718#D-#==================================================
1719#D-function IGCM_sys_getJobSchedulerID
1720#D-* Purpose: Get the job ID during execution
1721#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1722#D-
1723function IGCM_sys_getJobSchedulerID {
1724  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1725  if ( $DEBUG_sys ) ; then
1726    echo "IGCM_sys_getJobSchedulerID"
1727  fi
1728
1729  eval ${1}=${BRIDGE_MSUB_JOBID}
1730
1731  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1732}
1733
1734#D-#==================================================
1735#D-function IGCM_sys_GetJobID
1736#D-* Purpose: Get the job ID from the JobName
1737#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1738#D-
1739function IGCM_sys_GetJobID {
1740  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1741  if ( $DEBUG_sys ) ; then
1742    echo "IGCM_sys_GetJobID"
1743  fi
1744
1745  # With -f option, the full job name is given in the last column
1746  ID=$( ccc_mstat -f -u $2 | \
1747        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1748
1749  eval ${3}=${ID}
1750  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1751}
1752
1753#D-#==================================================
1754#D-function IGCM_sys_CountJobInQueue
1755#D-* Purpose: Count number of users job
1756#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1757#D-
1758function IGCM_sys_CountJobInQueue {
1759  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1760  if ( $DEBUG_sys ) ; then
1761    echo "IGCM_sys_CountJobInQueue"
1762  fi
1763
1764  # With -f option, the full job name is given in the last column
1765  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1766
1767  eval ${2}=${NbRun}
1768
1769  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1770}
1771
1772#D-#==================================================
1773#D-function IGCM_sys_ListJobInQueue
1774#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1775#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1776#D-
1777function IGCM_sys_ListJobInQueue {
1778  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1779  if ( $DEBUG_sys ) ; then
1780    echo "IGCM_sys_ListJobInQueue"
1781  fi
1782
1783  # With -f option, the full job name is given in the last column
1784  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1785                                        '( $2  == User      && \
1786                                           $NF != /TS/      && \
1787                                           $NF !~ /PACK/    && \
1788                                           $NF !~ /REBUILD/ && \
1789                                           $NF !~ /pack/ )     \
1790                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1791
1792  eval set -A ${2} ${JobList[*]}
1793
1794  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1795}
1796
1797#D-#==================================================
1798#D-function IGCM_sys_atlas
1799#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1800#D-* Examples:
1801#D-
1802function IGCM_sys_atlas {
1803  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1804  if ( $DEBUG_sys ) ; then
1805    echo "IGCM_sys_atlas :" $@
1806  fi
1807
1808  typeset status
1809
1810  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1811  status=$?
1812  if [ ${status} -gt 0 ] ; then
1813    echo "IGCM_sys_atlas : error code ${status}"
1814    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1815    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1816    IGCM_debug_PopStack "IGCM_sys_atlas"
1817    return 1
1818  else
1819    IGCM_debug_PopStack "IGCM_sys_atlas"
1820    return 0
1821  fi
1822
1823  IGCM_debug_PopStack "IGCM_sys_atlas"
1824}
1825
1826#D-#==================================================
1827#D-function IGCM_sys_rebuild_nemo
1828#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1829#D-* Examples:
1830#D-
1831
1832function IGCM_sys_rebuild_nemo {
1833  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1834  if ( $DEBUG_sys ) ; then
1835    echo "IGCM_sys_rebuild_nemo :" $@
1836  fi
1837
1838  $( ccc_home -u p86ipsl )/REBUILD_NEMO/rebuild_nemo ${1} ${2}
1839
1840  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1841}
1842
Note: See TracBrowser for help on using the repository browser.