source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_irene.ksh @ 1573

Last change on this file since 1573 was 1569, checked in by aclsce, 17 months ago

Added fonctionality to create RUN_DIR and Job_debug to be launched directly in the RUN_DIR directory.
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 64.0 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Irene
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40if ([ X${DRYRUN} != X ] && [ ${DRYRUN} = 4 ]) ; then
41  typeset -r DRYRUN=0
42  typeset -r DRYRUN_DEBUG=4
43else
44  typeset -r DRYRUN=${DRYRUN:=0}
45  typeset -r DRYRUN_DEBUG=0
46fi
47
48
49# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
50# -------------------------------------------------------------------------------------
51# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
52# |          |  Cp/Exe/param/files |            |         |                           |
53# |          |  Chmod Qsub         |            |         |                           |
54# -------------------------------------------------------------------------------------
55# |    0     |       yes           |    yes     |  yes    |      yes                  |
56# -------------------------------------------------------------------------------------
57# |    1     |       yes           |    yes     |  yes    |      no                   |
58# -------------------------------------------------------------------------------------
59# |    2     |       yes           |    yes     |  no     |      no                   |
60# -------------------------------------------------------------------------------------
61# |    3     |       yes           |    no      |  no     |      no                   |
62# -------------------------------------------------------------------------------------
63# |    4     |       yes           |    yes     |  creation of RUN_DIR and Job_debug  |
64# -------------------------------------------------------------------------------------
65
66
67#=====================================================
68# Global Variables :
69#=====================================================
70# Language : "fr" or "en"
71typeset -r MYLANG="fr"
72
73#=====================================================
74# Host user names project maxCpuTime
75# $hostname ou hostname
76typeset HOST=${HOST:=$( hostname )}
77# $username ou whoami
78typeset LOGIN=${LOGIN:=$( whoami )}
79# $hostname of the MASTER job
80typeset MASTER=irene
81# add default project on irene
82typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=::default_project::} | cut -d@ -f1 )
83# jobWarningDelay in seconds
84typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
85
86#D-
87#D-#==================================================
88#D-Program used in libIGCM
89#D-#==================================================
90
91# Submit command
92typeset SUBMIT=${SUBMIT:=ccc_msub}
93# rsync with path
94typeset -r RSYNC=/usr/bin/rsync
95# RSYNC_opt args to rsync
96typeset -r RSYNC_opt="-va"
97# ie storage filesystem
98typeset -r STOREHOST=${MASTER}
99typeset -r REMOTE_RSYNC=/usr/bin/rsync
100
101#====================================================
102# Set environment tools (ferret, nco, cdo, rebuild, ...)
103#====================================================
104if [ X${TaskType} = Xcomputing ] ; then
105  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
106else
107  module purge > /dev/null 2>&1
108  ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_irene > /dev/null 2>&1
109  . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_irene > /dev/null 2>&1
110  ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
111  . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
112
113  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
114  export UVCDAT_ANONYMOUS_LOG=FALSE
115fi
116
117# FYI
118[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
119[ ! X${TaskType} = Xchecking ] && module list
120
121# For AddNoise and AddPertu3DOCE
122##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/curie/bin
123export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/bin
124
125# For rebuild
126##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
127export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
128
129#====================================================
130# Host specific DIRECTORIES
131#====================================================
132
133# ============ CESIUM START ============ #
134
135#====================================================
136#- Mirror libIGCM from titane to cesium if needed
137#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
138#if [ ! ${ROOTSYS} = "home" ] ; then
139#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
140#else
141#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
142#fi
143
144#====================================================
145#- libIGCM_POST
146#if ( ${MirrorlibIGCM} ) ; then
147#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
148#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
149#else
150#  typeset -r libIGCM_POST=${libIGCM}
151#fi
152
153# ============ CESIUM  END  ============ #
154
155#====================================================
156#- MirrorlibIGCM for frontend
157typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
158
159#====================================================
160#- libIGCM_POST for frontend
161typeset -r libIGCM_POST=${libIGCM}
162
163#====================================================
164#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
165typeset -r R_EXE="${MODIPSL}/bin"
166
167#====================================================
168#- SUBMIT_DIR : submission dir
169typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
170
171#====================================================
172#- IN
173typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
174
175#====================================================
176#- RUN_DIR_PATH : Temporary working directory (=> TMP)
177typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
178
179#====================================================
180#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
181typeset -r OUTCOMMAND_PATH=/tmp
182
183#====================================================
184#- HOST_MPIRUN_COMMAND
185typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
186
187#====================================================
188#- Max number of arguments passed to nco operator or demigration command
189UNIX_MAX_LIMIT=360
190
191#====================================================
192#- set PackDefault to true on irene
193PackDefault=true
194
195#====================================================
196#- Default number of MPI task for IPSL coupled model
197#- required for backward compatibility
198#-
199DEFAULT_NUM_PROC_OCE=5
200DEFAULT_NUM_PROC_CPL=1
201(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
202
203#====================================================
204#- Number of computing cores per node
205#-
206NB_CORE_PER_NODE=48
207
208#D-#==================================================
209#D-function IGCM_sys_defineArchives
210#D-* Purpose:
211#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
212#D-* Define ARCHIVE : Dedicated to large files
213#D-* Define STORAGE : Dedicated to small/medium files
214#D-* Define R_OUT   : Output tree located on ARCHIVE
215#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
216#D-* Define R_BUF   : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
217#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
218#D-* Examples:
219#D-
220function IGCM_sys_defineArchives {
221  IGCM_debug_PushStack "IGCM_sys_defineArchives"
222
223  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
224  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = DEFAULT ]; then
225    # Default option: Change the dfldatadir according to the project used for submission
226    # The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
227    if [ X"$( echo ${PROJECT} | grep cmip6 )" != "X" ] ; then
228      module switch dfldatadir dfldatadir/gencmip6
229      IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
230    else
231      module switch dfldatadir dfldatadir/${PROJECT}
232      IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
233    fi
234  else
235    # Use the project set by the variable DataProject in section UserChoices in config.card
236    module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
237    IGCM_debug_Print 1 "Load dfldatadir for DataProject red in config.card : ${config_UserChoices_DataProject}"
238  fi
239
240 #====================================================
241 #- RUN_DIR_PATH : Temporary working directory (=> TMP)
242 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
243
244  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
245    #====================================================
246    #- ARCHIVE (dedicated to large files)
247    ARCHIVE=${config_UserChoices_ARCHIVE}
248  else
249    #====================================================
250    #- ARCHIVE (dedicated to large files)
251    ARCHIVE=${CCCSTOREDIR}
252  fi
253
254  if [ ! X${config_UserChoices_STORAGE} = X ]; then
255    #====================================================
256    #- STORAGE (dedicated to small/medium files)
257    STORAGE=${config_UserChoices_STORAGE}
258  else
259    #====================================================
260    #- STORAGE (dedicated to small/medium files)
261    STORAGE=${CCCWORKDIR}
262  fi
263
264  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
265    #====================================================
266    #- R_OUT
267    R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
268
269    #====================================================
270    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
271    R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
272
273    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
274
275  else
276    #====================================================
277    #- R_OUT
278    R_OUT=${ARCHIVE}/IGCM_OUT
279
280    #====================================================
281    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
282    R_FIG=${STORAGE}/IGCM_OUT
283  fi
284
285  #====================================================
286  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
287  CMIP6_BUF=${STORAGE}/IGCM_OUT
288
289  #====================================================
290  #- R_BUF (ONLY FOR double copy an scratch)
291  R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
292
293  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
294  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
295  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
296
297  IGCM_debug_PopStack "IGCM_sys_defineArchives"
298}
299
300#D-#==================================================
301#D-function IGCM_sys_RshArchive
302#D-* Purpose: Archive rsh command
303#D-* Examples:
304#D-
305function IGCM_sys_RshArchive {
306  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
307  /bin/ksh <<-EOF
308    ${@}
309EOF
310  status=$?
311  if [ ${status} -gt 0 ] ; then
312    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
313    IGCM_debug_Exit "IGCM_sys_RshArchive"
314  fi
315  IGCM_debug_PopStack "IGCM_sys_RshArchive"
316}
317
318#D-#==================================================
319#D-function IGCM_sys_RshArchive_NoError
320#D-* Purpose: Archive rsh command, without error
321#D-*          used only in monitoring.job
322#D-* Examples:
323#D-
324function IGCM_sys_RshArchive_NoError {
325  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
326  /bin/ksh <<-EOF
327    ${@} 2> /dev/null
328EOF
329  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
330}
331
332#D-#==================================================
333#D-function IGCM_sys_MkdirArchive
334#D-* Purpose: Mkdir on Archive
335#D-* Examples:
336#D-
337function IGCM_sys_MkdirArchive {
338  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
339  if ( $DEBUG_sys ) ; then
340    echo "IGCM_sys_MkdirArchive :" $@
341  fi
342  #- creation de repertoire sur le serveur fichier
343  if [ ! -d ${1} ]; then
344    \mkdir -p $1
345    status=$?
346
347    if [ ${status} -gt 0 ] ; then
348      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
349      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
350    fi
351  fi
352  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
353}
354
355#D-#==================================================
356#D-function IGCM_sys_TestDirArchive
357#D-* Purpose: Test Directory that must exists on Archive
358#D-* Examples:
359#D-
360function IGCM_sys_TestDirArchive {
361  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
362  if ( $DEBUG_sys ) ; then
363    echo "IGCM_sys_TestDirArchive :" $@
364  fi
365  typeset ExistFlag
366  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
367  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
368  return ${ExistFlag}
369}
370
371#D-#==================================================
372#D-function IGCM_sys_IsFileArchived
373#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
374#D-* Examples:
375#D-
376function IGCM_sys_IsFileArchived {
377  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
378  if ( $DEBUG_sys ) ; then
379    echo "IGCM_sys_IsFileArchived :" $@
380  fi
381  typeset IsArchivedFlag
382  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
383  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
384
385  return ${IsArchivedFlag}
386}
387
388#D-#==================================================
389#D-function IGCM_sys_TestFileArchive
390#D-* Purpose: Test file that must NOT EXISTS on Archive
391#D-* Examples:
392#D-
393function IGCM_sys_TestFileArchive {
394  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
395  typeset ExistFlag
396  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
397  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
398
399  return ${ExistFlag}
400}
401
402#D-#==================================================
403#D-function IGCM_sys_CountFileArchive
404#D-* Purpose: Count files on Archive filesystem
405#D-* Examples:
406#D-
407function IGCM_sys_CountFileArchive {
408  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
409  ls ${@} 2>/dev/null | wc -l
410  if [ $? -gt 0 ] ; then
411    echo "IGCM_sys_CountFileArchive : erreur."
412  fi
413  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
414}
415
416#D-#==================================================
417#D-function IGCM_sys_Tree
418#D-* Purpose: Tree directories with files on ${ARCHIVE}
419#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
420#D-
421function IGCM_sys_Tree {
422  IGCM_debug_PushStack "IGCM_sys_Tree" $@
423  if ( $DEBUG_sys ) ; then
424    echo "IGCM_sys_Tree :" $@
425  fi
426
427  \ls -lR ${@}
428
429  IGCM_debug_PopStack "IGCM_sys_Tree"
430}
431
432#D-#==================================================
433#D-function IGCM_sys_Qsub
434#D-* Purpose: Qsub new job
435#D-* Examples:
436#D-
437function IGCM_sys_Qsub {
438  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
439  if ( $DEBUG_sys ) ; then
440    echo "IGCM_sys_Qsub :" $@
441  fi
442  typeset options status
443  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
444
445  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
446  status=$?
447
448  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
449  if [ ${status} -gt 0 ] ; then
450    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
451    IGCM_debug_Exit "IGCM_sys_Qsub"
452  else
453    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
454  fi
455  IGCM_debug_PopStack "IGCM_sys_Qsub"
456}
457
458#D-#==================================================
459#D-function IGCM_sys_QsubPost
460#D-* Purpose: Qsub new job on scalaire
461#D-* Examples:
462#D-
463function IGCM_sys_QsubPost {
464  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
465  if ( $DEBUG_sys ) ; then
466    echo "IGCM_sys_QsubPost :" $@
467  fi
468  typeset options status
469  options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
470
471  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
472  status=$?
473
474  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
475  if [ ${status} -gt 0 ] ; then
476    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
477    IGCM_debug_Exit "IGCM_sys_QsubPost"
478  else
479    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
480  fi
481  IGCM_debug_PopStack "IGCM_sys_QsubPost"
482}
483
484#D-*************************
485#D- File transfer functions
486#D-*************************
487#D-
488
489#D-#==================================================
490#D-function IGCM_sys_RmRunDir
491#D-* Purpose: rm tmpdir (dummy function most of the time batch
492#D-                      scheduler will do the job)
493#D-* Examples:
494#D-
495function IGCM_sys_RmRunDir {
496  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
497  if ( $DEBUG_sys ) ; then
498    echo "IGCM_sys_RmRunDir :" $@
499  fi
500
501  typeset status
502
503  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
504  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
505  status=$?
506
507  if [ ${status} -gt 0 ] ; then
508    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
509    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
510    IGCM_debug_Exit "IGCM_sys_RmRunDir"
511  else
512    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
513  fi
514  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
515}
516
517#D-#==================================================
518#D-function IGCM_sys_Put_Dir
519#D-* Purpose: Copy a complete directory on $(ARCHIVE)
520#D-* Examples:
521#D-
522function IGCM_sys_Put_Dir {
523  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
524  if ( $DEBUG_sys ) ; then
525    echo "IGCM_sys_Put_Dir :" $@
526  fi
527  if [ $DRYRUN = 0 ]; then
528    if [ ! -d ${1} ] ; then
529      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
530      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
531      return
532    fi
533
534    typeset status
535
536    # Only if we use rsync
537    #IGCM_sys_TestDirArchive $( dirname $2 )
538    #
539    #USUAL WAY
540    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
541    status=$?
542
543    if [ ${status} -gt 0 ] ; then
544      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
545      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
546      IGCM_debug_Exit "IGCM_sys_Put_Dir"
547    else
548      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
549    fi
550  fi
551  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
552}
553
554#D-#==================================================
555#D-function IGCM_sys_Get_Dir
556#D-* Purpose: Copy a complete directory from ${ARCHIVE}
557#D-* Examples:
558#D-
559function IGCM_sys_Get_Dir {
560  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
561  if ( $DEBUG_sys ) ; then
562    echo "IGCM_sys_Get_Dir :" $@
563  fi
564  if [ $DRYRUN = 0 ]; then
565    typeset NB_ESSAI DELAI status i
566    # number of tentative
567    NB_ESSAI=3
568    # time delay between tentative
569    DELAI=2
570
571    #
572    # USUAL WAY
573    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
574    ccc_hsm get -r $1
575
576    i=0
577    while [ $i -lt $NB_ESSAI ] ; do
578      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
579      status=$?
580      if [ ${status} -gt 0 ] ; then
581        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
582        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
583        sleep $DELAI
584      else
585        break
586      fi
587      (( i = i + 1 ))
588    done
589
590    if [ ${status} -gt 0 ] ; then
591      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
592      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
593      IGCM_debug_Exit "IGCM_sys_Get_Dir"
594    else
595      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
596    fi
597  fi
598  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
599}
600
601#D-#==================================================
602#D-function IGCM_sys_Put_Rest
603#D-* Purpose: Put computied restarts on ${ARCHIVE}.
604#D-           File and target directory must exist.
605#D-* Examples:
606#D-
607function IGCM_sys_Put_Rest {
608  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
609  if ( $DEBUG_sys ) ; then
610    echo "IGCM_sys_Put_Rest :" $@
611  fi
612  if [ $DRYRUN = 0 ]; then
613    if [ ! -f ${1} ] ; then
614      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
615      IGCM_debug_Exit "IGCM_sys_Put_Rest"
616    fi
617
618    typeset status
619    #
620    # USUAL WAY
621    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
622    status=$?
623
624#       #RSYNC WITH NETWORK SSH CALL
625#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
626#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
627
628#       #RSYNC WITH NFS USE
629#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
630#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
631
632#       status=$?
633#       IGCM_sys_Rsync_out $status
634
635#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
636#       (( status=status+$? ))
637
638    if [ ${status} -gt 0 ] ; then
639      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
640      [ -f ${1} ] && ls -l ${1}
641      [ -f ${2} ] && ls -l ${2}
642      [ -f ${2}/${1} ] && ls -l ${2}/${1}
643      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
644      IGCM_debug_Exit "IGCM_sys_Put_Rest"
645    else
646
647      if [ X${JobType} = XRUN ] ; then
648        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
649        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
650      fi
651
652      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
653    fi
654  fi
655  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
656}
657
658#D-#==================================================
659#D-function IGCM_sys_Put_Out
660#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
661#D-* Examples:
662#D-
663function IGCM_sys_Put_Out {
664  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
665  if ( $DEBUG_sys ) ; then
666    echo "IGCM_sys_Put_Out :" $@
667  fi
668
669  typeset NB_ESSAI DELAI status i exist skip
670  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
671
672  # number of tentative
673  NB_ESSAI=3
674  # time delay between tentative
675  DELAI=2
676
677  if [ $DRYRUN = 0 ]; then
678    if [ ! -f ${1} ] ; then
679      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
680      IGCM_debug_PopStack "IGCM_sys_Put_Out"
681      return 1
682    fi
683    #
684    IGCM_sys_MkdirArchive $( dirname $2 )
685    #
686    exist=false
687    skip=false
688    if [ -f $2 ] ; then
689      IGCM_debug_Print 1 "$2 already exist"
690      ccc_hsm get $2
691      exist=true
692      if [ "X$( diff $1 $2 )" = X ] ; then
693        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
694        skip=true
695      else
696        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
697        skip=false
698      fi
699    fi
700    #
701    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
702      IGCM_sys_Chmod u+w $2
703    fi
704
705    if [ X${skip} = Xfalse ] ; then
706      i=0
707      while [ $i -lt $NB_ESSAI ] ; do
708        # Identify file system
709        fileDeviceNumberInHex=$( stat -c %d $1 )
710        status=$?
711        if [ ${status} -gt 0 ] ; then
712          IGCM_debug_Exit "IGCM_sys_Put_Out"
713        fi
714        # Identify file system
715        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
716        status=$?
717        if [ ${status} -gt 0 ] ; then
718          IGCM_debug_Exit "IGCM_sys_Put_Out"
719        fi
720
721        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
722          # They are not on the same device. USUAL WAY
723          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
724          status=$?
725        else
726          # They are on the same device. NOT SO USUAL WAY
727          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
728          status=$?
729        fi
730        if [ ${status} -gt 0 ]; then
731          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
732          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
733          [ -f ${1} ] && ls -l ${1}
734          [ -f ${2} ] && ls -l ${2}
735          [ -f ${2}/${1} ] && ls -l ${2}/${1}
736          sleep $DELAI
737        else
738          break
739        fi
740        (( i = i + 1 ))
741      done
742    fi
743
744#       #RSYNC WITH NETWORK SSH CALL
745#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
746#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
747
748#       #RSYNC WITH NFS USE
749#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
750#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
751
752#       status=$?
753#       IGCM_sys_Rsync_out $status
754
755#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
756#       (( status=status+$? ))
757
758    if [ ${status} -gt 0 ] ; then
759      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
760      [ -f ${1} ] && ls -l ${1}
761      [ -f ${2} ] && ls -l ${2}
762      [ -f ${2}/${1} ] && ls -l ${2}/${1}
763      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
764      IGCM_debug_Exit "IGCM_sys_Put_Out"
765    else
766
767      if [ X${JobType} = XRUN ] ; then
768        if [ X${3} = X ] ; then
769          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
770          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
771        fi
772      fi
773
774      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
775    fi
776  fi
777  IGCM_debug_PopStack "IGCM_sys_Put_Out"
778  return 0
779}
780
781#D-#==================================================
782#D-function IGCM_sys_Get
783#D-* Purpose: Get a file from ${ARCHIVE}
784#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
785#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
786function IGCM_sys_Get {
787  IGCM_debug_PushStack "IGCM_sys_Get" $@
788
789  typeset DEST dm_liste target file_work
790  typeset NB_ESSAI DELAI status i
791
792  if ( $DEBUG_sys ) ; then
793    echo "IGCM_sys_Get :" $@
794  fi
795
796  # number of tentative
797  NB_ESSAI=3
798  # time delay between tentative
799  DELAI=2
800
801  if [ $DRYRUN -le 2 ]; then
802    if [ X${1} = X'/l' ] ; then
803      eval set +A dm_liste \${${2}}
804    else
805      eval set +A dm_liste ${1}
806    fi
807    eval DEST=\${${#}}
808    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
809    status=$?
810    if [ ${status} -gt 0 ] ; then
811      echo "WARNING IGCM_sys_Get : error code ${status}"
812      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
813      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
814    fi
815
816    #if [ ${status} -gt 0 ] ; then
817    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
818    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
819    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
820    #    sleep 30
821    #    echo "We try another time"
822    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
823    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
824    #    status=$?
825    #    if [ ${status} -gt 0 ] ; then
826    #      echo "ERROR IGCM_sys_Get : again demigration error :"
827    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
828    #      IGCM_debug_Exit "IGCM_sys_Get"
829    #    fi
830    #  else
831    #    echo "ERROR IGCM_sys_Get : demigration error :"
832    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
833    #    IGCM_debug_Exit "IGCM_sys_Get"
834    #  fi
835    #fi
836
837    #   #RSYNC WITH NETWORK SSH CALL
838    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
839    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
840
841    #   #RSYNC WITH NFS USE
842    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
843    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
844
845    #   status=$?
846    #   IGCM_sys_Rsync_out $status
847
848    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
849    #   (( status=status+$? ))
850
851    #USUAL WAY
852    if [ X${1} = X'/l' ] ; then
853      for target in ${dm_liste[*]} ; do
854        local_file=$( basename ${target} )
855        # test if the target file is present before the loop
856        IGCM_sys_TestFileArchive ${target}
857        status=$?
858        if [ ${status} -gt 0 ] ; then
859          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
860          IGCM_debug_Exit "IGCM_sys_Get"
861        else
862          i=0
863          while [ $i -lt $NB_ESSAI ] ; do
864            #if [ X${DoLink} = Xtrue ] ; then
865            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
866            #  status=$?
867            #  else
868            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
869            #  status=$?
870            #fi
871            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
872            status=$?
873            if [ ${status} -gt 0 ]; then
874              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
875              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
876              sleep $DELAI
877            else
878              break
879            fi
880            (( i = i + 1 ))
881          done
882          if [ ${status} -gt 0 ] ; then
883            echo "IGCM_sys_Get : error"
884            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
885            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
886            IGCM_debug_Exit "IGCM_sys_Get"
887          else
888            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
889          fi
890        fi
891      done
892    else
893      i=0
894      while [ $i -lt $NB_ESSAI ] ; do
895        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
896        status=$?
897        if [ ${status} -gt 0 ]; then
898          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
899          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
900          sleep $DELAI
901        else
902          break
903        fi
904        (( i = i + 1 ))
905      done
906      if [ ${status} -gt 0 ] ; then
907        echo "IGCM_sys_Get : error"
908        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
909        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
910        IGCM_debug_Exit "IGCM_sys_Get"
911      else
912        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
913      fi
914    fi
915  fi
916  IGCM_debug_PopStack "IGCM_sys_Get"
917}
918
919#D-#==================================================
920#D-function IGCM_sys_GetDate_Monitoring
921#D-* Purpose: get the last year for which the monitoring has been computed
922#D-* Examples:
923#D-
924function IGCM_sys_GetDate_Monitoring {
925  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
926  if ( $DEBUG_sys ) ; then
927    echo "IGCM_sys_GetDate_Monitoring :" $@
928  fi
929
930  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
931
932  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
933}
934
935#D-#==================================================
936#D-function IGCM_sys_Dods_Rm
937#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
938#D-* Examples:
939#D-
940function IGCM_sys_Dods_Rm {
941  if ( $DEBUG_sys ) ; then
942    echo "IGCM_sys_Dods_Rm :" $@
943  fi
944  typeset status
945  if [ $DRYRUN = 0 ]; then
946
947#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
948#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
949#      echo "Nothing has been done."
950#      return
951#    fi
952
953    if [ "$#" -eq 1 ]; then
954      ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
955      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
956      status=$?
957    else
958      ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
959      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
960      status=$?
961    fi
962
963#    if [ ${status} -gt 0 ] ; then
964#      echo "IGCM_sys_Dods_Rm : error."
965#      cat out_dods_rm
966#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
967#    else
968#      rm out_dods_rm
969#    fi
970
971  fi
972  return $status
973}
974
975#D-#==================================================
976#D-function IGCM_sys_Dods_Cp
977#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
978#D-* Examples:
979#D-
980function IGCM_sys_Dods_Cp {
981  if ( $DEBUG_sys ) ; then
982    echo "IGCM_sys_Dods_Cp :" $@
983  fi
984  typeset status
985  if [ $DRYRUN = 0 ]; then
986
987#    if [ ! -d ${R_SAVE}/${1} ] ; then
988#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
989#      echo "Nothing has been done."
990#      return
991#    fi
992
993    ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/bin/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
994    $(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
995    status=$?
996
997#       if [ ${status} -gt 0 ] ; then
998#           echo "IGCM_sys_Dods_Cp : error."
999#           cat out_dods_cp
1000#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1001#       else
1002#           rm out_dods_cp
1003#       fi
1004
1005  fi
1006  return $status
1007}
1008
1009#D-#==================================================
1010#D-function IGCM_sys_Put_Dods
1011#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1012#D-* Examples:
1013#D-
1014function IGCM_sys_Put_Dods {
1015  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1016  if ( $DEBUG_sys ) ; then
1017    echo "IGCM_sys_Put_Dods :" $@
1018  fi
1019  typeset status
1020  if [ $DRYRUN = 0 ]; then
1021    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1022      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1023      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1024      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1025      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1026      return
1027    fi
1028
1029    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1030      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1031      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1032      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1033      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1034      return
1035    fi
1036    #
1037    if [ -d ${R_SAVE}/${1} ] ; then
1038      cd ${R_SAVE}
1039    elif [ -d ${R_FIGR}/${1} ] ; then
1040      cd ${R_FIGR}
1041    fi
1042
1043    IGCM_sys_Dods_Rm ${1}
1044    IGCM_sys_Dods_Cp ${1}
1045    status=0
1046
1047    if [ ${status} -gt 0 ] ; then
1048      echo "IGCM_sys_Put_Dods : error."
1049      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1050    fi
1051  fi
1052  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1053}
1054
1055##############################################################
1056# REBUILD OPERATOR
1057
1058#D-#==================================================
1059#D-function IGCM_sys_sync
1060#D-* Purpose: flush buffer on disk
1061#D-* Examples:
1062#D-
1063function IGCM_sys_sync {
1064  IGCM_debug_PushStack "IGCM_sys_sync" $@
1065  if ( $DEBUG_sys ) ; then
1066    echo "IGCM_sys_sync :" $@
1067  fi
1068
1069  /bin/sync
1070
1071  IGCM_debug_PopStack "IGCM_sys_sync"
1072}
1073
1074############################################################
1075# Activate Running Environnment Variables
1076
1077#D-#==================================================
1078#D-function IGCM_sys_activ_variables
1079#D-* Purpose: set environement variables prior to execution
1080#D-* Examples:
1081#D-
1082function IGCM_sys_activ_variables {
1083  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1084  if ( $DEBUG_sys ) ; then
1085    echo "IGCM_sys_activ_variables"
1086  fi
1087
1088# --------------------------------------------------------------------
1089#D- MODULE specifications
1090# --------------------------------------------------------------------
1091  # Source the file EnvFile containing all module load needed to run the model.
1092  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1093  # it will be used. Otherwise a default file will be used.
1094  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1095      # EnvFile is set in config.card and the file exists
1096      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1097      EnvFile=${config_UserChoices_EnvFile}
1098  else
1099      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1100      module purge > /dev/null 2>&1
1101      if [ X${compilerVersion} = XV17 ] ; then
1102          IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V17"
1103          ## EnvFile=/ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_irene > /dev/null 2>&1
1104          EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene/env_irene > /dev/null 2>&1
1105      else
1106          IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1107      fi
1108  fi
1109  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1110  IGCM_debug_Print 1 ${EnvFile}
1111  . ${EnvFile}
1112
1113  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene. "
1114  module list
1115
1116# --------------------------------------------------------------------
1117#D- MPI specifications
1118# --------------------------------------------------------------------
1119
1120# --------------------------------------------------------------------
1121#D- Other specifications
1122# --------------------------------------------------------------------
1123
1124  ulimit -s unlimited
1125
1126  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1127}
1128
1129############################################################
1130# Desactivate Running Environnment Variables
1131
1132#D-#==================================================
1133#D-function IGCM_sys_desactiv_variables
1134#D-* Purpose: unset environement variables after execution
1135#D-* Examples:
1136#D-
1137function IGCM_sys_desactiv_variables {
1138  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1139  if ( $DEBUG_sys ) ; then
1140    echo "IGCM_sys_desactiv_variables"
1141  fi
1142# --------------------------------------------------------------------
1143#D- MPI specifications
1144# --------------------------------------------------------------------
1145
1146# --------------------------------------------------------------------
1147#D- Other specifications
1148# --------------------------------------------------------------------
1149
1150  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1151}
1152
1153############################################################
1154# Update job headers to be used by the scheduler
1155
1156#D-#==================================================
1157#D-function IGCM_sys_updateHeaders
1158#D-* Purpose: Update job headers to be used by the scheduler
1159#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1160#D-
1161function IGCM_sys_updateHeaders {
1162  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1163  if ( $DEBUG_sys ) ; then
1164    echo "IGCM_sys_updateHeaders"
1165  fi
1166  typeset file
1167  file=$1
1168
1169  if [ ${executionType} -eq 1 ] ; then
1170    # MPMD + MPI
1171    sed -e "/::openMPthreads::/d"                  \
1172        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1173        -e "/#MSUB -x/d"                           \
1174        -e "/--cpu_bind=none/d"                    \
1175      ${file} > ${file}.tmp
1176
1177  elif [ ${executionType} -eq 2 ] ; then
1178    # MPMD + MPI + OMP : mpirun/ccc_mprun/error
1179    if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1180      sed -e "/::openMPthreads::/d"                  \
1181          -e "s/::JobNumProcTot::/${coreNumber}/"    \
1182        ${file} > ${file}.tmp
1183    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1184      sed -e "/::openMPthreads::/d"                  \
1185          -e "s/::JobNumProcTot::/${coreNumber}/"    \
1186          -e "/--cpu_bind=none/d"                    \
1187        ${file} > ${file}.tmp
1188    else
1189     IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
1190     IGCM_debug_Exit "IGCM_sys_updateHeaders"
1191    fi
1192  elif [ ${executionType} -eq 3 ] ; then
1193    # SPMD + MPI/OMP
1194    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1195        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1196        -e "/#MSUB -x/d"                           \
1197        -e "/--cpu_bind=none/d"                    \
1198      ${file} > ${file}.tmp
1199
1200  elif [ ${executionType} -eq 4 ] ; then
1201    # SPMD + MPI only
1202    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1203        -e "/::openMPthreads::/d"                  \
1204        -e "/#MSUB -x/d"                           \
1205        -e "/--cpu_bind=none/d"                    \
1206      ${file} > ${file}.tmp
1207
1208  elif [ ${executionType} -eq 5 ] ; then
1209    # SPMD + OMP only
1210    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1211        -e "/::JobNumProcTot::/d"                  \
1212        -e "/#MSUB -x/d"                           \
1213        -e "/--cpu_bind=none/d"                    \
1214      ${file} > ${file}.tmp
1215
1216  elif [ ${executionType} -eq 6 ] ; then
1217    # SEQUENTIAL THEN
1218    sed -e "s/::JobNumProcTot::/1/"                \
1219        -e "/::openMPthreads::/d"                  \
1220        -e "/#MSUB -x/d"                           \
1221        -e "/--cpu_bind=none/d"                    \
1222      ${file} > ${file}.tmp
1223
1224  fi
1225
1226  IGCM_sys_Mv ${file}.tmp ${file}
1227
1228  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1229}
1230
1231############################################################
1232# Build MPI/OMP scripts run file (dummy function)
1233
1234#D-#==================================================
1235#D-function IGCM_sys_build_run_file
1236#D-* Purpose: build run file (deprecated)
1237#D-* Examples:
1238#D-
1239function IGCM_sys_build_run_file {
1240  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1241}
1242
1243############################################################
1244# Build MPI/OMP scripts
1245
1246#D-#==================================================
1247#D-function IGCM_sys_build_execution_scripts
1248#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1249#D-* Examples:
1250#D-
1251function IGCM_sys_build_execution_scripts
1252{
1253  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1254  if ( $DEBUG_sys ) ; then
1255    echo "IGCM_sys_build_execution_scripts " $@
1256  fi
1257
1258  EXECUTION=${HOST_MPIRUN_COMMAND}
1259
1260  if ( ${OK_PARA_MPMD} ) ; then
1261
1262    # MPMD mode
1263    # 1 MPI only
1264    # 2 MPI/OpenMP mpirun method
1265    # 3 MPI/OpenMP ccc_mprun method
1266
1267    if [ -f run_file ] ; then
1268      IGCM_sys_Rm -f run_file
1269    fi
1270    touch run_file
1271
1272    # case 1 : Only MPI (MPMD)
1273    if  ( ! ${OK_PARA_OMP} ) ; then
1274
1275      # Build run_file
1276
1277      # First loop on the components for the coupler ie oasis (only if oasis3)
1278      # the coupler ie oasis3 must be the first one
1279      for comp in ${config_ListOfComponents[*]} ; do
1280
1281        eval ExeNameIn=\${config_Executable_${comp}[0]}
1282        eval ExeNameOut=\${config_Executable_${comp}[1]}
1283
1284        # for CPL component only
1285        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1286
1287          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1288          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1289          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1290        fi
1291      done
1292
1293      # Then second loop on the components
1294      for comp in ${config_ListOfComponents[*]} ; do
1295
1296        eval ExeNameIn=\${config_Executable_${comp}[0]}
1297        eval ExeNameOut=\${config_Executable_${comp}[1]}
1298
1299        # Only if we really have an executable for the component and not the coupler ie oasis:
1300        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1301
1302          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1303          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1304          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1305        fi
1306      done
1307
1308      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1309
1310      IGCM_sys_Chmod u+x run_file
1311      if ( $DEBUG_sys ) ; then
1312        echo "run_file contains : "
1313        cat run_file
1314      fi
1315
1316    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1317
1318    # 2 MPI/OpenMP mpirun method
1319
1320      # Use of mpirun instead of ccc_mprun
1321      EXECUTION="time mpirun"
1322
1323      #  Hosts treatment
1324      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1325
1326      i=0
1327      rm -f hosts rankfile
1328      IGCM_debug_Print 1 "sys Irene, Hosts available :"
1329      for nodes in `cat hosts.tmp`
1330      do
1331        host[$i]=$nodes
1332        echo "${host[$i]}" >> hosts
1333        IGCM_debug_Print 1 ${host[$i]}
1334        i=$((i+1))
1335      done
1336      rm -f hosts.tmp
1337
1338      listnodes=${host[*]}
1339
1340      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1341
1342      # Initialisation
1343      rank=0
1344      current_core=0
1345      init_exec=n
1346
1347      # Loop on the components
1348      for comp in ${config_ListOfComponents[*]} ; do
1349
1350        eval ExeNameIn=\${config_Executable_${comp}[0]}
1351        eval ExeNameOut=\${config_Executable_${comp}[1]}
1352
1353        # Not possible if oasis has an executable (i.e old version of oasis3)
1354        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1355          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1356          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1357          IGCM_debug_Verif_Exit
1358        fi
1359
1360        # Only if we really have an executable for the component :
1361        if [ "X${ExeNameOut}" != X\"\" ] ; then
1362
1363          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1364          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1365          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1366
1367          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1368          echo ""  >> script_${ExeNameOut}.ksh
1369          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1370
1371            # Check if the number of threads is correct
1372            case ${comp_proc_omp_loc} in
1373            2|4|8|16)
1374              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1375              ;;
1376            *)
1377              IGCM_debug_Exit "ERROR with OMP parameters !"
1378              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1379              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1380              IGCM_debug_Verif_Exit
1381              ;;
1382            esac
1383            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1384            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1385            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1386            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1387          fi
1388
1389          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1390          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1391          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1392          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1393          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1394
1395          if [ ${init_exec} = y ] ; then
1396            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1397          else
1398            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1399            init_exec=y
1400          fi
1401
1402          # Build rankfile : method used to assign cores and nodes for the MPI process
1403          # Ex :
1404          #rank 0=curie5296 slot=0,1,2,3
1405          #rank 1=curie5296 slot=4,5,6,7
1406          # Example of final command :
1407          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1408          # with script_lmdz.x.ksh :
1409          # #!/bin/ksh
1410          #export KMP_STACKSIZE=3g
1411          #export KMP_LIBRARY=turnaround
1412          #export MKL_SERIAL=YES
1413          #OMP_NUM_THREADS=4
1414          #./lmdz.x
1415          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1416              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1417          else
1418              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1419          fi
1420          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1421             (( index_host = current_core / NB_CORE_PER_NODE ))
1422             host_value=${host[${index_host}]}
1423             (( slot =  current_core % NB_CORE_PER_NODE ))
1424             virg=","
1425             string_final=""
1426             for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1427                 string=$index$virg
1428                 string_final=$string_final$string
1429             done
1430             string_final=$( echo $string_final | sed "s/.$//" )
1431             echo "rank $rank=$host_value slot=$string_final" >> rankfile
1432             (( rank = rank + 1 ))
1433             (( current_core = current_core + offset_comp_proc_loc ))
1434          done
1435        fi
1436      done
1437
1438    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1439
1440    # 3 MPI/OpenMP ccc_mprun method
1441
1442    # MPI-OpenMP (MPMD)
1443    # example of run_file
1444    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1445    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1446    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1447
1448      # Loop on the components
1449      for comp in ${config_ListOfComponents[*]} ; do
1450
1451        eval ExeNameIn=\${config_Executable_${comp}[0]}
1452        eval ExeNameOut=\${config_Executable_${comp}[1]}
1453
1454        # Not possible if oasis has an executable (i.e old version of oasis3)
1455        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1456          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1457          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1458          IGCM_debug_Verif_Exit
1459        fi
1460
1461        # Only if we really have an executable for the component :
1462        if [ "X${ExeNameOut}" != X\"\" ] ; then
1463
1464          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1465          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1466
1467          # Build script files
1468
1469          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1470          echo ""  >> script_${ExeNameOut}.ksh
1471          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1472
1473            # Check if the number of threads is correct
1474            case ${comp_proc_omp_loc} in
1475            2|4|8|12|16|24)
1476              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1477              ;;
1478            *)
1479              IGCM_debug_Exit "ERROR with OMP parameters !"
1480              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1481              IGCM_debug_Print 2 "Only 2,4,8,12,16,24 as number of OMP threads are possible "
1482              IGCM_debug_Verif_Exit
1483              ;;
1484            esac
1485            ### only for LMDZ?
1486            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1487            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1488            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1489            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1490
1491          fi
1492
1493          # to have out/err per process on different files
1494          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1495          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1496
1497          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1498
1499          # Complete run_file
1500
1501          echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1502
1503        fi
1504      done
1505
1506      ## variable added to stop after 60s instead of 600s by default.
1507      ## This is used when no error comes from executables and when something stopped an executable without notice.
1508      export SLURM_WAIT=60
1509
1510      ## module advised by TGCC (instead of 2 variables)
1511      module load feature/bridge/heterogenous_mpmd
1512
1513      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1514
1515      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1516
1517      IGCM_sys_Chmod u+x run_file
1518      if ( $DEBUG_sys ) ; then
1519        echo "run_file contains : "
1520        cat run_file
1521      fi
1522
1523    else
1524
1525      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1526
1527    fi # 1 2 or 3 if ${OK_PARA_MPMD}
1528
1529  else
1530  # Only one executable (SPMD mode).
1531
1532    for comp in ${config_ListOfComponents[*]} ; do
1533
1534      # Only if we really have an executable for the component :
1535      eval ExeNameOut=\${config_Executable_${comp}[1]}
1536      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1537
1538        # Build script files
1539
1540        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1541        echo ""  >> script_${ExeNameOut}.ksh
1542        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1543
1544        if ( ${OK_PARA_OMP} ) ; then
1545          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1546          echo ""  >> script_${ExeNameOut}.ksh
1547          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1548          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1549          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1550          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1551        fi
1552
1553        if  ( ${OK_PARA_MPI} ) ; then
1554          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1555          # Default : ccc_mprun used if nb_proc gt 1
1556          # to have out/err per process on different files
1557          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1558          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1559          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1560        else
1561          # Default : ccc_mprun is NOT used if nb_proc eq 1
1562          # to have out/err per process on different files
1563          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1564          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1565          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1566        fi
1567
1568        IGCM_debug_Print 1 "sys Irene : script_${ExeNameOut}.ksh contains"
1569        cat script_${ExeNameOut}.ksh
1570
1571      fi
1572
1573    done
1574
1575  fi # ${OK_PARA_MPMD}
1576
1577  IGCM_debug_Print 1 "sys Irene : execution command is "
1578  IGCM_debug_Print 1 "$EXECUTION"
1579
1580  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1581}
1582
1583#D-#==================================================
1584#D-function IGCM_sys_check_path
1585#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1586#D-* do not point to an important use directory. Stop immediately in that case.
1587#D-* Examples:
1588#D-
1589function IGCM_sys_check_path {
1590  IGCM_debug_PushStack "IGCM_sys_check_path"
1591  if ( $DEBUG_sys ) ; then
1592    echo "IGCM_sys_check_path"
1593  fi
1594
1595  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1596    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1597    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1598    IGCM_debug_Exit "This will stop the job"
1599  fi
1600  IGCM_debug_PopStack "IGCM_sys_check_path"
1601}
1602
1603#D-#==================================================
1604#D-function IGCM_sys_check_quota
1605#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1606#D-* Examples:
1607#D-
1608function IGCM_sys_check_quota {
1609  IGCM_debug_PushStack "IGCM_sys_check_quota"
1610  if ( $DEBUG_sys ) ; then
1611    echo "IGCM_sys_check_quota"
1612  fi
1613  # Limit of quota (in %)
1614  limit_quota=90
1615
1616  # Check of the volume
1617  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1618  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1619
1620  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1621
1622    unit_avail=${volume_avail: -1}
1623    unit_quota=${volume_quota: -1}
1624
1625    if [ "${unit_quota}" = "*" ] ; then
1626      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1627      IGCM_debug_Print 1 "More than 100% of your quota is used"
1628      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1629      IGCM_debug_Print 1 "You must have more than 10% available to run"
1630      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1631      IGCM_debug_Verif_Exit
1632    fi
1633
1634    temp_avail=${volume_avail%%${volume_avail: -1}*}
1635    temp_quota=${volume_quota%%${volume_quota: -1}*}
1636
1637    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1638
1639    # Convertion
1640      if [ ${volume_avail: -1} = "T" ] ; then
1641        (( temp_avail = temp_avail * 1000000000000 ))
1642      elif [ ${volume_avail: -1} = "G" ] ; then
1643        (( temp_avail = temp_avail * 1000000000 ))
1644      elif [ ${volume_avail: -1} = "M" ] ; then
1645        (( temp_avail = temp_avail * 1000000 ))
1646      elif [ ${volume_avail: -1} = "k" ] ; then
1647        (( temp_avail = temp_avail * 1000 ))
1648      else
1649        (( temp_avail = volume_avail ))
1650      fi
1651      if [ ${volume_quota: -1} = "T" ] ; then
1652        (( temp_quota = temp_quota * 1000000000000 ))
1653      elif [ ${volume_quota: -1} = "G" ] ; then
1654        (( temp_quota = temp_quota * 1000000000 ))
1655      elif [ ${volume_quota: -1} = "M" ] ; then
1656        (( temp_quota = temp_quota * 1000000 ))
1657      elif [ ${volume_quota: -1} = "k" ] ; then
1658        (( temp_quota = temp_quota * 1000 ))
1659      else
1660        (( temp_quota = volume_quota ))
1661      fi
1662    fi
1663
1664    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1665#    echo "volume ratio is " $quota_volume
1666
1667    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1668      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1669      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1670      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1671      IGCM_debug_Print 1 "You must have more than 10% available to run"
1672      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1673      IGCM_debug_Verif_Exit
1674    fi
1675
1676  fi
1677
1678# Check of the number of inodes
1679
1680  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1681  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1682
1683  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1684
1685    unit_avail=${inode_avail: -1}
1686    unit_quota=${inode_quota: -1}
1687
1688    if [ "${unit_quota}" = "*" ] ; then
1689      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1690      IGCM_debug_Print 1 "More than 100% of your quota is used"
1691      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1692      IGCM_debug_Print 1 "You must have more than 10% available to run"
1693      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1694      IGCM_debug_Verif_Exit
1695    fi
1696
1697    temp_avail=${inode_avail%%${inode_avail: -1}*}
1698    temp_quota=${inode_quota%%${inode_quota: -1}*}
1699
1700    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1701
1702    # Convertion
1703      if [ ${inode_avail: -1} = "T" ] ; then
1704        (( temp_avail = temp_avail * 1000000000000 ))
1705      elif [ ${inode_avail: -1} = "G" ] ; then
1706        (( temp_avail = temp_avail * 1000000000 ))
1707      elif [ ${inode_avail: -1} = "M" ] ; then
1708        (( temp_avail = temp_avail * 1000000 ))
1709      elif [ ${inode_avail: -1} = "k" ] ; then
1710        (( temp_avail = temp_avail * 1000 ))
1711      else
1712        (( temp_avail = inode_avail ))
1713      fi
1714
1715      if [ ${inode_quota: -1} = "T" ] ; then
1716        (( temp_quota = temp_quota * 1000000000000 ))
1717      elif [ ${inode_quota: -1} = "G" ] ; then
1718        (( temp_quota = temp_quota * 1000000000 ))
1719      elif [ ${inode_quota: -1} = "M" ] ; then
1720        (( temp_quota = temp_quota * 1000000 ))
1721      elif [ ${inode_quota: -1} = "k" ] ; then
1722        (( temp_quota = temp_quota * 1000 ))
1723      else
1724        (( temp_quota = inode_quota ))
1725      fi
1726    fi
1727    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1728#    echo "inode ratio is " $quota_inode
1729
1730    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1731      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1732      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1733      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1734      IGCM_debug_Print 1 "You must have more than 10% available to run"
1735      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1736      IGCM_debug_Verif_Exit
1737    fi
1738  fi
1739  IGCM_debug_PopStack "IGCM_sys_check_quota"
1740}
1741
1742#D-#==================================================
1743#D-function IGCM_sys_projectAccounting
1744#D-* Purpose: store project accounting information in a file
1745#D-* Examples:
1746#D-
1747function IGCM_sys_projectAccounting {
1748  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1749  if ( $DEBUG_sys ) ; then
1750    echo "IGCM_sys_check_quota"
1751  fi
1752
1753  ssh irene191 /usr/bin/ccc_myproject > $1
1754
1755  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1756}
1757
1758#D-#==================================================
1759#D-function IGCM_sys_getJobSchedulerID
1760#D-* Purpose: Get the job ID during execution
1761#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1762#D-
1763function IGCM_sys_getJobSchedulerID {
1764  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1765  if ( $DEBUG_sys ) ; then
1766    echo "IGCM_sys_getJobSchedulerID"
1767  fi
1768
1769  eval ${1}=${BRIDGE_MSUB_JOBID}
1770
1771  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1772}
1773
1774#D-#==================================================
1775#D-function IGCM_sys_GetJobID
1776#D-* Purpose: Get the job ID from the JobName
1777#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1778#D-
1779function IGCM_sys_GetJobID {
1780  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1781  if ( $DEBUG_sys ) ; then
1782    echo "IGCM_sys_GetJobID"
1783  fi
1784
1785  # With -f option, the full job name is given in the last column
1786  ID=$( ccc_mstat -f -u $2 | \
1787        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1788
1789  eval ${3}=${ID}
1790  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1791}
1792
1793#D-#==================================================
1794#D-function IGCM_sys_CountJobInQueue
1795#D-* Purpose: Count number of users job
1796#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1797#D-
1798function IGCM_sys_CountJobInQueue {
1799  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1800  if ( $DEBUG_sys ) ; then
1801    echo "IGCM_sys_CountJobInQueue"
1802  fi
1803
1804  # With -f option, the full job name is given in the last column
1805  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1806
1807  eval ${2}=${NbRun}
1808
1809  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1810}
1811
1812#D-#==================================================
1813#D-function IGCM_sys_ListJobInQueue
1814#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1815#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1816#D-
1817function IGCM_sys_ListJobInQueue {
1818  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1819  if ( $DEBUG_sys ) ; then
1820    echo "IGCM_sys_ListJobInQueue"
1821  fi
1822
1823  # With -f option, the full job name is given in the last column
1824  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1825                                        '( $2  == User      && \
1826                                           $NF != /TS/      && \
1827                                           $NF !~ /PACK/    && \
1828                                           $NF !~ /REBUILD/ && \
1829                                           $NF !~ /pack/ )     \
1830                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1831
1832  eval set -A ${2} ${JobList[*]}
1833
1834  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1835}
1836
1837#D-#==================================================
1838#D-function IGCM_sys_atlas
1839#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1840#D-* Examples:
1841#D-
1842function IGCM_sys_atlas {
1843  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1844  if ( $DEBUG_sys ) ; then
1845    echo "IGCM_sys_atlas :" $@
1846  fi
1847
1848  typeset status
1849
1850  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1851  status=$?
1852  if [ ${status} -gt 0 ] ; then
1853    echo "IGCM_sys_atlas : error code ${status}"
1854    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1855    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1856    IGCM_debug_PopStack "IGCM_sys_atlas"
1857    return 1
1858  else
1859    IGCM_debug_PopStack "IGCM_sys_atlas"
1860    return 0
1861  fi
1862
1863  IGCM_debug_PopStack "IGCM_sys_atlas"
1864}
1865
1866#D-#==================================================
1867#D-function IGCM_sys_rebuild_nemo
1868#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1869#D-* Examples:
1870#D-
1871
1872function IGCM_sys_rebuild_nemo {
1873  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1874  if ( $DEBUG_sys ) ; then
1875    echo "IGCM_sys_rebuild_nemo :" $@
1876  fi
1877
1878  $( ccc_home -u igcmg)/Tools/irene/bin/rebuild_nemo ${1} ${2}
1879
1880  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1881}
1882
Note: See TracBrowser for help on using the repository browser.