source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_irene.ksh

Last change on this file was 1636, checked in by aclsce, 3 weeks ago

Modified to use dedicated node on irene-skl, even if we use less than 128 cores.
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 64.3 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Irene
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40if ([ X${DRYRUN} != X ] && [ ${DRYRUN} = 4 ]) ; then
41  typeset -r DRYRUN=0
42  typeset -r DRYRUN_DEBUG=4
43else
44  typeset -r DRYRUN=${DRYRUN:=0}
45  typeset -r DRYRUN_DEBUG=0
46fi
47
48
49# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
50# -------------------------------------------------------------------------------------
51# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
52# |          |  Cp/Exe/param/files |            |         |                           |
53# |          |  Chmod Qsub         |            |         |                           |
54# -------------------------------------------------------------------------------------
55# |    0     |       yes           |    yes     |  yes    |      yes                  |
56# -------------------------------------------------------------------------------------
57# |    1     |       yes           |    yes     |  yes    |      no                   |
58# -------------------------------------------------------------------------------------
59# |    2     |       yes           |    yes     |  no     |      no                   |
60# -------------------------------------------------------------------------------------
61# |    3     |       yes           |    no      |  no     |      no                   |
62# -------------------------------------------------------------------------------------
63# |    4     |       yes           |    yes     |  creation of RUN_DIR and Job_debug  |
64# -------------------------------------------------------------------------------------
65
66
67#=====================================================
68# Global Variables :
69#=====================================================
70# Language : "fr" or "en"
71typeset -r MYLANG="fr"
72
73#=====================================================
74# Host user names project maxCpuTime
75# $hostname ou hostname
76typeset HOST=${HOST:=$( hostname )}
77# $username ou whoami
78typeset LOGIN=${LOGIN:=$( whoami )}
79# $hostname of the MASTER job
80typeset MASTER=irene
81# add default project on irene
82typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=::default_project::} | cut -d@ -f1 )
83# jobWarningDelay in seconds
84typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
85
86#D-
87#D-#==================================================
88#D-Program used in libIGCM
89#D-#==================================================
90
91# Submit command
92typeset SUBMIT=${SUBMIT:=ccc_msub}
93# rsync with path
94typeset -r RSYNC=/usr/bin/rsync
95# RSYNC_opt args to rsync
96typeset -r RSYNC_opt="-va"
97# ie storage filesystem
98typeset -r STOREHOST=${MASTER}
99typeset -r REMOTE_RSYNC=/usr/bin/rsync
100
101#====================================================
102# Set environment tools (ferret, nco, cdo, rebuild, ...)
103#====================================================
104typeset CCCOS=${CCCOS:=$( ccc_os )}
105
106if [ X${TaskType} = Xcomputing ] ; then
107  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
108else
109  module purge > /dev/null 2>&1
110  if [ X${CCCOS} = X"Rhel_8__x86_64" ] ; then
111      . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
112  else
113      . $( ccc_home -u igcmg)/MachineEnvironment/irene_prev/env_irene > /dev/null 2>&1
114      . $( ccc_home -u igcmg)/MachineEnvironment/irene_prev/env_atlas_irene > /dev/null 2>&1
115  fi
116
117  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
118  export UVCDAT_ANONYMOUS_LOG=FALSE
119fi
120
121# FYI
122[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
123[ ! X${TaskType} = Xchecking ] && module list
124
125# For AddNoise and AddPertu3DOCE
126##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/curie/bin
127export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/bin
128
129# For rebuild
130##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
131export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
132
133#====================================================
134# Host specific DIRECTORIES
135#====================================================
136
137# ============ CESIUM START ============ #
138
139#====================================================
140#- Mirror libIGCM from titane to cesium if needed
141#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
142#if [ ! ${ROOTSYS} = "home" ] ; then
143#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
144#else
145#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
146#fi
147
148#====================================================
149#- libIGCM_POST
150#if ( ${MirrorlibIGCM} ) ; then
151#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
152#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
153#else
154#  typeset -r libIGCM_POST=${libIGCM}
155#fi
156
157# ============ CESIUM  END  ============ #
158
159#====================================================
160#- MirrorlibIGCM for frontend
161typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
162
163#====================================================
164#- libIGCM_POST for frontend
165typeset -r libIGCM_POST=${libIGCM}
166
167#====================================================
168#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
169typeset -r R_EXE="${MODIPSL}/bin"
170
171#====================================================
172#- SUBMIT_DIR : submission dir
173typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
174
175#====================================================
176#- IN
177typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
178
179#====================================================
180#- RUN_DIR_PATH : Temporary working directory (=> TMP)
181typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
182
183#====================================================
184#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
185typeset -r OUTCOMMAND_PATH=/tmp
186
187#====================================================
188#- HOST_MPIRUN_COMMAND
189typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
190
191#====================================================
192#- Max number of arguments passed to nco operator or demigration command
193UNIX_MAX_LIMIT=360
194
195#====================================================
196#- set PackDefault to true on irene
197PackDefault=true
198
199#====================================================
200#- Default number of MPI task for IPSL coupled model
201#- required for backward compatibility
202#-
203DEFAULT_NUM_PROC_OCE=5
204DEFAULT_NUM_PROC_CPL=1
205(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
206
207#====================================================
208#- Number of computing cores per node
209#-
210NB_CORE_PER_NODE=48
211
212#D-#==================================================
213#D-function IGCM_sys_defineArchives
214#D-* Purpose:
215#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
216#D-* Define ARCHIVE : Dedicated to large files
217#D-* Define STORAGE : Dedicated to small/medium files
218#D-* Define R_OUT   : Output tree located on ARCHIVE
219#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
220#D-* Define R_BUF   : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
221#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
222#D-* Examples:
223#D-
224function IGCM_sys_defineArchives {
225  IGCM_debug_PushStack "IGCM_sys_defineArchives"
226
227  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
228  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = XDEFAULT ]; then
229    # Default option: Change the dfldatadir according to the project used for submission
230    # The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
231    if [ X"$( echo ${PROJECT} | grep cmip6 )" != "X" ] ; then
232      module switch dfldatadir dfldatadir/gencmip6
233      IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
234    else
235      module switch dfldatadir dfldatadir/${PROJECT}
236      IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
237    fi
238  else
239    # Use the project set by the variable DataProject in section UserChoices in config.card
240    module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
241    IGCM_debug_Print 1 "Load dfldatadir for DataProject read in config.card : ${config_UserChoices_DataProject}"
242  fi
243
244 #====================================================
245 #- RUN_DIR_PATH : Temporary working directory (=> TMP)
246 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
247
248  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
249    #====================================================
250    #- ARCHIVE (dedicated to large files)
251    ARCHIVE=${config_UserChoices_ARCHIVE}
252  else
253    #====================================================
254    #- ARCHIVE (dedicated to large files)
255    ARCHIVE=${CCCSTOREDIR}
256  fi
257
258  if [ ! X${config_UserChoices_STORAGE} = X ]; then
259    #====================================================
260    #- STORAGE (dedicated to small/medium files)
261    STORAGE=${config_UserChoices_STORAGE}
262  else
263    #====================================================
264    #- STORAGE (dedicated to small/medium files)
265    STORAGE=${CCCWORKDIR}
266  fi
267
268  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
269    #====================================================
270    #- R_OUT
271    R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
272
273    #====================================================
274    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
275    R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
276
277    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
278
279  else
280    #====================================================
281    #- R_OUT
282    R_OUT=${ARCHIVE}/IGCM_OUT
283
284    #====================================================
285    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
286    R_FIG=${STORAGE}/IGCM_OUT
287  fi
288
289  #====================================================
290  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
291  CMIP6_BUF=${STORAGE}/IGCM_OUT
292
293  #====================================================
294  #- R_BUF (ONLY FOR double copy an scratch)
295  R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
296
297  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
298  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
299  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
300
301  IGCM_debug_PopStack "IGCM_sys_defineArchives"
302}
303
304#D-#==================================================
305#D-function IGCM_sys_RshArchive
306#D-* Purpose: Archive rsh command
307#D-* Examples:
308#D-
309function IGCM_sys_RshArchive {
310  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
311  /bin/ksh <<-EOF
312    ${@}
313EOF
314  status=$?
315  if [ ${status} -gt 0 ] ; then
316    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
317    IGCM_debug_Exit "IGCM_sys_RshArchive"
318  fi
319  IGCM_debug_PopStack "IGCM_sys_RshArchive"
320}
321
322#D-#==================================================
323#D-function IGCM_sys_RshArchive_NoError
324#D-* Purpose: Archive rsh command, without error
325#D-*          used only in monitoring.job
326#D-* Examples:
327#D-
328function IGCM_sys_RshArchive_NoError {
329  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
330  /bin/ksh <<-EOF
331    ${@} 2> /dev/null
332EOF
333  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
334}
335
336#D-#==================================================
337#D-function IGCM_sys_MkdirArchive
338#D-* Purpose: Mkdir on Archive
339#D-* Examples:
340#D-
341function IGCM_sys_MkdirArchive {
342  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
343  if ( $DEBUG_sys ) ; then
344    echo "IGCM_sys_MkdirArchive :" $@
345  fi
346  #- creation de repertoire sur le serveur fichier
347  if [ ! -d ${1} ]; then
348    \mkdir -p $1
349    status=$?
350
351    if [ ${status} -gt 0 ] ; then
352      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
353      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
354    fi
355  fi
356  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
357}
358
359#D-#==================================================
360#D-function IGCM_sys_TestDirArchive
361#D-* Purpose: Test Directory that must exists on Archive
362#D-* Examples:
363#D-
364function IGCM_sys_TestDirArchive {
365  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
366  if ( $DEBUG_sys ) ; then
367    echo "IGCM_sys_TestDirArchive :" $@
368  fi
369  typeset ExistFlag
370  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
371  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
372  return ${ExistFlag}
373}
374
375#D-#==================================================
376#D-function IGCM_sys_IsFileArchived
377#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
378#D-* Examples:
379#D-
380function IGCM_sys_IsFileArchived {
381  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
382  if ( $DEBUG_sys ) ; then
383    echo "IGCM_sys_IsFileArchived :" $@
384  fi
385  typeset IsArchivedFlag
386  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
387  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
388
389  return ${IsArchivedFlag}
390}
391
392#D-#==================================================
393#D-function IGCM_sys_TestFileArchive
394#D-* Purpose: Test file that must NOT EXISTS on Archive
395#D-* Examples:
396#D-
397function IGCM_sys_TestFileArchive {
398  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
399  typeset ExistFlag
400  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
401  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
402
403  return ${ExistFlag}
404}
405
406#D-#==================================================
407#D-function IGCM_sys_CountFileArchive
408#D-* Purpose: Count files on Archive filesystem
409#D-* Examples:
410#D-
411function IGCM_sys_CountFileArchive {
412  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
413  ls ${@} 2>/dev/null | wc -l
414  if [ $? -gt 0 ] ; then
415    echo "IGCM_sys_CountFileArchive : erreur."
416  fi
417  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
418}
419
420#D-#==================================================
421#D-function IGCM_sys_Tree
422#D-* Purpose: Tree directories with files on ${ARCHIVE}
423#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
424#D-
425function IGCM_sys_Tree {
426  IGCM_debug_PushStack "IGCM_sys_Tree" $@
427  if ( $DEBUG_sys ) ; then
428    echo "IGCM_sys_Tree :" $@
429  fi
430
431  \ls -lR ${@}
432
433  IGCM_debug_PopStack "IGCM_sys_Tree"
434}
435
436#D-#==================================================
437#D-function IGCM_sys_Qsub
438#D-* Purpose: Qsub new job
439#D-* Examples:
440#D-
441function IGCM_sys_Qsub {
442  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
443  if ( $DEBUG_sys ) ; then
444    echo "IGCM_sys_Qsub :" $@
445  fi
446  typeset options status
447  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
448
449  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
450  status=$?
451
452  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
453  if [ ${status} -gt 0 ] ; then
454    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
455    IGCM_debug_Exit "IGCM_sys_Qsub"
456  else
457    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
458  fi
459  IGCM_debug_PopStack "IGCM_sys_Qsub"
460}
461
462#D-#==================================================
463#D-function IGCM_sys_QsubPost
464#D-* Purpose: Qsub new job on scalaire
465#D-* Examples:
466#D-
467function IGCM_sys_QsubPost {
468  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
469  if ( $DEBUG_sys ) ; then
470    echo "IGCM_sys_QsubPost :" $@
471  fi
472  typeset options status
473  options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
474
475  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
476  status=$?
477
478  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
479  if [ ${status} -gt 0 ] ; then
480    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
481    IGCM_debug_Exit "IGCM_sys_QsubPost"
482  else
483    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
484  fi
485  IGCM_debug_PopStack "IGCM_sys_QsubPost"
486}
487
488#D-*************************
489#D- File transfer functions
490#D-*************************
491#D-
492
493#D-#==================================================
494#D-function IGCM_sys_RmRunDir
495#D-* Purpose: rm tmpdir (dummy function most of the time batch
496#D-                      scheduler will do the job)
497#D-* Examples:
498#D-
499function IGCM_sys_RmRunDir {
500  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
501  if ( $DEBUG_sys ) ; then
502    echo "IGCM_sys_RmRunDir :" $@
503  fi
504
505  typeset status
506
507  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
508  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
509  status=$?
510
511  if [ ${status} -gt 0 ] ; then
512    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
513    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
514    IGCM_debug_Exit "IGCM_sys_RmRunDir"
515  else
516    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
517  fi
518  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
519}
520
521#D-#==================================================
522#D-function IGCM_sys_Put_Dir
523#D-* Purpose: Copy a complete directory on $(ARCHIVE)
524#D-* Examples:
525#D-
526function IGCM_sys_Put_Dir {
527  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
528  if ( $DEBUG_sys ) ; then
529    echo "IGCM_sys_Put_Dir :" $@
530  fi
531  if [ $DRYRUN = 0 ]; then
532    if [ ! -d ${1} ] ; then
533      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
534      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
535      return
536    fi
537
538    typeset status
539
540    # Only if we use rsync
541    #IGCM_sys_TestDirArchive $( dirname $2 )
542    #
543    #USUAL WAY
544    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
545    status=$?
546
547    if [ ${status} -gt 0 ] ; then
548      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
549      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
550      IGCM_debug_Exit "IGCM_sys_Put_Dir"
551    else
552      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
553    fi
554  fi
555  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
556}
557
558#D-#==================================================
559#D-function IGCM_sys_Get_Dir
560#D-* Purpose: Copy a complete directory from ${ARCHIVE}
561#D-* Examples:
562#D-
563function IGCM_sys_Get_Dir {
564  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
565  if ( $DEBUG_sys ) ; then
566    echo "IGCM_sys_Get_Dir :" $@
567  fi
568  if [ $DRYRUN = 0 ]; then
569    typeset NB_ESSAI DELAI status i
570    # number of tentative
571    NB_ESSAI=3
572    # time delay between tentative
573    DELAI=2
574
575    #
576    # USUAL WAY
577    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
578    ccc_hsm get -r $1
579
580    i=0
581    while [ $i -lt $NB_ESSAI ] ; do
582      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
583      status=$?
584      if [ ${status} -gt 0 ] ; then
585        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
586        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
587        sleep $DELAI
588      else
589        break
590      fi
591      (( i = i + 1 ))
592    done
593
594    if [ ${status} -gt 0 ] ; then
595      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
596      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
597      IGCM_debug_Exit "IGCM_sys_Get_Dir"
598    else
599      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
600    fi
601  fi
602  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
603}
604
605#D-#==================================================
606#D-function IGCM_sys_Put_Rest
607#D-* Purpose: Put computied restarts on ${ARCHIVE}.
608#D-           File and target directory must exist.
609#D-* Examples:
610#D-
611function IGCM_sys_Put_Rest {
612  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
613  if ( $DEBUG_sys ) ; then
614    echo "IGCM_sys_Put_Rest :" $@
615  fi
616  if [ $DRYRUN = 0 ]; then
617    if [ ! -f ${1} ] ; then
618      # The file is missing       
619      if [ X${3} = X'OPTIONAL' ] ; then
620          echo "WARINING : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
621          IGCM_debug_Print 1 "IGCM_sys_Put_Rest: Optional restart file ${1} is missing, now continue for the next"
622      else
623          echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
624          IGCM_debug_Exit "IGCM_sys_Put_Rest"
625      fi
626     
627    else
628      # The file exist
629
630    typeset status
631    #
632    # USUAL WAY
633    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
634    status=$?
635
636#       #RSYNC WITH NETWORK SSH CALL
637#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
638#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
639
640#       #RSYNC WITH NFS USE
641#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
642#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
643
644#       status=$?
645#       IGCM_sys_Rsync_out $status
646
647#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
648#       (( status=status+$? ))
649
650    if [ ${status} -gt 0 ] ; then
651      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
652      [ -f ${1} ] && ls -l ${1}
653      [ -f ${2} ] && ls -l ${2}
654      [ -f ${2}/${1} ] && ls -l ${2}/${1}
655      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
656      IGCM_debug_Exit "IGCM_sys_Put_Rest"
657    else
658
659      if [ X${JobType} = XRUN ] ; then
660        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
661        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
662      fi
663
664      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
665    fi
666   
667    fi
668   
669  fi
670  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
671}
672
673#D-#==================================================
674#D-function IGCM_sys_Put_Out
675#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
676#D-* Examples:
677#D-
678function IGCM_sys_Put_Out {
679  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
680  if ( $DEBUG_sys ) ; then
681    echo "IGCM_sys_Put_Out :" $@
682  fi
683
684  typeset NB_ESSAI DELAI status i exist skip
685  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
686
687  # number of tentative
688  NB_ESSAI=3
689  # time delay between tentative
690  DELAI=2
691
692  if [ $DRYRUN = 0 ]; then
693    if [ ! -f ${1} ] ; then
694      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
695      IGCM_debug_PopStack "IGCM_sys_Put_Out"
696      return 1
697    fi
698    #
699    IGCM_sys_MkdirArchive $( dirname $2 )
700    #
701    exist=false
702    skip=false
703    if [ -f $2 ] ; then
704      IGCM_debug_Print 1 "$2 already exist"
705      ccc_hsm get $2
706      exist=true
707      if [ "X$( diff $1 $2 )" = X ] ; then
708        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
709        skip=true
710      else
711        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
712        skip=false
713      fi
714    fi
715    #
716    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
717      IGCM_sys_Chmod u+w $2
718    fi
719
720    if [ X${skip} = Xfalse ] ; then
721      i=0
722      while [ $i -lt $NB_ESSAI ] ; do
723        # Identify file system
724        fileDeviceNumberInHex=$( stat -c %d $1 )
725        status=$?
726        if [ ${status} -gt 0 ] ; then
727          IGCM_debug_Exit "IGCM_sys_Put_Out"
728        fi
729        # Identify file system
730        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
731        status=$?
732        if [ ${status} -gt 0 ] ; then
733          IGCM_debug_Exit "IGCM_sys_Put_Out"
734        fi
735
736        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
737          # They are not on the same device. USUAL WAY
738          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
739          status=$?
740        else
741          # They are on the same device. NOT SO USUAL WAY
742          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
743          status=$?
744        fi
745        if [ ${status} -gt 0 ]; then
746          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
747          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
748          [ -f ${1} ] && ls -l ${1}
749          [ -f ${2} ] && ls -l ${2}
750          [ -f ${2}/${1} ] && ls -l ${2}/${1}
751          sleep $DELAI
752        else
753          break
754        fi
755        (( i = i + 1 ))
756      done
757    fi
758
759#       #RSYNC WITH NETWORK SSH CALL
760#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
761#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
762
763#       #RSYNC WITH NFS USE
764#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
765#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
766
767#       status=$?
768#       IGCM_sys_Rsync_out $status
769
770#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
771#       (( status=status+$? ))
772
773    if [ ${status} -gt 0 ] ; then
774      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
775      [ -f ${1} ] && ls -l ${1}
776      [ -f ${2} ] && ls -l ${2}
777      [ -f ${2}/${1} ] && ls -l ${2}/${1}
778      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
779      IGCM_debug_Exit "IGCM_sys_Put_Out"
780    else
781
782      if [ X${JobType} = XRUN ] ; then
783        if [ X${3} = X ] ; then
784          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
785          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
786        fi
787      fi
788
789      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
790    fi
791  fi
792  IGCM_debug_PopStack "IGCM_sys_Put_Out"
793  return 0
794}
795
796#D-#==================================================
797#D-function IGCM_sys_Get
798#D-* Purpose: Get a file from ${ARCHIVE}
799#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
800#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
801function IGCM_sys_Get {
802  IGCM_debug_PushStack "IGCM_sys_Get" $@
803
804  typeset DEST dm_liste target file_work
805  typeset NB_ESSAI DELAI status i
806
807  if ( $DEBUG_sys ) ; then
808    echo "IGCM_sys_Get :" $@
809  fi
810
811  # number of tentative
812  NB_ESSAI=3
813  # time delay between tentative
814  DELAI=2
815
816  if [ $DRYRUN -le 2 ]; then
817    if [ X${1} = X'/l' ] ; then
818      eval set +A dm_liste \${${2}}
819    else
820      eval set +A dm_liste ${1}
821    fi
822    eval DEST=\${${#}}
823    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
824    status=$?
825    if [ ${status} -gt 0 ] ; then
826      echo "WARNING IGCM_sys_Get : error code ${status}"
827      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
828      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
829    fi
830
831    #if [ ${status} -gt 0 ] ; then
832    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
833    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
834    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
835    #    sleep 30
836    #    echo "We try another time"
837    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
838    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
839    #    status=$?
840    #    if [ ${status} -gt 0 ] ; then
841    #      echo "ERROR IGCM_sys_Get : again demigration error :"
842    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
843    #      IGCM_debug_Exit "IGCM_sys_Get"
844    #    fi
845    #  else
846    #    echo "ERROR IGCM_sys_Get : demigration error :"
847    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
848    #    IGCM_debug_Exit "IGCM_sys_Get"
849    #  fi
850    #fi
851
852    #   #RSYNC WITH NETWORK SSH CALL
853    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
854    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
855
856    #   #RSYNC WITH NFS USE
857    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
858    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
859
860    #   status=$?
861    #   IGCM_sys_Rsync_out $status
862
863    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
864    #   (( status=status+$? ))
865
866    #USUAL WAY
867    if [ X${1} = X'/l' ] ; then
868      for target in ${dm_liste[*]} ; do
869        local_file=$( basename ${target} )
870        # test if the target file is present before the loop
871        IGCM_sys_TestFileArchive ${target}
872        status=$?
873        if [ ${status} -gt 0 ] ; then
874          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
875          IGCM_debug_Exit "IGCM_sys_Get"
876        else
877          i=0
878          while [ $i -lt $NB_ESSAI ] ; do
879            #if [ X${DoLink} = Xtrue ] ; then
880            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
881            #  status=$?
882            #  else
883            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
884            #  status=$?
885            #fi
886            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
887            status=$?
888            if [ ${status} -gt 0 ]; then
889              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
890              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
891              sleep $DELAI
892            else
893              break
894            fi
895            (( i = i + 1 ))
896          done
897          if [ ${status} -gt 0 ] ; then
898            echo "IGCM_sys_Get : error"
899            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
900            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
901            IGCM_debug_Exit "IGCM_sys_Get"
902          else
903            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
904          fi
905        fi
906      done
907    else
908      i=0
909      while [ $i -lt $NB_ESSAI ] ; do
910        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
911        status=$?
912        if [ ${status} -gt 0 ]; then
913          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
914          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
915          sleep $DELAI
916        else
917          break
918        fi
919        (( i = i + 1 ))
920      done
921      if [ ${status} -gt 0 ] ; then
922        echo "IGCM_sys_Get : error"
923        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
924        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
925        IGCM_debug_Exit "IGCM_sys_Get"
926      else
927        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
928      fi
929    fi
930  fi
931  IGCM_debug_PopStack "IGCM_sys_Get"
932}
933
934#D-#==================================================
935#D-function IGCM_sys_GetDate_Monitoring
936#D-* Purpose: get the last year for which the monitoring has been computed
937#D-* Examples:
938#D-
939function IGCM_sys_GetDate_Monitoring {
940  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
941  if ( $DEBUG_sys ) ; then
942    echo "IGCM_sys_GetDate_Monitoring :" $@
943  fi
944
945  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
946
947  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
948}
949
950#D-#==================================================
951#D-function IGCM_sys_Dods_Rm
952#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
953#D-* Examples:
954#D-
955function IGCM_sys_Dods_Rm {
956  if ( $DEBUG_sys ) ; then
957    echo "IGCM_sys_Dods_Rm :" $@
958  fi
959  typeset status
960  if [ $DRYRUN = 0 ]; then
961
962#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
963#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
964#      echo "Nothing has been done."
965#      return
966#    fi
967
968    if [ "$#" -eq 1 ]; then
969      ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
970      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
971      status=$?
972    else
973      ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
974      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
975      status=$?
976    fi
977
978#    if [ ${status} -gt 0 ] ; then
979#      echo "IGCM_sys_Dods_Rm : error."
980#      cat out_dods_rm
981#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
982#    else
983#      rm out_dods_rm
984#    fi
985
986  fi
987  return $status
988}
989
990#D-#==================================================
991#D-function IGCM_sys_Dods_Cp
992#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
993#D-* Examples:
994#D-
995function IGCM_sys_Dods_Cp {
996  if ( $DEBUG_sys ) ; then
997    echo "IGCM_sys_Dods_Cp :" $@
998  fi
999  typeset status
1000  if [ $DRYRUN = 0 ]; then
1001
1002#    if [ ! -d ${R_SAVE}/${1} ] ; then
1003#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1004#      echo "Nothing has been done."
1005#      return
1006#    fi
1007
1008    ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/bin/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1009    $(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1010    status=$?
1011
1012#       if [ ${status} -gt 0 ] ; then
1013#           echo "IGCM_sys_Dods_Cp : error."
1014#           cat out_dods_cp
1015#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1016#       else
1017#           rm out_dods_cp
1018#       fi
1019
1020  fi
1021  return $status
1022}
1023
1024#D-#==================================================
1025#D-function IGCM_sys_Put_Dods
1026#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1027#D-* Examples:
1028#D-
1029function IGCM_sys_Put_Dods {
1030  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1031  if ( $DEBUG_sys ) ; then
1032    echo "IGCM_sys_Put_Dods :" $@
1033  fi
1034  typeset status
1035  if [ $DRYRUN = 0 ]; then
1036    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1037      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1038      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1039      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1040      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1041      return
1042    fi
1043
1044    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1045      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1046      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1047      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1048      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1049      return
1050    fi
1051    #
1052    if [ -d ${R_SAVE}/${1} ] ; then
1053      cd ${R_SAVE}
1054    elif [ -d ${R_FIGR}/${1} ] ; then
1055      cd ${R_FIGR}
1056    fi
1057
1058    IGCM_sys_Dods_Rm ${1}
1059    IGCM_sys_Dods_Cp ${1}
1060    status=0
1061
1062    if [ ${status} -gt 0 ] ; then
1063      echo "IGCM_sys_Put_Dods : error."
1064      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1065    fi
1066  fi
1067  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1068}
1069
1070##############################################################
1071# REBUILD OPERATOR
1072
1073#D-#==================================================
1074#D-function IGCM_sys_sync
1075#D-* Purpose: flush buffer on disk
1076#D-* Examples:
1077#D-
1078function IGCM_sys_sync {
1079  IGCM_debug_PushStack "IGCM_sys_sync" $@
1080  if ( $DEBUG_sys ) ; then
1081    echo "IGCM_sys_sync :" $@
1082  fi
1083
1084  /bin/sync
1085
1086  IGCM_debug_PopStack "IGCM_sys_sync"
1087}
1088
1089############################################################
1090# Activate Running Environnment Variables
1091
1092#D-#==================================================
1093#D-function IGCM_sys_activ_variables
1094#D-* Purpose: set environement variables prior to execution
1095#D-* Examples:
1096#D-
1097function IGCM_sys_activ_variables {
1098  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1099  if ( $DEBUG_sys ) ; then
1100    echo "IGCM_sys_activ_variables"
1101  fi
1102
1103# --------------------------------------------------------------------
1104#D- MODULE specifications
1105# --------------------------------------------------------------------
1106  # Source the file EnvFile containing all module load needed to run the model.
1107  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1108  # it will be used. Otherwise a default file will be used.
1109  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1110      # EnvFile is set in config.card and the file exists
1111      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1112      EnvFile=${config_UserChoices_EnvFile}
1113  else
1114      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1115      module purge > /dev/null 2>&1
1116      if [ X${compilerVersion} = XV17 ] ; then
1117          IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V17"
1118          ## EnvFile=/ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_irene > /dev/null 2>&1
1119          EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene/env_irene > /dev/null 2>&1
1120      else
1121          IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1122      fi
1123  fi
1124  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1125  IGCM_debug_Print 1 ${EnvFile}
1126  . ${EnvFile}
1127
1128  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene. "
1129  module list
1130
1131# --------------------------------------------------------------------
1132#D- MPI specifications
1133# --------------------------------------------------------------------
1134
1135# --------------------------------------------------------------------
1136#D- Other specifications
1137# --------------------------------------------------------------------
1138
1139  ulimit -s unlimited
1140
1141  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1142}
1143
1144############################################################
1145# Desactivate Running Environnment Variables
1146
1147#D-#==================================================
1148#D-function IGCM_sys_desactiv_variables
1149#D-* Purpose: unset environement variables after execution
1150#D-* Examples:
1151#D-
1152function IGCM_sys_desactiv_variables {
1153  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1154  if ( $DEBUG_sys ) ; then
1155    echo "IGCM_sys_desactiv_variables"
1156  fi
1157# --------------------------------------------------------------------
1158#D- MPI specifications
1159# --------------------------------------------------------------------
1160
1161# --------------------------------------------------------------------
1162#D- Other specifications
1163# --------------------------------------------------------------------
1164
1165  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1166}
1167
1168############################################################
1169# Update job headers to be used by the scheduler
1170
1171#D-#==================================================
1172#D-function IGCM_sys_updateHeaders
1173#D-* Purpose: Update job headers to be used by the scheduler
1174#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1175#D-
1176function IGCM_sys_updateHeaders {
1177  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1178  if ( $DEBUG_sys ) ; then
1179    echo "IGCM_sys_updateHeaders"
1180  fi
1181  typeset file
1182  file=$1
1183
1184  if [ ${executionType} -eq 1 ] ; then
1185    # MPMD + MPI
1186    sed -e "/::openMPthreads::/d"                  \
1187        -e "s/::JobNumProcTot::/${coreNumber}/"    \
1188        -e "/--cpu_bind=none/d"                    \
1189      ${file} > ${file}.tmp
1190
1191  elif [ ${executionType} -eq 2 ] ; then
1192    # MPMD + MPI + OMP : mpirun/ccc_mprun/error
1193    if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1194      sed -e "/::openMPthreads::/d"                  \
1195          -e "s/::JobNumProcTot::/${coreNumber}/"    \
1196        ${file} > ${file}.tmp
1197    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1198      sed -e "/::openMPthreads::/d"                  \
1199          -e "s/::JobNumProcTot::/${coreNumber}/"    \
1200          -e "/--cpu_bind=none/d"                    \
1201        ${file} > ${file}.tmp
1202    else
1203     IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
1204     IGCM_debug_Exit "IGCM_sys_updateHeaders"
1205    fi
1206  elif [ ${executionType} -eq 3 ] ; then
1207    # SPMD + MPI/OMP
1208    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1209        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1210        -e "/#MSUB -x/d"                           \
1211        -e "/--cpu_bind=none/d"                    \
1212      ${file} > ${file}.tmp
1213
1214  elif [ ${executionType} -eq 4 ] ; then
1215    # SPMD + MPI only
1216    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1217        -e "/::openMPthreads::/d"                  \
1218        -e "/#MSUB -x/d"                           \
1219        -e "/--cpu_bind=none/d"                    \
1220      ${file} > ${file}.tmp
1221
1222  elif [ ${executionType} -eq 5 ] ; then
1223    # SPMD + OMP only
1224    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1225        -e "/::JobNumProcTot::/d"                  \
1226        -e "/#MSUB -x/d"                           \
1227        -e "/--cpu_bind=none/d"                    \
1228      ${file} > ${file}.tmp
1229
1230  elif [ ${executionType} -eq 6 ] ; then
1231    # SEQUENTIAL THEN
1232    sed -e "s/::JobNumProcTot::/1/"                \
1233        -e "/::openMPthreads::/d"                  \
1234        -e "/#MSUB -x/d"                           \
1235        -e "/--cpu_bind=none/d"                    \
1236      ${file} > ${file}.tmp
1237
1238  fi
1239
1240  IGCM_sys_Mv ${file}.tmp ${file}
1241
1242  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1243}
1244
1245############################################################
1246# Build MPI/OMP scripts run file (dummy function)
1247
1248#D-#==================================================
1249#D-function IGCM_sys_build_run_file
1250#D-* Purpose: build run file (deprecated)
1251#D-* Examples:
1252#D-
1253function IGCM_sys_build_run_file {
1254  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1255}
1256
1257############################################################
1258# Build MPI/OMP scripts
1259
1260#D-#==================================================
1261#D-function IGCM_sys_build_execution_scripts
1262#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1263#D-* Examples:
1264#D-
1265function IGCM_sys_build_execution_scripts
1266{
1267  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1268  if ( $DEBUG_sys ) ; then
1269    echo "IGCM_sys_build_execution_scripts " $@
1270  fi
1271
1272  EXECUTION=${HOST_MPIRUN_COMMAND}
1273
1274  if ( ${OK_PARA_MPMD} ) ; then
1275
1276    # MPMD mode
1277    # 1 MPI only
1278    # 2 MPI/OpenMP mpirun method
1279    # 3 MPI/OpenMP ccc_mprun method
1280
1281    if [ -f run_file ] ; then
1282      IGCM_sys_Rm -f run_file
1283    fi
1284    touch run_file
1285
1286    # case 1 : Only MPI (MPMD)
1287    if  ( ! ${OK_PARA_OMP} ) ; then
1288
1289      # Build run_file
1290
1291      # First loop on the components for the coupler ie oasis (only if oasis3)
1292      # the coupler ie oasis3 must be the first one
1293      for comp in ${config_ListOfComponents[*]} ; do
1294
1295        eval ExeNameIn=\${config_Executable_${comp}[0]}
1296        eval ExeNameOut=\${config_Executable_${comp}[1]}
1297
1298        # for CPL component only
1299        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1300
1301          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1302          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1303          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1304        fi
1305      done
1306
1307      # Then second loop on the components
1308      for comp in ${config_ListOfComponents[*]} ; do
1309
1310        eval ExeNameIn=\${config_Executable_${comp}[0]}
1311        eval ExeNameOut=\${config_Executable_${comp}[1]}
1312
1313        # Only if we really have an executable for the component and not the coupler ie oasis:
1314        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1315
1316          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1317          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1318          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1319        fi
1320      done
1321
1322      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1323
1324      IGCM_sys_Chmod u+x run_file
1325      if ( $DEBUG_sys ) ; then
1326        echo "run_file contains : "
1327        cat run_file
1328      fi
1329
1330    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1331
1332    # 2 MPI/OpenMP mpirun method
1333
1334      # Use of mpirun instead of ccc_mprun
1335      EXECUTION="time mpirun"
1336
1337      #  Hosts treatment
1338      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1339
1340      i=0
1341      rm -f hosts rankfile
1342      IGCM_debug_Print 1 "sys Irene, Hosts available :"
1343      for nodes in `cat hosts.tmp`
1344      do
1345        host[$i]=$nodes
1346        echo "${host[$i]}" >> hosts
1347        IGCM_debug_Print 1 ${host[$i]}
1348        i=$((i+1))
1349      done
1350      rm -f hosts.tmp
1351
1352      listnodes=${host[*]}
1353
1354      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1355
1356      # Initialisation
1357      rank=0
1358      current_core=0
1359      init_exec=n
1360
1361      # Loop on the components
1362      for comp in ${config_ListOfComponents[*]} ; do
1363
1364        eval ExeNameIn=\${config_Executable_${comp}[0]}
1365        eval ExeNameOut=\${config_Executable_${comp}[1]}
1366
1367        # Not possible if oasis has an executable (i.e old version of oasis3)
1368        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1369          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1370          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1371          IGCM_debug_Verif_Exit
1372        fi
1373
1374        # Only if we really have an executable for the component :
1375        if [ "X${ExeNameOut}" != X\"\" ] ; then
1376
1377          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1378          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1379          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1380
1381          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1382          echo ""  >> script_${ExeNameOut}.ksh
1383          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1384
1385            # Check if the number of threads is correct
1386            case ${comp_proc_omp_loc} in
1387            2|4|8|16)
1388              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1389              ;;
1390            *)
1391              IGCM_debug_Exit "ERROR with OMP parameters !"
1392              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1393              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1394              IGCM_debug_Verif_Exit
1395              ;;
1396            esac
1397            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1398            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1399            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1400            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1401          fi
1402
1403          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1404          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1405          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1406          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1407          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1408
1409          if [ ${init_exec} = y ] ; then
1410            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1411          else
1412            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1413            init_exec=y
1414          fi
1415
1416          # Build rankfile : method used to assign cores and nodes for the MPI process
1417          # Ex :
1418          #rank 0=curie5296 slot=0,1,2,3
1419          #rank 1=curie5296 slot=4,5,6,7
1420          # Example of final command :
1421          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1422          # with script_lmdz.x.ksh :
1423          # #!/bin/ksh
1424          #export KMP_STACKSIZE=3g
1425          #export KMP_LIBRARY=turnaround
1426          #export MKL_SERIAL=YES
1427          #OMP_NUM_THREADS=4
1428          #./lmdz.x
1429          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1430              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1431          else
1432              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1433          fi
1434          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1435             (( index_host = current_core / NB_CORE_PER_NODE ))
1436             host_value=${host[${index_host}]}
1437             (( slot =  current_core % NB_CORE_PER_NODE ))
1438             virg=","
1439             string_final=""
1440             for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1441                 string=$index$virg
1442                 string_final=$string_final$string
1443             done
1444             string_final=$( echo $string_final | sed "s/.$//" )
1445             echo "rank $rank=$host_value slot=$string_final" >> rankfile
1446             (( rank = rank + 1 ))
1447             (( current_core = current_core + offset_comp_proc_loc ))
1448          done
1449        fi
1450      done
1451
1452    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1453
1454    # 3 MPI/OpenMP ccc_mprun method
1455
1456    # MPI-OpenMP (MPMD)
1457    # example of run_file
1458    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1459    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1460    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1461
1462      # Loop on the components
1463      for comp in ${config_ListOfComponents[*]} ; do
1464
1465        eval ExeNameIn=\${config_Executable_${comp}[0]}
1466        eval ExeNameOut=\${config_Executable_${comp}[1]}
1467
1468        # Not possible if oasis has an executable (i.e old version of oasis3)
1469        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1470          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1471          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1472          IGCM_debug_Verif_Exit
1473        fi
1474
1475        # Only if we really have an executable for the component :
1476        if [ "X${ExeNameOut}" != X\"\" ] ; then
1477
1478          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1479          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1480
1481          # Build script files
1482
1483          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1484          echo ""  >> script_${ExeNameOut}.ksh
1485          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1486
1487            # Check if the number of threads is correct
1488            case ${comp_proc_omp_loc} in
1489            2|4|8|12|16|24)
1490              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1491              ;;
1492            *)
1493              IGCM_debug_Exit "ERROR with OMP parameters !"
1494              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1495              IGCM_debug_Print 2 "Only 2,4,8,12,16,24 as number of OMP threads are possible "
1496              IGCM_debug_Verif_Exit
1497              ;;
1498            esac
1499            ### only for LMDZ?
1500            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1501            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1502            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1503            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1504
1505          fi
1506
1507          # to have out/err per process on different files
1508          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1509          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1510
1511          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1512
1513          # Complete run_file
1514
1515          echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1516
1517        fi
1518      done
1519
1520      ## variable added to stop after 60s instead of 600s by default.
1521      ## This is used when no error comes from executables and when something stopped an executable without notice.
1522      export SLURM_WAIT=60
1523
1524      ## module advised by TGCC (instead of 2 variables)
1525      module load feature/bridge/heterogenous_mpmd
1526
1527      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1528
1529      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1530
1531      IGCM_sys_Chmod u+x run_file
1532      if ( $DEBUG_sys ) ; then
1533        echo "run_file contains : "
1534        cat run_file
1535      fi
1536
1537    else
1538
1539      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1540
1541    fi # 1 2 or 3 if ${OK_PARA_MPMD}
1542
1543  else
1544  # Only one executable (SPMD mode).
1545
1546    for comp in ${config_ListOfComponents[*]} ; do
1547
1548      # Only if we really have an executable for the component :
1549      eval ExeNameOut=\${config_Executable_${comp}[1]}
1550      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1551
1552        # Build script files
1553
1554        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1555        echo ""  >> script_${ExeNameOut}.ksh
1556        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1557
1558        if ( ${OK_PARA_OMP} ) ; then
1559          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1560          echo ""  >> script_${ExeNameOut}.ksh
1561          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1562          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1563          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1564          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1565        fi
1566
1567        if  ( ${OK_PARA_MPI} ) ; then
1568          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1569          # Default : ccc_mprun used if nb_proc gt 1
1570          # to have out/err per process on different files
1571          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1572          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1573          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1574        else
1575          # Default : ccc_mprun is NOT used if nb_proc eq 1
1576          # to have out/err per process on different files
1577          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1578          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1579          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1580        fi
1581
1582        IGCM_debug_Print 1 "sys Irene : script_${ExeNameOut}.ksh contains"
1583        cat script_${ExeNameOut}.ksh
1584
1585      fi
1586
1587    done
1588
1589  fi # ${OK_PARA_MPMD}
1590
1591  IGCM_debug_Print 1 "sys Irene : execution command is "
1592  IGCM_debug_Print 1 "$EXECUTION"
1593
1594  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1595}
1596
1597#D-#==================================================
1598#D-function IGCM_sys_check_path
1599#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1600#D-* do not point to an important use directory. Stop immediately in that case.
1601#D-* Examples:
1602#D-
1603function IGCM_sys_check_path {
1604  IGCM_debug_PushStack "IGCM_sys_check_path"
1605  if ( $DEBUG_sys ) ; then
1606    echo "IGCM_sys_check_path"
1607  fi
1608
1609  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1610    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1611    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1612    IGCM_debug_Exit "This will stop the job"
1613  fi
1614  IGCM_debug_PopStack "IGCM_sys_check_path"
1615}
1616
1617#D-#==================================================
1618#D-function IGCM_sys_check_quota
1619#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1620#D-* Examples:
1621#D-
1622function IGCM_sys_check_quota {
1623  IGCM_debug_PushStack "IGCM_sys_check_quota"
1624  if ( $DEBUG_sys ) ; then
1625    echo "IGCM_sys_check_quota"
1626  fi
1627  # Limit of quota (in %)
1628  limit_quota=90
1629
1630  # Check of the volume
1631  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1632  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1633
1634  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1635
1636    unit_avail=${volume_avail: -1}
1637    unit_quota=${volume_quota: -1}
1638
1639    if [ "${unit_quota}" = "*" ] ; then
1640      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1641      IGCM_debug_Print 1 "More than 100% of your quota is used"
1642      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1643      IGCM_debug_Print 1 "You must have more than 10% available to run"
1644      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1645      IGCM_debug_Verif_Exit
1646    fi
1647
1648    temp_avail=${volume_avail%%${volume_avail: -1}*}
1649    temp_quota=${volume_quota%%${volume_quota: -1}*}
1650
1651    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1652
1653    # Convertion
1654      if [ ${volume_avail: -1} = "T" ] ; then
1655        (( temp_avail = temp_avail * 1000000000000 ))
1656      elif [ ${volume_avail: -1} = "G" ] ; then
1657        (( temp_avail = temp_avail * 1000000000 ))
1658      elif [ ${volume_avail: -1} = "M" ] ; then
1659        (( temp_avail = temp_avail * 1000000 ))
1660      elif [ ${volume_avail: -1} = "k" ] ; then
1661        (( temp_avail = temp_avail * 1000 ))
1662      else
1663        (( temp_avail = volume_avail ))
1664      fi
1665      if [ ${volume_quota: -1} = "T" ] ; then
1666        (( temp_quota = temp_quota * 1000000000000 ))
1667      elif [ ${volume_quota: -1} = "G" ] ; then
1668        (( temp_quota = temp_quota * 1000000000 ))
1669      elif [ ${volume_quota: -1} = "M" ] ; then
1670        (( temp_quota = temp_quota * 1000000 ))
1671      elif [ ${volume_quota: -1} = "k" ] ; then
1672        (( temp_quota = temp_quota * 1000 ))
1673      else
1674        (( temp_quota = volume_quota ))
1675      fi
1676    fi
1677
1678    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1679#    echo "volume ratio is " $quota_volume
1680
1681    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1682      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1683      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1684      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1685      IGCM_debug_Print 1 "You must have more than 10% available to run"
1686      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1687      IGCM_debug_Verif_Exit
1688    fi
1689
1690  fi
1691
1692# Check of the number of inodes
1693
1694  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1695  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1696
1697  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1698
1699    unit_avail=${inode_avail: -1}
1700    unit_quota=${inode_quota: -1}
1701
1702    if [ "${unit_quota}" = "*" ] ; then
1703      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1704      IGCM_debug_Print 1 "More than 100% of your quota is used"
1705      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1706      IGCM_debug_Print 1 "You must have more than 10% available to run"
1707      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1708      IGCM_debug_Verif_Exit
1709    fi
1710
1711    temp_avail=${inode_avail%%${inode_avail: -1}*}
1712    temp_quota=${inode_quota%%${inode_quota: -1}*}
1713
1714    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1715
1716    # Convertion
1717      if [ ${inode_avail: -1} = "T" ] ; then
1718        (( temp_avail = temp_avail * 1000000000000 ))
1719      elif [ ${inode_avail: -1} = "G" ] ; then
1720        (( temp_avail = temp_avail * 1000000000 ))
1721      elif [ ${inode_avail: -1} = "M" ] ; then
1722        (( temp_avail = temp_avail * 1000000 ))
1723      elif [ ${inode_avail: -1} = "k" ] ; then
1724        (( temp_avail = temp_avail * 1000 ))
1725      else
1726        (( temp_avail = inode_avail ))
1727      fi
1728
1729      if [ ${inode_quota: -1} = "T" ] ; then
1730        (( temp_quota = temp_quota * 1000000000000 ))
1731      elif [ ${inode_quota: -1} = "G" ] ; then
1732        (( temp_quota = temp_quota * 1000000000 ))
1733      elif [ ${inode_quota: -1} = "M" ] ; then
1734        (( temp_quota = temp_quota * 1000000 ))
1735      elif [ ${inode_quota: -1} = "k" ] ; then
1736        (( temp_quota = temp_quota * 1000 ))
1737      else
1738        (( temp_quota = inode_quota ))
1739      fi
1740    fi
1741    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1742#    echo "inode ratio is " $quota_inode
1743
1744    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1745      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1746      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1747      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1748      IGCM_debug_Print 1 "You must have more than 10% available to run"
1749      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1750      IGCM_debug_Verif_Exit
1751    fi
1752  fi
1753  IGCM_debug_PopStack "IGCM_sys_check_quota"
1754}
1755
1756#D-#==================================================
1757#D-function IGCM_sys_projectAccounting
1758#D-* Purpose: store project accounting information in a file
1759#D-* Examples:
1760#D-
1761function IGCM_sys_projectAccounting {
1762  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1763  if ( $DEBUG_sys ) ; then
1764    echo "IGCM_sys_check_quota"
1765  fi
1766
1767  ssh irene191 /usr/bin/ccc_myproject > $1
1768
1769  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1770}
1771
1772#D-#==================================================
1773#D-function IGCM_sys_getJobSchedulerID
1774#D-* Purpose: Get the job ID during execution
1775#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1776#D-
1777function IGCM_sys_getJobSchedulerID {
1778  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1779  if ( $DEBUG_sys ) ; then
1780    echo "IGCM_sys_getJobSchedulerID"
1781  fi
1782
1783  eval ${1}=${BRIDGE_MSUB_JOBID}
1784
1785  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1786}
1787
1788#D-#==================================================
1789#D-function IGCM_sys_GetJobID
1790#D-* Purpose: Get the job ID from the JobName
1791#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1792#D-
1793function IGCM_sys_GetJobID {
1794  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1795  if ( $DEBUG_sys ) ; then
1796    echo "IGCM_sys_GetJobID"
1797  fi
1798
1799  # With -f option, the full job name is given in the last column
1800  ID=$( ccc_mstat -f -u $2 | \
1801        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1802
1803  eval ${3}=${ID}
1804  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1805}
1806
1807#D-#==================================================
1808#D-function IGCM_sys_CountJobInQueue
1809#D-* Purpose: Count number of users job
1810#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1811#D-
1812function IGCM_sys_CountJobInQueue {
1813  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1814  if ( $DEBUG_sys ) ; then
1815    echo "IGCM_sys_CountJobInQueue"
1816  fi
1817
1818  # With -f option, the full job name is given in the last column
1819  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1820
1821  eval ${2}=${NbRun}
1822
1823  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1824}
1825
1826#D-#==================================================
1827#D-function IGCM_sys_ListJobInQueue
1828#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1829#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1830#D-
1831function IGCM_sys_ListJobInQueue {
1832  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1833  if ( $DEBUG_sys ) ; then
1834    echo "IGCM_sys_ListJobInQueue"
1835  fi
1836
1837  # With -f option, the full job name is given in the last column
1838  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1839                                        '( $2  == User      && \
1840                                           $NF != /TS/      && \
1841                                           $NF !~ /PACK/    && \
1842                                           $NF !~ /REBUILD/ && \
1843                                           $NF !~ /pack/ )     \
1844                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1845
1846  eval set -A ${2} ${JobList[*]}
1847
1848  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1849}
1850
1851#D-#==================================================
1852#D-function IGCM_sys_atlas
1853#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1854#D-* Examples:
1855#D-
1856function IGCM_sys_atlas {
1857  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1858  if ( $DEBUG_sys ) ; then
1859    echo "IGCM_sys_atlas :" $@
1860  fi
1861
1862  typeset status
1863
1864  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1865  status=$?
1866  if [ ${status} -gt 0 ] ; then
1867    echo "IGCM_sys_atlas : error code ${status}"
1868    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1869    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1870    IGCM_debug_PopStack "IGCM_sys_atlas"
1871    return 1
1872  else
1873    IGCM_debug_PopStack "IGCM_sys_atlas"
1874    return 0
1875  fi
1876
1877  IGCM_debug_PopStack "IGCM_sys_atlas"
1878}
1879
1880#D-#==================================================
1881#D-function IGCM_sys_rebuild_nemo
1882#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1883#D-* Examples:
1884#D-
1885
1886function IGCM_sys_rebuild_nemo {
1887  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1888  if ( $DEBUG_sys ) ; then
1889    echo "IGCM_sys_rebuild_nemo :" $@
1890  fi
1891
1892  $( ccc_home -u igcmg)/Tools/irene/bin/rebuild_nemo ${1} ${2}
1893
1894  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1895}
1896
Note: See TracBrowser for help on using the repository browser.