source: branches/libIGCM_concurrent/libIGCM_sys/libIGCM_sys_irene.ksh

Last change on this file was 1638, checked in by aclsce, 2 months ago

Added functionality to run concurrent execution with 2 or 3 model instances.
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 72.1 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Irene
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40if ([ X${DRYRUN} != X ] && [ ${DRYRUN} = 4 ]) ; then
41  typeset -r DRYRUN=0
42  typeset -r DRYRUN_DEBUG=4
43else
44  typeset -r DRYRUN=${DRYRUN:=0}
45  typeset -r DRYRUN_DEBUG=0
46fi
47
48
49# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
50# -------------------------------------------------------------------------------------
51# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
52# |          |  Cp/Exe/param/files |            |         |                           |
53# |          |  Chmod Qsub         |            |         |                           |
54# -------------------------------------------------------------------------------------
55# |    0     |       yes           |    yes     |  yes    |      yes                  |
56# -------------------------------------------------------------------------------------
57# |    1     |       yes           |    yes     |  yes    |      no                   |
58# -------------------------------------------------------------------------------------
59# |    2     |       yes           |    yes     |  no     |      no                   |
60# -------------------------------------------------------------------------------------
61# |    3     |       yes           |    no      |  no     |      no                   |
62# -------------------------------------------------------------------------------------
63# |    4     |       yes           |    yes     |  creation of RUN_DIR and Job_debug  |
64# -------------------------------------------------------------------------------------
65
66
67#=====================================================
68# Global Variables :
69#=====================================================
70# Language : "fr" or "en"
71typeset -r MYLANG="fr"
72
73#=====================================================
74# Host user names project maxCpuTime
75# $hostname ou hostname
76typeset HOST=${HOST:=$( hostname )}
77# $username ou whoami
78typeset LOGIN=${LOGIN:=$( whoami )}
79# $hostname of the MASTER job
80typeset MASTER=irene
81# add default project on irene
82typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=::default_project::} | cut -d@ -f1 )
83# jobWarningDelay in seconds
84typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
85
86#D-
87#D-#==================================================
88#D-Program used in libIGCM
89#D-#==================================================
90
91# Submit command
92typeset SUBMIT=${SUBMIT:=ccc_msub}
93# rsync with path
94typeset -r RSYNC=/usr/bin/rsync
95# RSYNC_opt args to rsync
96typeset -r RSYNC_opt="-va"
97# ie storage filesystem
98typeset -r STOREHOST=${MASTER}
99typeset -r REMOTE_RSYNC=/usr/bin/rsync
100
101#====================================================
102# Set environment tools (ferret, nco, cdo, rebuild, ...)
103#====================================================
104typeset CCCOS=${CCCOS:=$( ccc_os )}
105
106if [ X${TaskType} = Xcomputing ] ; then
107  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
108else
109  module purge > /dev/null 2>&1
110  if [ X${CCCOS} = X"Rhel_8__x86_64" ] ; then
111      . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
112  else
113      . $( ccc_home -u igcmg)/MachineEnvironment/irene_prev/env_irene > /dev/null 2>&1
114      . $( ccc_home -u igcmg)/MachineEnvironment/irene_prev/env_atlas_irene > /dev/null 2>&1
115  fi
116
117  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
118  export UVCDAT_ANONYMOUS_LOG=FALSE
119fi
120
121# FYI
122[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
123[ ! X${TaskType} = Xchecking ] && module list
124
125# For AddNoise and AddPertu3DOCE
126##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/curie/bin
127export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/bin
128
129# For rebuild
130##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
131export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl_IOIPSL_PLUS_v2_2_4/bin
132
133#====================================================
134# Host specific DIRECTORIES
135#====================================================
136
137# ============ CESIUM START ============ #
138
139#====================================================
140#- Mirror libIGCM from titane to cesium if needed
141#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
142#if [ ! ${ROOTSYS} = "home" ] ; then
143#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
144#else
145#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
146#fi
147
148#====================================================
149#- libIGCM_POST
150#if ( ${MirrorlibIGCM} ) ; then
151#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
152#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
153#else
154#  typeset -r libIGCM_POST=${libIGCM}
155#fi
156
157# ============ CESIUM  END  ============ #
158
159#====================================================
160#- MirrorlibIGCM for frontend
161typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
162
163#====================================================
164#- libIGCM_POST for frontend
165typeset -r libIGCM_POST=${libIGCM}
166
167#====================================================
168#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
169typeset -r R_EXE="${MODIPSL}/bin"
170
171#====================================================
172#- SUBMIT_DIR : submission dir
173typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
174
175#====================================================
176#- IN
177typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
178
179#====================================================
180#- RUN_DIR_PATH : Temporary working directory (=> TMP)
181typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
182
183#====================================================
184#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
185typeset -r OUTCOMMAND_PATH=/tmp
186
187#====================================================
188#- HOST_MPIRUN_COMMAND
189typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
190
191#====================================================
192#- Max number of arguments passed to nco operator or demigration command
193UNIX_MAX_LIMIT=360
194
195#====================================================
196#- set PackDefault to true on irene
197PackDefault=true
198
199#====================================================
200#- Default number of MPI task for IPSL coupled model
201#- required for backward compatibility
202#-
203DEFAULT_NUM_PROC_OCE=5
204DEFAULT_NUM_PROC_CPL=1
205(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
206
207#====================================================
208#- Number of computing cores per node
209#-
210NB_CORE_PER_NODE=48
211
212#D-#==================================================
213#D-function IGCM_sys_defineArchives
214#D-* Purpose:
215#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
216#D-* Define ARCHIVE : Dedicated to large files
217#D-* Define STORAGE : Dedicated to small/medium files
218#D-* Define R_OUT   : Output tree located on ARCHIVE
219#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
220#D-* Define R_BUF   : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
221#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
222#D-* Examples:
223#D-
224function IGCM_sys_defineArchives {
225  IGCM_debug_PushStack "IGCM_sys_defineArchives"
226
227  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
228  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = XDEFAULT ]; then
229    # Default option: Change the dfldatadir according to the project used for submission
230    # The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
231    if [ X"$( echo ${PROJECT} | grep cmip6 )" != "X" ] ; then
232      module switch dfldatadir dfldatadir/gencmip6
233      IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
234    else
235      module switch dfldatadir dfldatadir/${PROJECT}
236      IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
237    fi
238  else
239    # Use the project set by the variable DataProject in section UserChoices in config.card
240    module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
241    IGCM_debug_Print 1 "Load dfldatadir for DataProject read in config.card : ${config_UserChoices_DataProject}"
242  fi
243
244 #====================================================
245 #- RUN_DIR_PATH : Temporary working directory (=> TMP)
246 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
247
248  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
249    #====================================================
250    #- ARCHIVE (dedicated to large files)
251    ARCHIVE=${config_UserChoices_ARCHIVE}
252  else
253    #====================================================
254    #- ARCHIVE (dedicated to large files)
255    ARCHIVE=${CCCSTOREDIR}
256  fi
257
258  if [ ! X${config_UserChoices_STORAGE} = X ]; then
259    #====================================================
260    #- STORAGE (dedicated to small/medium files)
261    STORAGE=${config_UserChoices_STORAGE}
262  else
263    #====================================================
264    #- STORAGE (dedicated to small/medium files)
265    STORAGE=${CCCWORKDIR}
266  fi
267
268  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
269    #====================================================
270    #- R_OUT
271    R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
272
273    #====================================================
274    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
275    R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
276
277    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
278
279  else
280    #====================================================
281    #- R_OUT
282    R_OUT=${ARCHIVE}/IGCM_OUT
283
284    #====================================================
285    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
286    R_FIG=${STORAGE}/IGCM_OUT
287  fi
288
289  #====================================================
290  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
291  CMIP6_BUF=${STORAGE}/IGCM_OUT
292
293  #====================================================
294  #- R_BUF (ONLY FOR double copy an scratch)
295  R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
296
297  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
298  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
299  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
300
301  IGCM_debug_PopStack "IGCM_sys_defineArchives"
302}
303
304#D-#==================================================
305#D-function IGCM_sys_RshArchive
306#D-* Purpose: Archive rsh command
307#D-* Examples:
308#D-
309function IGCM_sys_RshArchive {
310  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
311  /bin/ksh <<-EOF
312    ${@}
313EOF
314  status=$?
315  if [ ${status} -gt 0 ] ; then
316    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
317    IGCM_debug_Exit "IGCM_sys_RshArchive"
318  fi
319  IGCM_debug_PopStack "IGCM_sys_RshArchive"
320}
321
322#D-#==================================================
323#D-function IGCM_sys_RshArchive_NoError
324#D-* Purpose: Archive rsh command, without error
325#D-*          used only in monitoring.job
326#D-* Examples:
327#D-
328function IGCM_sys_RshArchive_NoError {
329  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
330  /bin/ksh <<-EOF
331    ${@} 2> /dev/null
332EOF
333  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
334}
335
336#D-#==================================================
337#D-function IGCM_sys_MkdirArchive
338#D-* Purpose: Mkdir on Archive
339#D-* Examples:
340#D-
341function IGCM_sys_MkdirArchive {
342  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
343  if ( $DEBUG_sys ) ; then
344    echo "IGCM_sys_MkdirArchive :" $@
345  fi
346  #- creation de repertoire sur le serveur fichier
347  if [ ! -d ${1} ]; then
348    \mkdir -p $1
349    status=$?
350
351    if [ ${status} -gt 0 ] ; then
352      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
353      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
354    fi
355  fi
356  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
357}
358
359#D-#==================================================
360#D-function IGCM_sys_TestDirArchive
361#D-* Purpose: Test Directory that must exists on Archive
362#D-* Examples:
363#D-
364function IGCM_sys_TestDirArchive {
365  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
366  if ( $DEBUG_sys ) ; then
367    echo "IGCM_sys_TestDirArchive :" $@
368  fi
369  typeset ExistFlag
370  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
371  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
372  return ${ExistFlag}
373}
374
375#D-#==================================================
376#D-function IGCM_sys_IsFileArchived
377#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
378#D-* Examples:
379#D-
380function IGCM_sys_IsFileArchived {
381  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
382  if ( $DEBUG_sys ) ; then
383    echo "IGCM_sys_IsFileArchived :" $@
384  fi
385  typeset IsArchivedFlag
386  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
387  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
388
389  return ${IsArchivedFlag}
390}
391
392#D-#==================================================
393#D-function IGCM_sys_TestFileArchive
394#D-* Purpose: Test file that must NOT EXISTS on Archive
395#D-* Examples:
396#D-
397function IGCM_sys_TestFileArchive {
398  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
399  typeset ExistFlag
400  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
401  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
402
403  return ${ExistFlag}
404}
405
406#D-#==================================================
407#D-function IGCM_sys_CountFileArchive
408#D-* Purpose: Count files on Archive filesystem
409#D-* Examples:
410#D-
411function IGCM_sys_CountFileArchive {
412  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
413  ls ${@} 2>/dev/null | wc -l
414  if [ $? -gt 0 ] ; then
415    echo "IGCM_sys_CountFileArchive : erreur."
416  fi
417  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
418}
419
420#D-#==================================================
421#D-function IGCM_sys_Tree
422#D-* Purpose: Tree directories with files on ${ARCHIVE}
423#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
424#D-
425function IGCM_sys_Tree {
426  IGCM_debug_PushStack "IGCM_sys_Tree" $@
427  if ( $DEBUG_sys ) ; then
428    echo "IGCM_sys_Tree :" $@
429  fi
430
431  \ls -lR ${@}
432
433  IGCM_debug_PopStack "IGCM_sys_Tree"
434}
435
436#D-#==================================================
437#D-function IGCM_sys_Qsub
438#D-* Purpose: Qsub new job
439#D-* Examples:
440#D-
441function IGCM_sys_Qsub {
442  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
443  if ( $DEBUG_sys ) ; then
444    echo "IGCM_sys_Qsub :" $@
445  fi
446  typeset options status
447  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
448
449  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
450  status=$?
451
452  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
453  if [ ${status} -gt 0 ] ; then
454    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
455    IGCM_debug_Exit "IGCM_sys_Qsub"
456  else
457    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
458  fi
459  IGCM_debug_PopStack "IGCM_sys_Qsub"
460}
461
462#D-#==================================================
463#D-function IGCM_sys_QsubPost
464#D-* Purpose: Qsub new job on scalaire
465#D-* Examples:
466#D-
467function IGCM_sys_QsubPost {
468  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
469  if ( $DEBUG_sys ) ; then
470    echo "IGCM_sys_QsubPost :" $@
471  fi
472  typeset options status
473  options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
474
475  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
476  status=$?
477
478  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
479  if [ ${status} -gt 0 ] ; then
480    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
481    IGCM_debug_Exit "IGCM_sys_QsubPost"
482  else
483    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
484  fi
485  IGCM_debug_PopStack "IGCM_sys_QsubPost"
486}
487
488#D-*************************
489#D- File transfer functions
490#D-*************************
491#D-
492
493#D-#==================================================
494#D-function IGCM_sys_RmRunDir
495#D-* Purpose: rm tmpdir (dummy function most of the time batch
496#D-                      scheduler will do the job)
497#D-* Examples:
498#D-
499function IGCM_sys_RmRunDir {
500  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
501  if ( $DEBUG_sys ) ; then
502    echo "IGCM_sys_RmRunDir :" $@
503  fi
504
505  typeset status
506
507  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
508  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
509  status=$?
510
511  if [ ${status} -gt 0 ] ; then
512    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
513    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
514    IGCM_debug_Exit "IGCM_sys_RmRunDir"
515  else
516    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
517  fi
518  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
519}
520
521#D-#==================================================
522#D-function IGCM_sys_Put_Dir
523#D-* Purpose: Copy a complete directory on $(ARCHIVE)
524#D-* Examples:
525#D-
526function IGCM_sys_Put_Dir {
527  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
528  if ( $DEBUG_sys ) ; then
529    echo "IGCM_sys_Put_Dir :" $@
530  fi
531  if [ $DRYRUN = 0 ]; then
532    if [ ! -d ${1} ] ; then
533      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
534      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
535      return
536    fi
537
538    typeset status
539
540    # Only if we use rsync
541    #IGCM_sys_TestDirArchive $( dirname $2 )
542    #
543    #USUAL WAY
544    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
545    status=$?
546
547    if [ ${status} -gt 0 ] ; then
548      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
549      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
550      IGCM_debug_Exit "IGCM_sys_Put_Dir"
551    else
552      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
553    fi
554  fi
555  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
556}
557
558#D-#==================================================
559#D-function IGCM_sys_Get_Dir
560#D-* Purpose: Copy a complete directory from ${ARCHIVE}
561#D-* Examples:
562#D-
563function IGCM_sys_Get_Dir {
564  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
565  if ( $DEBUG_sys ) ; then
566    echo "IGCM_sys_Get_Dir :" $@
567  fi
568  if [ $DRYRUN = 0 ]; then
569    typeset NB_ESSAI DELAI status i
570    # number of tentative
571    NB_ESSAI=3
572    # time delay between tentative
573    DELAI=2
574
575    #
576    # USUAL WAY
577    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
578    ccc_hsm get -r $1
579
580    i=0
581    while [ $i -lt $NB_ESSAI ] ; do
582      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
583      status=$?
584      if [ ${status} -gt 0 ] ; then
585        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
586        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
587        sleep $DELAI
588      else
589        break
590      fi
591      (( i = i + 1 ))
592    done
593
594    if [ ${status} -gt 0 ] ; then
595      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
596      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
597      IGCM_debug_Exit "IGCM_sys_Get_Dir"
598    else
599      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
600    fi
601  fi
602  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
603}
604
605#D-#==================================================
606#D-function IGCM_sys_Put_Rest
607#D-* Purpose: Put computied restarts on ${ARCHIVE}.
608#D-           File and target directory must exist.
609#D-* Examples:
610#D-
611function IGCM_sys_Put_Rest {
612  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
613  if ( $DEBUG_sys ) ; then
614    echo "IGCM_sys_Put_Rest :" $@
615  fi
616  if [ $DRYRUN = 0 ]; then
617    if [ ! -f ${1} ] ; then
618      # The file is missing       
619      if [ X${3} = X'OPTIONAL' ] ; then
620          echo "WARINING : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
621          IGCM_debug_Print 1 "IGCM_sys_Put_Rest: Optional restart file ${1} is missing, now continue for the next"
622      else
623          echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
624          IGCM_debug_Exit "IGCM_sys_Put_Rest"
625      fi
626     
627    else
628      # The file exist
629
630    typeset status
631    #
632    # USUAL WAY
633    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
634    status=$?
635
636#       #RSYNC WITH NETWORK SSH CALL
637#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
638#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
639
640#       #RSYNC WITH NFS USE
641#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
642#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
643
644#       status=$?
645#       IGCM_sys_Rsync_out $status
646
647#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
648#       (( status=status+$? ))
649
650    if [ ${status} -gt 0 ] ; then
651      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
652      [ -f ${1} ] && ls -l ${1}
653      [ -f ${2} ] && ls -l ${2}
654      [ -f ${2}/${1} ] && ls -l ${2}/${1}
655      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
656      IGCM_debug_Exit "IGCM_sys_Put_Rest"
657    else
658
659      if [ X${JobType} = XRUN ] ; then
660        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
661        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
662      fi
663
664      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
665    fi
666   
667    fi
668   
669  fi
670  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
671}
672
673#D-#==================================================
674#D-function IGCM_sys_Put_Out
675#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
676#D-* Examples:
677#D-
678function IGCM_sys_Put_Out {
679  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
680  if ( $DEBUG_sys ) ; then
681    echo "IGCM_sys_Put_Out :" $@
682  fi
683
684  typeset NB_ESSAI DELAI status i exist skip
685  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
686
687  # number of tentative
688  NB_ESSAI=3
689  # time delay between tentative
690  DELAI=2
691
692  if [ $DRYRUN = 0 ]; then
693    if [ ! -f ${1} ] ; then
694      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
695      IGCM_debug_PopStack "IGCM_sys_Put_Out"
696      return 1
697    fi
698    #
699    IGCM_sys_MkdirArchive $( dirname $2 )
700    #
701    exist=false
702    skip=false
703    if [ -f $2 ] ; then
704      IGCM_debug_Print 1 "$2 already exist"
705      ccc_hsm get $2
706      exist=true
707      if [ "X$( diff $1 $2 )" = X ] ; then
708        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
709        skip=true
710      else
711        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
712        skip=false
713      fi
714    fi
715    #
716    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
717      IGCM_sys_Chmod u+w $2
718    fi
719
720    if [ X${skip} = Xfalse ] ; then
721      i=0
722      while [ $i -lt $NB_ESSAI ] ; do
723        # Identify file system
724        fileDeviceNumberInHex=$( stat -c %d $1 )
725        status=$?
726        if [ ${status} -gt 0 ] ; then
727          IGCM_debug_Exit "IGCM_sys_Put_Out"
728        fi
729        # Identify file system
730        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
731        status=$?
732        if [ ${status} -gt 0 ] ; then
733          IGCM_debug_Exit "IGCM_sys_Put_Out"
734        fi
735
736        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
737          # They are not on the same device. USUAL WAY
738          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
739          status=$?
740        else
741          # They are on the same device. NOT SO USUAL WAY
742          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
743          status=$?
744        fi
745        if [ ${status} -gt 0 ]; then
746          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
747          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
748          [ -f ${1} ] && ls -l ${1}
749          [ -f ${2} ] && ls -l ${2}
750          [ -f ${2}/${1} ] && ls -l ${2}/${1}
751          sleep $DELAI
752        else
753          break
754        fi
755        (( i = i + 1 ))
756      done
757    fi
758
759#       #RSYNC WITH NETWORK SSH CALL
760#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
761#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
762
763#       #RSYNC WITH NFS USE
764#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
765#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
766
767#       status=$?
768#       IGCM_sys_Rsync_out $status
769
770#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
771#       (( status=status+$? ))
772
773    if [ ${status} -gt 0 ] ; then
774      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
775      [ -f ${1} ] && ls -l ${1}
776      [ -f ${2} ] && ls -l ${2}
777      [ -f ${2}/${1} ] && ls -l ${2}/${1}
778      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
779      IGCM_debug_Exit "IGCM_sys_Put_Out"
780    else
781
782      if [ X${JobType} = XRUN ] ; then
783        if [ X${3} = X ] ; then
784          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
785          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
786        fi
787      fi
788
789      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
790    fi
791  fi
792  IGCM_debug_PopStack "IGCM_sys_Put_Out"
793  return 0
794}
795
796#D-#==================================================
797#D-function IGCM_sys_Get
798#D-* Purpose: Get a file from ${ARCHIVE}
799#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
800#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
801function IGCM_sys_Get {
802  IGCM_debug_PushStack "IGCM_sys_Get" $@
803
804  typeset DEST dm_liste target file_work
805  typeset NB_ESSAI DELAI status i
806
807  if ( $DEBUG_sys ) ; then
808    echo "IGCM_sys_Get :" $@
809  fi
810
811  # number of tentative
812  NB_ESSAI=3
813  # time delay between tentative
814  DELAI=2
815
816  if [ $DRYRUN -le 2 ]; then
817    if [ X${1} = X'/l' ] ; then
818      eval set +A dm_liste \${${2}}
819    else
820      eval set +A dm_liste ${1}
821    fi
822    eval DEST=\${${#}}
823    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
824    status=$?
825    if [ ${status} -gt 0 ] ; then
826      echo "WARNING IGCM_sys_Get : error code ${status}"
827      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
828      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
829    fi
830
831    #if [ ${status} -gt 0 ] ; then
832    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
833    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
834    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
835    #    sleep 30
836    #    echo "We try another time"
837    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
838    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
839    #    status=$?
840    #    if [ ${status} -gt 0 ] ; then
841    #      echo "ERROR IGCM_sys_Get : again demigration error :"
842    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
843    #      IGCM_debug_Exit "IGCM_sys_Get"
844    #    fi
845    #  else
846    #    echo "ERROR IGCM_sys_Get : demigration error :"
847    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
848    #    IGCM_debug_Exit "IGCM_sys_Get"
849    #  fi
850    #fi
851
852    #   #RSYNC WITH NETWORK SSH CALL
853    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
854    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
855
856    #   #RSYNC WITH NFS USE
857    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
858    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
859
860    #   status=$?
861    #   IGCM_sys_Rsync_out $status
862
863    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
864    #   (( status=status+$? ))
865
866    #USUAL WAY
867    if [ X${1} = X'/l' ] ; then
868      for target in ${dm_liste[*]} ; do
869        local_file=$( basename ${target} )
870        # test if the target file is present before the loop
871        IGCM_sys_TestFileArchive ${target}
872        status=$?
873        if [ ${status} -gt 0 ] ; then
874          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
875          IGCM_debug_Exit "IGCM_sys_Get"
876        else
877          i=0
878          while [ $i -lt $NB_ESSAI ] ; do
879            #if [ X${DoLink} = Xtrue ] ; then
880            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
881            #  status=$?
882            #  else
883            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
884            #  status=$?
885            #fi
886            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
887            status=$?
888            if [ ${status} -gt 0 ]; then
889              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
890              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
891              sleep $DELAI
892            else
893              break
894            fi
895            (( i = i + 1 ))
896          done
897          if [ ${status} -gt 0 ] ; then
898            echo "IGCM_sys_Get : error"
899            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
900            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
901            IGCM_debug_Exit "IGCM_sys_Get"
902          else
903            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
904          fi
905        fi
906      done
907    else
908      i=0
909      while [ $i -lt $NB_ESSAI ] ; do
910        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
911        status=$?
912        if [ ${status} -gt 0 ]; then
913          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
914          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
915          sleep $DELAI
916        else
917          break
918        fi
919        (( i = i + 1 ))
920      done
921      if [ ${status} -gt 0 ] ; then
922        echo "IGCM_sys_Get : error"
923        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
924        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
925        IGCM_debug_Exit "IGCM_sys_Get"
926      else
927        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
928      fi
929    fi
930  fi
931  IGCM_debug_PopStack "IGCM_sys_Get"
932}
933
934#D-#==================================================
935#D-function IGCM_sys_GetDate_Monitoring
936#D-* Purpose: get the last year for which the monitoring has been computed
937#D-* Examples:
938#D-
939function IGCM_sys_GetDate_Monitoring {
940  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
941  if ( $DEBUG_sys ) ; then
942    echo "IGCM_sys_GetDate_Monitoring :" $@
943  fi
944
945  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
946
947  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
948}
949
950#D-#==================================================
951#D-function IGCM_sys_Dods_Rm
952#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
953#D-* Examples:
954#D-
955function IGCM_sys_Dods_Rm {
956  if ( $DEBUG_sys ) ; then
957    echo "IGCM_sys_Dods_Rm :" $@
958  fi
959  typeset status
960  if [ $DRYRUN = 0 ]; then
961
962#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
963#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
964#      echo "Nothing has been done."
965#      return
966#    fi
967
968    if [ "$#" -eq 1 ]; then
969      ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
970      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
971      status=$?
972    else
973      ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
974      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
975      status=$?
976    fi
977
978#    if [ ${status} -gt 0 ] ; then
979#      echo "IGCM_sys_Dods_Rm : error."
980#      cat out_dods_rm
981#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
982#    else
983#      rm out_dods_rm
984#    fi
985
986  fi
987  return $status
988}
989
990#D-#==================================================
991#D-function IGCM_sys_Dods_Cp
992#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
993#D-* Examples:
994#D-
995function IGCM_sys_Dods_Cp {
996  if ( $DEBUG_sys ) ; then
997    echo "IGCM_sys_Dods_Cp :" $@
998  fi
999  typeset status
1000  if [ $DRYRUN = 0 ]; then
1001
1002#    if [ ! -d ${R_SAVE}/${1} ] ; then
1003#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1004#      echo "Nothing has been done."
1005#      return
1006#    fi
1007
1008    ## /ccc/cont003/home/igcmg/igcmg/Tools/irene/bin/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1009    $(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1010    status=$?
1011
1012#       if [ ${status} -gt 0 ] ; then
1013#           echo "IGCM_sys_Dods_Cp : error."
1014#           cat out_dods_cp
1015#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1016#       else
1017#           rm out_dods_cp
1018#       fi
1019
1020  fi
1021  return $status
1022}
1023
1024#D-#==================================================
1025#D-function IGCM_sys_Put_Dods
1026#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1027#D-* Examples:
1028#D-
1029function IGCM_sys_Put_Dods {
1030  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1031  if ( $DEBUG_sys ) ; then
1032    echo "IGCM_sys_Put_Dods :" $@
1033  fi
1034  typeset status
1035  if [ $DRYRUN = 0 ]; then
1036    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1037      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1038      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1039      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1040      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1041      return
1042    fi
1043
1044    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1045      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1046      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1047      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1048      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1049      return
1050    fi
1051    #
1052    if [ -d ${R_SAVE}/${1} ] ; then
1053      cd ${R_SAVE}
1054    elif [ -d ${R_FIGR}/${1} ] ; then
1055      cd ${R_FIGR}
1056    fi
1057
1058    IGCM_sys_Dods_Rm ${1}
1059    IGCM_sys_Dods_Cp ${1}
1060    status=0
1061
1062    if [ ${status} -gt 0 ] ; then
1063      echo "IGCM_sys_Put_Dods : error."
1064      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1065    fi
1066  fi
1067  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1068}
1069
1070##############################################################
1071# REBUILD OPERATOR
1072
1073#D-#==================================================
1074#D-function IGCM_sys_sync
1075#D-* Purpose: flush buffer on disk
1076#D-* Examples:
1077#D-
1078function IGCM_sys_sync {
1079  IGCM_debug_PushStack "IGCM_sys_sync" $@
1080  if ( $DEBUG_sys ) ; then
1081    echo "IGCM_sys_sync :" $@
1082  fi
1083
1084  /bin/sync
1085
1086  IGCM_debug_PopStack "IGCM_sys_sync"
1087}
1088
1089############################################################
1090# Activate Running Environnment Variables
1091
1092#D-#==================================================
1093#D-function IGCM_sys_activ_variables
1094#D-* Purpose: set environement variables prior to execution
1095#D-* Examples:
1096#D-
1097function IGCM_sys_activ_variables {
1098  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1099  if ( $DEBUG_sys ) ; then
1100    echo "IGCM_sys_activ_variables"
1101  fi
1102
1103# --------------------------------------------------------------------
1104#D- MODULE specifications
1105# --------------------------------------------------------------------
1106  # Source the file EnvFile containing all module load needed to run the model.
1107  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1108  # it will be used. Otherwise a default file will be used.
1109  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1110      # EnvFile is set in config.card and the file exists
1111      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1112      EnvFile=${config_UserChoices_EnvFile}
1113  else
1114      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1115      module purge > /dev/null 2>&1
1116      if [ X${compilerVersion} = XV17 ] ; then
1117          IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V17"
1118          ## EnvFile=/ccc/cont003/home/igcmg/igcmg/MachineEnvironment/irene/env_irene > /dev/null 2>&1
1119          EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene/env_irene > /dev/null 2>&1
1120      else
1121          IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1122      fi
1123  fi
1124  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1125  IGCM_debug_Print 1 ${EnvFile}
1126  . ${EnvFile}
1127
1128  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene. "
1129  module list
1130
1131# --------------------------------------------------------------------
1132#D- MPI specifications
1133# --------------------------------------------------------------------
1134
1135# --------------------------------------------------------------------
1136#D- Other specifications
1137# --------------------------------------------------------------------
1138
1139  ulimit -s unlimited
1140
1141  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1142}
1143
1144############################################################
1145# Desactivate Running Environnment Variables
1146
1147#D-#==================================================
1148#D-function IGCM_sys_desactiv_variables
1149#D-* Purpose: unset environement variables after execution
1150#D-* Examples:
1151#D-
1152function IGCM_sys_desactiv_variables {
1153  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1154  if ( $DEBUG_sys ) ; then
1155    echo "IGCM_sys_desactiv_variables"
1156  fi
1157# --------------------------------------------------------------------
1158#D- MPI specifications
1159# --------------------------------------------------------------------
1160
1161# --------------------------------------------------------------------
1162#D- Other specifications
1163# --------------------------------------------------------------------
1164
1165  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1166}
1167
1168############################################################
1169# Update job headers to be used by the scheduler
1170
1171#D-#==================================================
1172#D-function IGCM_sys_updateHeaders
1173#D-* Purpose: Update job headers to be used by the scheduler
1174#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1175#D-
1176function IGCM_sys_updateHeaders {
1177  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1178  if ( $DEBUG_sys ) ; then
1179    echo "IGCM_sys_updateHeaders"
1180  fi
1181  typeset file
1182  file=$1
1183    # MPMD + MPI + OMP : mpirun/ccc_mprun/error
1184        (( nodeNumber = coreNumber / NB_CORE_PER_NODE ))
1185        [ $(( ${coreNumber} % ${NB_CORE_PER_NODE} )) -ne 0 ] && (( nodeNumber = nodeNumber + 1 ))
1186      sed -e "/::openMPthreads::/d"                  \
1187        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1188          -e "s/::NodeNumber::/${nodeNumber}/"    \
1189        -e "/--cpu_bind=none/d"                    \
1190      ${file} > ${file}.tmp
1191  IGCM_sys_Mv ${file}.tmp ${file}
1192
1193  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1194}
1195
1196############################################################
1197# Build MPI/OMP scripts run file (dummy function)
1198
1199#D-#==================================================
1200#D-function IGCM_sys_build_run_file
1201#D-* Purpose: build run file (deprecated)
1202#D-* Examples:
1203#D-
1204function IGCM_sys_build_run_file {
1205  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1206}
1207
1208############################################################
1209# Build MPI/OMP scripts
1210
1211#D-#==================================================
1212#D-function IGCM_sys_build_execution_scripts
1213#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1214#D-* Examples:
1215#D-
1216function IGCM_sys_build_execution_scripts
1217{
1218  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1219  if ( $DEBUG_sys ) ; then
1220    echo "IGCM_sys_build_execution_scripts " $@
1221  fi
1222  if ( [ "X${config_UserChoices_ExecutionMode}" = "Xslurm" ] ) ; then
1223
1224      EXECUTION="/usr/bin/time srun "
1225
1226        if ( ${OK_PARA_MPMD} ) ; then
1227
1228    # MPMD mode
1229    # 1 MPI only : executionType=1
1230    # 2 MPI/OpenMP : executionType=2
1231
1232    if [ -f run_file ] ; then
1233      IGCM_sys_Rm -f run_file
1234    fi
1235    if [ -f RUNDIR_2/run_file ] ; then
1236        IGCM_sys_Rm -f RUNDIR_2/run_file
1237    fi
1238    touch run_file
1239
1240
1241
1242    # 2 MPI/OpenMP : executionType=2
1243
1244    # MPI-OpenMP (MPMD)
1245    # export SLURM_HOSTFILE=./hostlist
1246    # srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file
1247    # example of  hostlist file :
1248    # r3i3n33
1249    # r3i3n33
1250    # ...
1251    # example of run_file :
1252    # 0-70 ./prog_lmdz.x.sh %o %t
1253    # 71-430 ./prog_opa.xx.sh %o %t
1254    # 431-431 ./prog_xios.x.sh %o %t
1255    # examples of prog_file :
1256    # prog_lmdz.x.sh :
1257    # (( init = 0 + $1 ))
1258    # (( index = init * 10 ))
1259    # (( slot = index % 40 ))
1260    # taskset -c $slot-$((slot + 10 - 1)) ./script_lmdz.x.ksh
1261    # that will become
1262    # taskset -c 0-9 ./script_lmdz.x.ksh
1263    # ...
1264    # with script_lmdz.x.ksh
1265    # export OMP_STACKSIZE=3g
1266    # export OMP_PLACES=cores
1267    # export OMP_NUM_THREADS=10
1268    # ./lmdz.x > out_lmdz.x.out.${SLURM_PROCID} 2>out_lmdz.x.err.${SLURM_PROCID}
1269
1270    #  Hosts treatment
1271    _bkIFS=$IFS;
1272        IFS=$'\n'; set -f
1273        listnodes=($(< <( scontrol show hostnames $SLURM_JOB_NODELIST )))
1274        IFS=$_bkIFS; set +f
1275        rm -f hostlist
1276
1277     # Loop on the components to build run_file and script_exec files
1278        rank=0
1279        current_core=0
1280        current_core_mpi=0
1281        current_core_tmp=0
1282        current_core_mpi_tmp=0
1283        first_slurm_comp=0
1284        second_slurm_comp=0
1285
1286      for comp in ${config_ListOfComponents[*]} ; do
1287
1288        number_rundir=$(echo ${comp} | sed 's/[^0-9]*//g')
1289          if [ X${number_rundir} != X ] ; then
1290            if [ X${number_rundir} = X2 ] ; then
1291              [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
1292              cd RUNDIR_${number_rundir} 
1293              if [ ${first_slurm_comp} = "0" ] ; then
1294                  (( NbMPItasks_run1 = current_core_mpi_tmp ))
1295                  ### On change de noeud pour le prochain srun
1296                  if [ $(( $current_core % $NB_CORE_PER_NODE )) -ne 0 ] ; then
1297                      (( current_core = current_core + NB_CORE_PER_NODE - current_core % NB_CORE_PER_NODE ))
1298                  fi
1299                  current_core_tmp=0 ; current_core_mpi_tmp=0 ; first_slurm_comp=1 ;
1300              fi
1301            elif [ X${number_rundir} = X3 ] ; then
1302                [ ! -d RUNDIR_${number_rundir} ] && mkdir RUNDIR_${number_rundir}
1303                cd RUNDIR_${number_rundir} 
1304                if [ ${second_slurm_comp} = "0" ] ; then
1305                    (( NbMPItasks_run2 = current_core_mpi_tmp ))
1306                    ### On change de noeud pour le prochain srun
1307                    if [ $(( $current_core % $NB_CORE_PER_NODE )) -ne 0 ] ; then
1308                        (( current_core = current_core + NB_CORE_PER_NODE - current_core % NB_CORE_PER_NODE ))
1309                    fi
1310                    current_core_tmp=0 ; current_core_mpi_tmp=0 ; second_slurm_comp=1 ;
1311                fi
1312            fi
1313          fi
1314
1315        eval ExeNameIn=\${config_Executable_${comp}[0]}
1316        eval ExeNameOut=\${config_Executable_${comp}[1]}
1317
1318        # Not possible if oasis has an executable (i.e old version of oasis3)
1319        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1320          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1321          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1322          IGCM_debug_Verif_Exit
1323        fi
1324
1325        # Only if we really have an executable for the component :
1326        if [ "X${ExeNameOut}" != X\"\" ] ; then
1327
1328          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1329          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1330          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1331     
1332
1333          # Build script files
1334
1335          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1336          echo ""  >> script_${ExeNameOut}.ksh
1337          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1338            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1339            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1340            echo "export OMP_NUM_THREADS=${comp_proc_omp_loc}"  >> script_${ExeNameOut}.ksh
1341          fi
1342
1343          # to have out/err per process on different files
1344          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1345
1346          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1347
1348        # Build run_file
1349        # Only if the component has an executable
1350        if ( [ "X${ExeNameOut}" != X\"\" ] ) ; then
1351
1352          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1353          (( end_core = ${current_core_mpi_tmp} + ${comp_proc_mpi_loc} - 1 ))
1354          echo "${current_core_mpi_tmp}-${end_core} ./prog_${ExeNameOut}.sh %o %t" >> run_file
1355          (( current_core_mpi_tmp = ${end_core} + 1 ))
1356        fi
1357
1358          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1359              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1360          else
1361              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1362          fi
1363
1364          # Build configuration file
1365 
1366              echo "#!/bin/sh" > prog_${ExeNameOut}.sh
1367          echo "(( init = $current_core_tmp + \$1 ))" >> prog_${ExeNameOut}.sh
1368          echo "(( index = init * $comp_proc_omp_loc ))" >> prog_${ExeNameOut}.sh
1369          echo "(( slot = index % $NB_CORE_PER_NODE ))" >> prog_${ExeNameOut}.sh
1370              echo "echo ${ExeNameOut} taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1))" >> prog_${ExeNameOut}.sh
1371          echo "taskset -c \$slot"-"\$((slot + $comp_proc_omp_loc - 1)) ./script_${ExeNameOut}.ksh" >> prog_${ExeNameOut}.sh
1372
1373          IGCM_sys_Chmod u+x prog_${ExeNameOut}.sh
1374
1375        # Build hostlist file
1376
1377          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1378             (( index_host = current_core / NB_CORE_PER_NODE ))
1379             host_value=${listnodes[${index_host}]}
1380             echo "$host_value" >> hostlist
1381             if [ ${DRYRUN_DEBUG} = 4 ] ; then
1382                 echo "node_${index_host}_X" >> hostlist_template
1383             fi
1384             (( current_core = current_core + offset_comp_proc_loc ))
1385             (( current_core_tmp = current_core_tmp + offset_comp_proc_loc ))
1386          done
1387        fi
1388        if [ X${number_rundir} != X ] ; then
1389            cd $RUN_DIR ;
1390        fi
1391      done
1392
1393      ## variable added to stop after 60s instead of 600s by default.
1394      ## This is used when no error comes from executables and when something stopped an executable without notice.
1395      export SLURM_WAIT=60
1396
1397      if [ X${number_rundir} = X2 ] ; then
1398          echo "cd $RUN_DIR ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${NbMPItasks_run1} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution  2>&1 &" > EXECUTION.exe
1399          echo "cd RUNDIR_2 ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${current_core_mpi_tmp} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution  2>&1 &" >> EXECUTION.exe
1400          echo "wait" >> EXECUTION.exe
1401          chmod u+x EXECUTION.exe
1402      elif [ X${number_rundir} = X3 ] ; then
1403          echo "cd $RUN_DIR ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${NbMPItasks_run1} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution  2>&1 &" > EXECUTION.exe
1404          echo "cd RUNDIR_2 ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${NbMPItasks_run2} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution  2>&1 &" >> EXECUTION.exe
1405          echo "cd ../RUNDIR_3 ; export SLURM_HOSTFILE=./hostlist ; /usr/bin/time srun --ntasks=${current_core_mpi_tmp} --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file > out_execution  2>&1 &" >> EXECUTION.exe
1406          echo "wait" >> EXECUTION.exe
1407          chmod u+x EXECUTION.exe
1408      else
1409          EXECUTION="/usr/bin/time srun --cpu-bind=none --distribution=arbitrary --multi-prog ./run_file"
1410      fi
1411      IGCM_sys_Chmod u+x run_file
1412      if ( $DEBUG_sys ) ; then
1413        echo "run_file contains : "
1414        cat run_file
1415      fi
1416
1417#    fi # if ${OK_PARA_MPMD}
1418
1419  else
1420  # Only one executable (SPMD mode):  executionType=3, 4, 5 and 6
1421
1422    for comp in ${config_ListOfComponents[*]} ; do
1423
1424      # Only if we really have an executable for the component :
1425      eval ExeNameOut=\${config_Executable_${comp}[1]}
1426      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1427
1428        # Build script files
1429
1430        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1431        echo ""  >> script_${ExeNameOut}.ksh
1432        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1433
1434        if ( ${OK_PARA_OMP} ) ; then
1435            eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1436            # Check if the number of threads is correct
1437            case ${comp_proc_omp_loc} in
1438                2|4|5|10|20)
1439                    IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1440                    ;;
1441                *)
1442                    IGCM_debug_Exit "ERROR with OMP parameters !"
1443                    IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1444                    IGCM_debug_Print 2 "Only 2,4,5,10,20 as number of OMP threads are possible "
1445                    IGCM_debug_Verif_Exit
1446                    ;;
1447            esac
1448            echo ""  >> script_${ExeNameOut}.ksh
1449            echo "export OMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1450            echo "export OMP_PLACES=cores"  >> script_${ExeNameOut}.ksh
1451            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1452        fi
1453
1454        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1455       
1456        # To have out/err per process on different files
1457        echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1458        EXECUTION="/usr/bin/time srun ./script_${ExeNameOut}.ksh"
1459
1460        IGCM_debug_Print 1 "sys Jean-Zay : script_${ExeNameOut}.ksh contains"
1461        cat script_${ExeNameOut}.ksh
1462
1463      fi
1464
1465    done
1466
1467  fi # ${OK_PARA_MPMD}
1468     
1469  else
1470
1471  EXECUTION=${HOST_MPIRUN_COMMAND}
1472
1473  if ( ${OK_PARA_MPMD} ) ; then
1474
1475    # MPMD mode
1476    # 1 MPI only
1477    # 2 MPI/OpenMP mpirun method
1478    # 3 MPI/OpenMP ccc_mprun method
1479
1480    if [ -f run_file ] ; then
1481      IGCM_sys_Rm -f run_file
1482    fi
1483    touch run_file
1484
1485    # case 1 : Only MPI (MPMD)
1486    if  ( ! ${OK_PARA_OMP} ) ; then
1487
1488      # Build run_file
1489
1490      # First loop on the components for the coupler ie oasis (only if oasis3)
1491      # the coupler ie oasis3 must be the first one
1492      for comp in ${config_ListOfComponents[*]} ; do
1493
1494        eval ExeNameIn=\${config_Executable_${comp}[0]}
1495        eval ExeNameOut=\${config_Executable_${comp}[1]}
1496
1497        # for CPL component only
1498        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1499
1500          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1501          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1502          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1503        fi
1504      done
1505
1506      # Then second loop on the components
1507      for comp in ${config_ListOfComponents[*]} ; do
1508
1509        eval ExeNameIn=\${config_Executable_${comp}[0]}
1510        eval ExeNameOut=\${config_Executable_${comp}[1]}
1511
1512        # Only if we really have an executable for the component and not the coupler ie oasis:
1513        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1514
1515          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1516          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1517          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1518        fi
1519      done
1520
1521      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1522
1523      IGCM_sys_Chmod u+x run_file
1524      if ( $DEBUG_sys ) ; then
1525        echo "run_file contains : "
1526        cat run_file
1527      fi
1528
1529    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1530
1531    # 2 MPI/OpenMP mpirun method
1532
1533      # Use of mpirun instead of ccc_mprun
1534      EXECUTION="time mpirun"
1535
1536      #  Hosts treatment
1537      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1538
1539      i=0
1540      rm -f hosts rankfile
1541      IGCM_debug_Print 1 "sys Irene, Hosts available :"
1542      for nodes in `cat hosts.tmp`
1543      do
1544        host[$i]=$nodes
1545        echo "${host[$i]}" >> hosts
1546        IGCM_debug_Print 1 ${host[$i]}
1547        i=$((i+1))
1548      done
1549      rm -f hosts.tmp
1550
1551      listnodes=${host[*]}
1552
1553      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1554
1555      # Initialisation
1556      rank=0
1557      current_core=0
1558      init_exec=n
1559
1560      # Loop on the components
1561      for comp in ${config_ListOfComponents[*]} ; do
1562
1563        eval ExeNameIn=\${config_Executable_${comp}[0]}
1564        eval ExeNameOut=\${config_Executable_${comp}[1]}
1565
1566        # Not possible if oasis has an executable (i.e old version of oasis3)
1567        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1568          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1569          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1570          IGCM_debug_Verif_Exit
1571        fi
1572
1573        # Only if we really have an executable for the component :
1574        if [ "X${ExeNameOut}" != X\"\" ] ; then
1575
1576          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1577          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1578          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1579
1580          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1581          echo ""  >> script_${ExeNameOut}.ksh
1582          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1583
1584            # Check if the number of threads is correct
1585            case ${comp_proc_omp_loc} in
1586            2|4|8|16)
1587              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1588              ;;
1589            *)
1590              IGCM_debug_Exit "ERROR with OMP parameters !"
1591              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1592              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1593              IGCM_debug_Verif_Exit
1594              ;;
1595            esac
1596            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1597            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1598            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1599            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1600          fi
1601
1602          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1603          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1604          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1605          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1606          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1607
1608          if [ ${init_exec} = y ] ; then
1609            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1610          else
1611            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1612            init_exec=y
1613          fi
1614
1615          # Build rankfile : method used to assign cores and nodes for the MPI process
1616          # Ex :
1617          #rank 0=curie5296 slot=0,1,2,3
1618          #rank 1=curie5296 slot=4,5,6,7
1619          # Example of final command :
1620          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1621          # with script_lmdz.x.ksh :
1622          # #!/bin/ksh
1623          #export KMP_STACKSIZE=3g
1624          #export KMP_LIBRARY=turnaround
1625          #export MKL_SERIAL=YES
1626          #OMP_NUM_THREADS=4
1627          #./lmdz.x
1628          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1629              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1630          else
1631              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1632          fi
1633          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1634             (( index_host = current_core / NB_CORE_PER_NODE ))
1635             host_value=${host[${index_host}]}
1636             (( slot =  current_core % NB_CORE_PER_NODE ))
1637             virg=","
1638             string_final=""
1639             for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1640                 string=$index$virg
1641                 string_final=$string_final$string
1642             done
1643             string_final=$( echo $string_final | sed "s/.$//" )
1644             echo "rank $rank=$host_value slot=$string_final" >> rankfile
1645             (( rank = rank + 1 ))
1646             (( current_core = current_core + offset_comp_proc_loc ))
1647          done
1648        fi
1649      done
1650
1651    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1652
1653    # 3 MPI/OpenMP ccc_mprun method
1654
1655    # MPI-OpenMP (MPMD)
1656    # example of run_file
1657    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1658    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1659    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1660
1661      # Loop on the components
1662      for comp in ${config_ListOfComponents[*]} ; do
1663
1664        eval ExeNameIn=\${config_Executable_${comp}[0]}
1665        eval ExeNameOut=\${config_Executable_${comp}[1]}
1666
1667        # Not possible if oasis has an executable (i.e old version of oasis3)
1668        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1669          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1670          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1671          IGCM_debug_Verif_Exit
1672        fi
1673
1674        # Only if we really have an executable for the component :
1675        if [ "X${ExeNameOut}" != X\"\" ] ; then
1676
1677          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1678          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1679
1680          # Build script files
1681
1682          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1683          echo ""  >> script_${ExeNameOut}.ksh
1684          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1685
1686            # Check if the number of threads is correct
1687            case ${comp_proc_omp_loc} in
1688            2|4|8|12|16|24)
1689              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1690              ;;
1691            *)
1692              IGCM_debug_Exit "ERROR with OMP parameters !"
1693              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1694              IGCM_debug_Print 2 "Only 2,4,8,12,16,24 as number of OMP threads are possible "
1695              IGCM_debug_Verif_Exit
1696              ;;
1697            esac
1698            ### only for LMDZ?
1699            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1700            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1701            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1702            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1703
1704          fi
1705
1706          # to have out/err per process on different files
1707          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1708          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1709
1710          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1711
1712          # Complete run_file
1713
1714          echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1715
1716        fi
1717      done
1718
1719      ## variable added to stop after 60s instead of 600s by default.
1720      ## This is used when no error comes from executables and when something stopped an executable without notice.
1721      export SLURM_WAIT=60
1722
1723      ## module advised by TGCC (instead of 2 variables)
1724      module load feature/bridge/heterogenous_mpmd
1725
1726      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1727
1728      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1729
1730      IGCM_sys_Chmod u+x run_file
1731      if ( $DEBUG_sys ) ; then
1732        echo "run_file contains : "
1733        cat run_file
1734      fi
1735
1736    else
1737
1738      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1739
1740    fi # 1 2 or 3 if ${OK_PARA_MPMD}
1741
1742  else
1743  # Only one executable (SPMD mode).
1744
1745    for comp in ${config_ListOfComponents[*]} ; do
1746
1747      # Only if we really have an executable for the component :
1748      eval ExeNameOut=\${config_Executable_${comp}[1]}
1749      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1750
1751        # Build script files
1752
1753        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1754        echo ""  >> script_${ExeNameOut}.ksh
1755        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1756
1757        if ( ${OK_PARA_OMP} ) ; then
1758          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1759          echo ""  >> script_${ExeNameOut}.ksh
1760          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1761          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1762          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1763          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1764        fi
1765
1766        if  ( ${OK_PARA_MPI} ) ; then
1767          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1768          # Default : ccc_mprun used if nb_proc gt 1
1769          # to have out/err per process on different files
1770          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1771          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1772          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1773        else
1774          # Default : ccc_mprun is NOT used if nb_proc eq 1
1775          # to have out/err per process on different files
1776          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1777          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1778          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1779        fi
1780
1781        IGCM_debug_Print 1 "sys Irene : script_${ExeNameOut}.ksh contains"
1782        cat script_${ExeNameOut}.ksh
1783
1784      fi
1785
1786    done
1787
1788  fi # ${OK_PARA_MPMD}
1789
1790  fi
1791  IGCM_debug_Print 1 "sys Irene : execution command is "
1792  IGCM_debug_Print 1 "$EXECUTION"
1793
1794  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1795}
1796
1797#D-#==================================================
1798#D-function IGCM_sys_check_path
1799#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1800#D-* do not point to an important use directory. Stop immediately in that case.
1801#D-* Examples:
1802#D-
1803function IGCM_sys_check_path {
1804  IGCM_debug_PushStack "IGCM_sys_check_path"
1805  if ( $DEBUG_sys ) ; then
1806    echo "IGCM_sys_check_path"
1807  fi
1808
1809  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1810    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1811    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1812    IGCM_debug_Exit "This will stop the job"
1813  fi
1814  IGCM_debug_PopStack "IGCM_sys_check_path"
1815}
1816
1817#D-#==================================================
1818#D-function IGCM_sys_check_quota
1819#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1820#D-* Examples:
1821#D-
1822function IGCM_sys_check_quota {
1823  IGCM_debug_PushStack "IGCM_sys_check_quota"
1824  if ( $DEBUG_sys ) ; then
1825    echo "IGCM_sys_check_quota"
1826  fi
1827  # Limit of quota (in %)
1828  limit_quota=90
1829
1830  # Check of the volume
1831  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1832  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1833
1834  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1835
1836    unit_avail=${volume_avail: -1}
1837    unit_quota=${volume_quota: -1}
1838
1839    if [ "${unit_quota}" = "*" ] ; then
1840      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1841      IGCM_debug_Print 1 "More than 100% of your quota is used"
1842      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1843      IGCM_debug_Print 1 "You must have more than 10% available to run"
1844      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1845      IGCM_debug_Verif_Exit
1846    fi
1847
1848    temp_avail=${volume_avail%%${volume_avail: -1}*}
1849    temp_quota=${volume_quota%%${volume_quota: -1}*}
1850
1851    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1852
1853    # Convertion
1854      if [ ${volume_avail: -1} = "T" ] ; then
1855        (( temp_avail = temp_avail * 1000000000000 ))
1856      elif [ ${volume_avail: -1} = "G" ] ; then
1857        (( temp_avail = temp_avail * 1000000000 ))
1858      elif [ ${volume_avail: -1} = "M" ] ; then
1859        (( temp_avail = temp_avail * 1000000 ))
1860      elif [ ${volume_avail: -1} = "k" ] ; then
1861        (( temp_avail = temp_avail * 1000 ))
1862      else
1863        (( temp_avail = volume_avail ))
1864      fi
1865      if [ ${volume_quota: -1} = "T" ] ; then
1866        (( temp_quota = temp_quota * 1000000000000 ))
1867      elif [ ${volume_quota: -1} = "G" ] ; then
1868        (( temp_quota = temp_quota * 1000000000 ))
1869      elif [ ${volume_quota: -1} = "M" ] ; then
1870        (( temp_quota = temp_quota * 1000000 ))
1871      elif [ ${volume_quota: -1} = "k" ] ; then
1872        (( temp_quota = temp_quota * 1000 ))
1873      else
1874        (( temp_quota = volume_quota ))
1875      fi
1876    fi
1877
1878    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1879#    echo "volume ratio is " $quota_volume
1880
1881    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1882      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1883      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1884      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1885      IGCM_debug_Print 1 "You must have more than 10% available to run"
1886      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1887      IGCM_debug_Verif_Exit
1888    fi
1889
1890  fi
1891
1892# Check of the number of inodes
1893
1894  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1895  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1896
1897  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1898
1899    unit_avail=${inode_avail: -1}
1900    unit_quota=${inode_quota: -1}
1901
1902    if [ "${unit_quota}" = "*" ] ; then
1903      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1904      IGCM_debug_Print 1 "More than 100% of your quota is used"
1905      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1906      IGCM_debug_Print 1 "You must have more than 10% available to run"
1907      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1908      IGCM_debug_Verif_Exit
1909    fi
1910
1911    temp_avail=${inode_avail%%${inode_avail: -1}*}
1912    temp_quota=${inode_quota%%${inode_quota: -1}*}
1913
1914    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1915
1916    # Convertion
1917      if [ ${inode_avail: -1} = "T" ] ; then
1918        (( temp_avail = temp_avail * 1000000000000 ))
1919      elif [ ${inode_avail: -1} = "G" ] ; then
1920        (( temp_avail = temp_avail * 1000000000 ))
1921      elif [ ${inode_avail: -1} = "M" ] ; then
1922        (( temp_avail = temp_avail * 1000000 ))
1923      elif [ ${inode_avail: -1} = "k" ] ; then
1924        (( temp_avail = temp_avail * 1000 ))
1925      else
1926        (( temp_avail = inode_avail ))
1927      fi
1928
1929      if [ ${inode_quota: -1} = "T" ] ; then
1930        (( temp_quota = temp_quota * 1000000000000 ))
1931      elif [ ${inode_quota: -1} = "G" ] ; then
1932        (( temp_quota = temp_quota * 1000000000 ))
1933      elif [ ${inode_quota: -1} = "M" ] ; then
1934        (( temp_quota = temp_quota * 1000000 ))
1935      elif [ ${inode_quota: -1} = "k" ] ; then
1936        (( temp_quota = temp_quota * 1000 ))
1937      else
1938        (( temp_quota = inode_quota ))
1939      fi
1940    fi
1941    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1942#    echo "inode ratio is " $quota_inode
1943
1944    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1945      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1946      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1947      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1948      IGCM_debug_Print 1 "You must have more than 10% available to run"
1949      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1950      IGCM_debug_Verif_Exit
1951    fi
1952  fi
1953  IGCM_debug_PopStack "IGCM_sys_check_quota"
1954}
1955
1956#D-#==================================================
1957#D-function IGCM_sys_projectAccounting
1958#D-* Purpose: store project accounting information in a file
1959#D-* Examples:
1960#D-
1961function IGCM_sys_projectAccounting {
1962  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1963  if ( $DEBUG_sys ) ; then
1964    echo "IGCM_sys_check_quota"
1965  fi
1966
1967  ssh irene191 /usr/bin/ccc_myproject > $1
1968
1969  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1970}
1971
1972#D-#==================================================
1973#D-function IGCM_sys_getJobSchedulerID
1974#D-* Purpose: Get the job ID during execution
1975#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1976#D-
1977function IGCM_sys_getJobSchedulerID {
1978  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1979  if ( $DEBUG_sys ) ; then
1980    echo "IGCM_sys_getJobSchedulerID"
1981  fi
1982
1983  eval ${1}=${BRIDGE_MSUB_JOBID}
1984
1985  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1986}
1987
1988#D-#==================================================
1989#D-function IGCM_sys_GetJobID
1990#D-* Purpose: Get the job ID from the JobName
1991#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1992#D-
1993function IGCM_sys_GetJobID {
1994  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1995  if ( $DEBUG_sys ) ; then
1996    echo "IGCM_sys_GetJobID"
1997  fi
1998
1999  # With -f option, the full job name is given in the last column
2000  ID=$( ccc_mstat -f -u $2 | \
2001        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
2002
2003  eval ${3}=${ID}
2004  IGCM_debug_PopStack "IGCM_sys_GetJobID"
2005}
2006
2007#D-#==================================================
2008#D-function IGCM_sys_CountJobInQueue
2009#D-* Purpose: Count number of users job
2010#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
2011#D-
2012function IGCM_sys_CountJobInQueue {
2013  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
2014  if ( $DEBUG_sys ) ; then
2015    echo "IGCM_sys_CountJobInQueue"
2016  fi
2017
2018  # With -f option, the full job name is given in the last column
2019  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
2020
2021  eval ${2}=${NbRun}
2022
2023  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
2024}
2025
2026#D-#==================================================
2027#D-function IGCM_sys_ListJobInQueue
2028#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
2029#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
2030#D-
2031function IGCM_sys_ListJobInQueue {
2032  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
2033  if ( $DEBUG_sys ) ; then
2034    echo "IGCM_sys_ListJobInQueue"
2035  fi
2036
2037  # With -f option, the full job name is given in the last column
2038  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
2039                                        '( $2  == User      && \
2040                                           $NF != /TS/      && \
2041                                           $NF !~ /PACK/    && \
2042                                           $NF !~ /REBUILD/ && \
2043                                           $NF !~ /pack/ )     \
2044                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
2045
2046  eval set -A ${2} ${JobList[*]}
2047
2048  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
2049}
2050
2051#D-#==================================================
2052#D-function IGCM_sys_atlas
2053#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
2054#D-* Examples:
2055#D-
2056function IGCM_sys_atlas {
2057  IGCM_debug_PushStack "IGCM_sys_atlas" $@
2058  if ( $DEBUG_sys ) ; then
2059    echo "IGCM_sys_atlas :" $@
2060  fi
2061
2062  typeset status
2063
2064  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
2065  status=$?
2066  if [ ${status} -gt 0 ] ; then
2067    echo "IGCM_sys_atlas : error code ${status}"
2068    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
2069    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
2070    IGCM_debug_PopStack "IGCM_sys_atlas"
2071    return 1
2072  else
2073    IGCM_debug_PopStack "IGCM_sys_atlas"
2074    return 0
2075  fi
2076
2077  IGCM_debug_PopStack "IGCM_sys_atlas"
2078}
2079
2080#D-#==================================================
2081#D-function IGCM_sys_rebuild_nemo
2082#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
2083#D-* Examples:
2084#D-
2085
2086function IGCM_sys_rebuild_nemo {
2087  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
2088  if ( $DEBUG_sys ) ; then
2089    echo "IGCM_sys_rebuild_nemo :" $@
2090  fi
2091
2092  $( ccc_home -u igcmg)/Tools/irene/bin/rebuild_nemo ${1} ${2}
2093
2094  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
2095}
2096
Note: See TracBrowser for help on using the repository browser.