source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 1134

Last change on this file since 1134 was 1134, checked in by sdipsl, 10 years ago
  • Bugfix due to a side effect of the IDRIS /tmp workaround
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 93.3 KB
RevLine 
[622]1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
[623]16# if you use the prefix #D- for comments to be extracted.
[622]17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
[623]21#D-LibIGCM_sys for Curie
[622]22#D-#==================================================
23#D-
[623]24#D- This ksh library if a layer under some usefull
[622]25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
[809]45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
[622]47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset  HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset  LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=curie
[880]71# add default project on curie
[900]72typeset DEFAULT_PROJECT=gen0826
[622]73
74#D-
75#D-#==================================================
76#D-Program used in libIGCM
77#D-#==================================================
78
[742]79# Submit command
[747]80typeset SUBMIT=${SUBMIT:=ccc_msub}
[622]81# rsync with path
82typeset -r RSYNC=/usr/bin/rsync
83# RSYNC_opt args to rsync
84typeset -r RSYNC_opt="-va"
85# ie storage filesystem
[623]86typeset -r STOREHOST=${MASTER}
[622]87
88#====================================================
89# Source default environment
90#====================================================
91##. /etc/profile
92
93#====================================================
94# Set environment tools (ferret, nco, cdo)
95#====================================================
[996]96if [ X${TaskType} = Xcomputing ] ; then
97  . /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4_curie_ksh > /dev/null 2>&1
98# to run with netcdf 3.6.3 ie compilation done before 17/2/2014
99# uncomment 2 lines :
100#  module unload netcdf
101#  module load netcdf/3.6.3
102  export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
[1026]103  export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
[996]104else
105  . /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4_curie_ksh > /dev/null 2>&1
106fi
[622]107
108#====================================================
109# Specific for ocean additionnal diagnostic
110export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
111export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
112
113#====================================================
114# Host specific DIRECTORIES
115#====================================================
116
[623]117# ============ CESIUM START ============ #
118
[622]119#====================================================
[623]120#- Mirror libIGCM from titane to cesium if needed
121#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
122#if [ ! ${ROOTSYS} = "home" ] ; then
123#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
124#else
125#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
126#fi
127
128#====================================================
129#- libIGCM_POST
130#if ( ${MirrorlibIGCM} ) ; then
131#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
132#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
133#else
134#  typeset -r libIGCM_POST=${libIGCM}
135#fi
136
137# ============ CESIUM  END  ============ #
138
139#====================================================
140#- MirrorlibIGCM for frontend
141typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
142
143#====================================================
144#- libIGCM_POST for frontend
145typeset -r libIGCM_POST=${libIGCM}
146
147#====================================================
[622]148#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
149typeset -r R_EXE="${MODIPSL}/bin"
150
151#====================================================
152#- SUBMIT_DIR : submission dir
[1010]153typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
[622]154
155#====================================================
156#- IN
157typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
158typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
159
160#====================================================
161#- RUN_DIR_PATH : Temporary working directory (=> TMP)
[850]162typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
[1134]163# Due to a workaround at IDRIS. /tmp is not functionning correctly there.
164IGCM_sys_MkdirWork ${RUN_DIR_PATH}
[622]165
166#====================================================
167#- HOST_MPIRUN_COMMAND
[808]168typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1"}
[622]169
170#====================================================
171#- Max number of arguments passed to nco operator or demigration command
172UNIX_MAX_LIMIT=360
173
[623]174#====================================================
175#- set PackDefault true on curie
176PackDefault=true
177
[664]178#====================================================
179#- Number of core per node (max number of OpenMP task)
[622]180NUM_COREPERNODE=8
181
[623]182#====================================================
183#- Default number of MPI task for IPSL coupled model
184#- required for backward compatibility
185#-
186DEFAULT_NUM_PROC_OCE=5
187DEFAULT_NUM_PROC_CPL=1
[637]188(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
189DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
[623]190
[725]191#D-#==================================================
[1057]192#D-function IGCM_sys_defineArchives
193#D-* Purpose:
194#D-* Define ARCHIVE : Dedicated to large files
195#D-* Define STORAGE : Dedicated to small/medium files
196#D-* Define R_OUT   : Output tree located on ARCHIVE
197#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
198#D-* Define R_BUF   : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
199#D-* if SpaceName=TEST everything is stored on SCRATCHDIR
[725]200#D-* Examples:
201#D-
[1057]202function IGCM_sys_defineArchives {
203  IGCM_debug_PushStack "IGCM_sys_defineArchives"
[676]204
[1060]205  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
[1057]206    #====================================================
207    #- ARCHIVE (dedicated to large files)
[1063]208    ARCHIVE=${config_UserChoices_ARCHIVE}
[1057]209  else
210    #====================================================
211    #- ARCHIVE (dedicated to large files)
[1063]212    ARCHIVE=${CCCSTOREDIR}
[1057]213  fi
[725]214
[1060]215  if [ ! X${config_UserChoices_STORAGE} = X ]; then
[1057]216    #====================================================
217    #- STORAGE (dedicated to small/medium files)
[1063]218    STORAGE=${config_UserChoices_STORAGE}
[1057]219  else
220    #====================================================
221    #- STORAGE (dedicated to small/medium files)
[1063]222    STORAGE=${CCCWORKDIR}
[1057]223  fi
[725]224
[1057]225  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
226    #====================================================
227    #- R_OUT
[1063]228    R_OUT=${SCRATCHDIR}/IGCM_OUT
[1057]229
230    #====================================================
231    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1063]232    R_FIG=${SCRATCHDIR}/IGCM_OUT
[1057]233
234    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
235
236  else
237    #====================================================
238    #- R_OUT
[1063]239    R_OUT=${ARCHIVE}/IGCM_OUT
[1057]240
241    #====================================================
242    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
[1063]243    R_FIG=${STORAGE}/IGCM_OUT
[1057]244  fi
245
246  #====================================================
247  #- R_BUF (ONLY FOR double copy an scratch)
[1063]248  R_BUF=${SCRATCHDIR}/IGCM_OUT
[1057]249
250  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
251  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
252  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
253
254  IGCM_debug_PopStack "IGCM_sys_defineArchives"
[725]255}
256
[622]257#D-#==================================================
258#D-function IGCM_sys_RshMaster
[623]259#D-* Purpose: Connection to frontend machine.
[622]260#D-* Examples:
261#D-
262function IGCM_sys_RshMaster {
[623]263  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
264  /bin/ksh <<-EOF
[622]265    export libIGCM=${libIGCM}
266    export DEBUG_debug=${DEBUG_debug}
267    . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
268    . ${libIGCM}/libIGCM_card/libIGCM_card.ksh
269    ${@}
270EOF
[623]271  if [ $? -gt 0 ] ; then
272    echo "IGCM_sys_RshMaster : erreur."
273    IGCM_debug_Exit "IGCM_sys_RshMaster"
274  fi
275  IGCM_debug_PopStack "IGCM_sys_RshMaster"
[622]276}
277
278#D-#==================================================
279#D-function IGCM_sys_RshArchive
280#D-* Purpose: Archive rsh command
281#D-* Examples:
282#D-
283function IGCM_sys_RshArchive {
[623]284  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
285  /bin/ksh <<-EOF
[622]286    ${@}
287EOF
[811]288  status=$?
289  if [ ${status} -gt 0 ] ; then
290    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
[623]291    IGCM_debug_Exit "IGCM_sys_RshArchive"
292  fi
293  IGCM_debug_PopStack "IGCM_sys_RshArchive"
[622]294}
295
296#D-#==================================================
[845]297#D-function IGCM_sys_RshArchive_NoError
298#D-* Purpose: Archive rsh command, without error
299#D-*          used only in monitoring.job
300#D-* Examples:
301#D-
302function IGCM_sys_RshArchive_NoError {
303  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
304  /bin/ksh <<-EOF
305    ${@} 2> \dev\null
306EOF
307  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
308}
309
310#D-#==================================================
[622]311#D-function IGCM_sys_RshPost
312#D-* Purpose: Post-process rsh command
313#D-* Examples:
314#D-
315function IGCM_sys_RshPost {
[623]316  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
317  if ( $DEBUG_sys ) ; then
318    echo "IGCM_sys_RshPost :" $@
319  fi
320  # keep standard input (stdin) for the loop onto temporary file
[811]321  cat >/tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
[623]322
[811]323  /bin/ksh </tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
[623]324  if [ $? -gt 0 ] ; then
325    echo "IGCM_sys_RshPost : erreur."
326    IGCM_debug_Exit "IGCM_sys_RshPost"
327  fi
328  # delete temporary file
[811]329  \rm /tmp/tmp_IGCM_sys_RshPost_$$_${LOGNAME}
[623]330
331# ============ FRONTEND  END  ============ #
332
333# ============ CESIUM START ============ #
334#  typeset NB_ESSAI DELAI status i
335#  if [ "X$( grep rebuild_from tmp_IGCM_sys_RshPost_$$ )" != "X" ] ; then
336#    #little hack so that rebuild submission is done on titane not an cesium
337#
338#    libIGCM_POST_sed=$( echo $libIGCM_POST | sed 's/\//\\\//g' )
339#    POST_DIR_sed=$( echo ${POST_DIR} | sed 's/\//\\\//g' )
340#    sed "s/IGCM_sys_QsubPost/IGCM_sys_Qsub/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
341#    sed "s/ rebuild_fromWorkdir/ ${libIGCM_POST_sed}\/rebuild_fromWorkdir.job/g" tmp.txt > tmp_IGCM_sys_RshPost_$$
342#    sed "s/ rebuild_fromArchive/ ${libIGCM_POST_sed}\/rebuild_fromArchive.job/g" tmp_IGCM_sys_RshPost_$$ > tmp.txt
343#    sed "s/Script_Post_Output=/Script_Output=${POST_DIR_sed}\//g" tmp.txt > tmp_IGCM_sys_RshPost_$$
344#    \mv tmp.txt tmp_IGCM_sys_RshPost_$$
345#
346#    echo cat tmp_IGCM_sys_RshPost_$$ AFTER
347#    cat tmp_IGCM_sys_RshPost_$$
348#
349#    /bin/ksh <tmp_IGCM_sys_RshPost_$$
350#    if [ $? -gt 0 ] ; then
351#      echo "IGCM_sys_RshPost : erreur."
352#      IGCM_debug_Exit "IGCM_sys_RshPost"
353#    fi
354#    # delete temporary file
355#    \rm tmp_IGCM_sys_RshPost_$$
356#
357#  else
358#    # number of tentative
359#    NB_ESSAI=10
360#    # time delay between tentative
361#    DELAI=10
362#    i=0
363#    while [ $i -ne $NB_ESSAI ] ; do
364#      ssh -t titane996 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
365#      status=$?
366#      if [ ${status} -ne 0 ];
367#      then
[785]368#        sleep $DELAI
[623]369#      else
[785]370#        break
[623]371#      fi
372#      let i=$i+1
373#    done
374#    # delete temporary file
375#    \rm tmp_IGCM_sys_RshPost_$$
376#
377#    if [ ${status} -gt 0 ] ; then
378#      echo "IGCM_sys_RshPost : erreur."
379#      IGCM_debug_Exit "IGCM_sys_RshPost"
380#    fi
381#  fi
382
383# ============ CESIUM  END  ============ #
384
385  IGCM_debug_PopStack "IGCM_sys_RshPost"
[622]386}
387
388#D-#==================================================
389#D-function IGCM_sys_SendMail
390#D-* Purpose: Send mail when simulation is over
391#D-* Examples:
392#D-
393function IGCM_sys_SendMail {
[623]394  IGCM_debug_PushStack "IGCM_sys_SendMail" $@
[922]395
[623]396  if ( $DEBUG_sys ) ; then
397    echo "IGCM_sys_SendMail :" $@
398  fi
[622]399
[922]400  if [ X${1} = XAccounting ] ; then
401    status=Accounting
402    mailText=jobAccounting.mail
403  elif ( ${ExitFlag} ) ; then
[623]404    status=failed
[922]405    mailText=jobEnd.mail
[623]406  else
407    status=completed
[922]408    mailText=jobEnd.mail
[623]409  fi
[622]410
[922]411  # Update selected mail template
412  while read -r line; do
413    eval echo $line >> mail.txt ;
414  done < ${libIGCM}/libIGCM_sys/${mailText}
[622]415
[806]416  if [ ! -z ${config_UserChoices_MailName} ] ; then
[922]417    mail -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} < mail.txt
[806]418  elif [ -f ~/.forward ] ; then
[922]419    mail -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < mail.txt
[806]420  fi
[811]421
[806]422  sleep 10
[922]423  rm -f mail.txt
[811]424
[623]425  IGCM_debug_PopStack "IGCM_sys_SendMail"
[622]426}
427
428#D-#==================================================
429#D-function IGCM_sys_Mkdir
430#D-* Purpose: Master locale mkdir command
431#D-* Examples:
432#D-
433function IGCM_sys_Mkdir {
[623]434  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
435  if ( $DEBUG_sys ) ; then
436    echo "IGCM_sys_Mkdir :" $@
437  fi
438  if [ ! -d ${1} ]; then
439    \mkdir -p $1
440    if [ $? -gt 0 ] ; then
441      echo "IGCM_sys_Mkdir : erreur."
442      IGCM_debug_Exit "IGCM_sys_Mkdir"
[622]443    fi
[623]444  fi
445  # vérification :
446  if [ ! -d ${1} ] ; then
447    echo "IGCM_sys_Mkdir : erreur."
448    IGCM_debug_Exit "IGCM_sys_Mkdir"
449  fi
450  IGCM_debug_PopStack "IGCM_sys_Mkdir"
[622]451}
452
453#D-#==================================================
454#D-function IGCM_sys_MkdirArchive
455#D-* Purpose: Mkdir on Archive
456#D-* Examples:
457#D-
458function IGCM_sys_MkdirArchive {
[623]459  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
460  if ( $DEBUG_sys ) ; then
461    echo "IGCM_sys_MkdirArchive :" $@
462  fi
463  #- creation de repertoire sur le serveur fichier
464  if [ ! -d ${1} ]; then
465    \mkdir -p $1
[811]466    status=$?
467
468    if [ ${status} -gt 0 ] ; then
469      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
[623]470      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
[622]471    fi
[623]472  fi
473  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
[622]474}
475
476#D-#==================================================
477#D-function IGCM_sys_MkdirWork
478#D-* Purpose: Mkdir on Work
479#D-* Examples:
480#D-
481function IGCM_sys_MkdirWork {
[623]482  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
483  if ( $DEBUG_sys ) ; then
484    echo "IGCM_sys_MkdirWork :" $@
485  fi
486  #- creation de repertoire sur le serveur fichier
487  if [ ! -d ${1} ]; then
488    \mkdir -p $1
489    if [ $? -gt 0 ] ; then
490      echo "IGCM_sys_MkdirWork : erreur."
491      IGCM_debug_Exit "IGCM_sys_MkdirWork"
[622]492    fi
[623]493  fi
494  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
[622]495}
496
497#D-#==================================================
498#D-function IGCM_sys_Cd
499#D-* Purpose: master cd command
500#D-* Examples:
501#D-
502function IGCM_sys_Cd {
[623]503  IGCM_debug_PushStack "IGCM_sys_Cd" $@
504  if ( $DEBUG_sys ) ; then
505    echo "IGCM_sys_Cd :" $@
506  fi
507  \cd $1
508  if [ $? -gt 0 ] ; then
509    echo "IGCM_sys_Cd : erreur."
510    IGCM_debug_Exit "IGCM_sys_Cd"
511  fi
512  IGCM_debug_PopStack "IGCM_sys_Cd"
[622]513}
514
515#D-#==================================================
516#D-function IGCM_sys_Chmod
517#D-* Purpose: Chmod
518#D-* Examples:
519#D-
520function IGCM_sys_Chmod {
[1057]521  IGCM_debug_PushStack "IGCM_sys_Chmod" $@
[623]522  if ( $DEBUG_sys ) ; then
523    echo "IGCM_sys_Chmod :" $@
524  fi
[809]525  \chmod $@
526  if [ $? -gt 0 ] ; then
527    echo "IGCM_sys_Chmod : erreur."
528    IGCM_debug_Exit "IGCM_sys_Chmod"
[623]529  fi
530  IGCM_debug_PopStack "IGCM_sys_Chmod"
[622]531}
532
533#D-#==================================================
534#D-function IGCM_sys_FileSize
535#D-* Purpose: Filesize
536#D-* Examples:
537#D-
538function IGCM_sys_FileSize {
[623]539  IGCM_debug_PushStack "IGCM_sys_FileSize" $@
[622]540
[623]541  typeset sizeF
542  set +A sizeF -- $( ls -la ${1} )
543  if [ $? -gt 0 ] ; then
544    IGCM_debug_Exit "IGCM_sys_FileSize"
545  fi
546  eval ${2}=${sizeF[4]}
[622]547
[623]548  IGCM_debug_PopStack "IGCM_sys_FileSize"
[622]549}
550
551#D-#==================================================
552#D-function IGCM_sys_TestDir
553#D-* Purpose: Test Directory that must exists
554#D-* Examples:
555#D-
556function IGCM_sys_TestDir {
[623]557  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
558  if ( $DEBUG_sys ) ; then
559    echo "IGCM_sys_TestDir :" $@
560  fi
561  typeset ExistFlag
562  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
563  IGCM_debug_PopStack "IGCM_sys_TestDir"
[622]564
[623]565  return ${ExistFlag}
[622]566}
567
568#D-#==================================================
569#D-function IGCM_sys_TestDirArchive
570#D-* Purpose: Test Directory that must exists on Archive
571#D-* Examples:
572#D-
573function IGCM_sys_TestDirArchive {
[623]574  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
575  if ( $DEBUG_sys ) ; then
576    echo "IGCM_sys_TestDirArchive :" $@
577  fi
578  typeset ExistFlag
579  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
580  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
581  return ${ExistFlag}
[622]582}
583
584#D-#==================================================
[623]585#D-function IGCM_sys_IsFileArchived
586#D-* Purpose: Test file that must NOT EXISTS on Archive
587#D-* Examples:
588#D-
589function IGCM_sys_IsFileArchived {
590  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
591  if ( $DEBUG_sys ) ; then
592    echo "IGCM_sys_IsFileArchived :" $@
593  fi
594  typeset IsArchivedFlag
[818]595  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
[623]596  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
597
598  return ${IsArchivedFlag}
599}
600
601#D-#==================================================
[622]602#D-function IGCM_sys_TestFileArchive
603#D-* Purpose: Test file that must NOT EXISTS on Archive
604#D-* Examples:
605#D-
606function IGCM_sys_TestFileArchive {
[623]607  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
608  typeset ExistFlag
609  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
610  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
[622]611
[623]612  return ${ExistFlag}
[622]613}
614
615#D-#==================================================
[623]616#D-function IGCM_sys_TestFileBuffer
617#D-* Purpose: Test file that must NOT EXISTS on Buffer
618#D-* Examples:
619#D-
620function IGCM_sys_TestFileBuffer {
621  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
622  typeset ExistFlag
623  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
624  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
625
626  return ${ExistFlag}
627}
628
629#D-#==================================================
[622]630#D-function IGCM_sys_CountFileArchive
631#D-* Purpose: Count files on Archive filesystem
632#D-* Examples:
633#D-
634function IGCM_sys_CountFileArchive {
[623]635  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
636  ls ${@} 2>/dev/null | wc -l
637  if [ $? -gt 0 ] ; then
638    echo "IGCM_sys_CountFileArchive : erreur."
639  fi
640  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
[622]641}
642
643#D-#==================================================
[623]644#D-function IGCM_sys_CountFileBuffer
645#D-* Purpose: Count files on Scratch filesystem
646#D-* Examples:
647#D-
648function IGCM_sys_CountFileBuffer {
649  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
650  ls ${@} 2>/dev/null | wc -l
651  if [ $? -gt 0 ] ; then
652    echo "IGCM_sys_CountFileBuffer : erreur."
653  fi
654  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
655}
656
657#D-#==================================================
[622]658#D-function IGCM_sys_Tree
659#D-* Purpose: Tree directories with files on ${ARCHIVE}
660#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
661#D-
662function IGCM_sys_Tree {
[623]663  IGCM_debug_PushStack "IGCM_sys_Tree" $@
664  if ( $DEBUG_sys ) ; then
665    echo "IGCM_sys_Tree :" $@
666  fi
[622]667
[623]668  \ls -lR ${@}
[622]669
[623]670  IGCM_debug_PopStack "IGCM_sys_Tree"
[622]671}
672
673#D-#==================================================
674#D-function IGCM_sys_Tar
[623]675#D-* Purpose: master tar command
[622]676#D-* Examples:
677#D-
678function IGCM_sys_Tar {
[623]679  IGCM_debug_PushStack "IGCM_sys_Tar" $@
680  if ( $DEBUG_sys ) ; then
681    echo "IGCM_sys_Tar :" $@
682  fi
683  \tar cf $@
684  if [ $? -gt 0 ] ; then
685    echo "IGCM_sys_Tar : erreur."
686    IGCM_debug_Exit "IGCM_sys_Tar"
687  fi
688  IGCM_debug_PopStack "IGCM_sys_Tar"
[622]689}
690
691#D-#==================================================
692#D-function IGCM_sys_UnTar
693#D-* Purpose: master un-tar command
694#D-* Examples:
695#D-
696function IGCM_sys_UnTar {
[623]697  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
698  if ( $DEBUG_sys ) ; then
699    echo "IGCM_sys_UnTar :" $@
700  fi
701  \tar xvf $1
702  if [ $? -gt 0 ] ; then
703    echo "IGCM_sys_UnTar : erreur."
704    IGCM_debug_Exit "IGCM_sys_UnTar"
705  fi
706  IGCM_debug_PopStack "IGCM_sys_UnTar"
[622]707}
708
709#D-#==================================================
710#D-function IGCM_sys_Qsub
711#D-* Purpose: Qsub new job
712#D-* Examples:
713#D-
714function IGCM_sys_Qsub {
[623]715  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
716  if ( $DEBUG_sys ) ; then
717    echo "IGCM_sys_Qsub :" $@
718  fi
[811]719  typeset options status
[923]720  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
[622]721
[948]722  /usr/bin/ccc_msub ${options} $1 > /tmp/out_command.$$ 2>&1
[811]723  status=$?
[948]724
725  cat /tmp/out_command.$$
[811]726  if [ ${status} -gt 0 ] ; then
727    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
[623]728    IGCM_debug_Exit "IGCM_sys_Qsub"
[948]729  else
730    JobID=$( gawk {'print $4'} /tmp/out_command.$$ )
[623]731  fi
732  IGCM_debug_PopStack "IGCM_sys_Qsub"
[622]733}
734
735#D-#==================================================
736#D-function IGCM_sys_QsubPost
737#D-* Purpose: Qsub new job on scalaire
738#D-* Examples:
739#D-
740function IGCM_sys_QsubPost {
[623]741  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
742  if ( $DEBUG_sys ) ; then
743    echo "IGCM_sys_QsubPost :" $@
744  fi
[923]745  typeset options status
[880]746  MY_PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=${DEFAULT_PROJECT}} | cut -d@ -f1 )
[923]747
[1062]748  # EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
749  case $( hostname -s ) in
750  curie*)
751    options="-Q normal -A ${MY_PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
752  airain*)
753    options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
754  esac
755
[948]756  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > /tmp/out_command.$$ 2>&1
[811]757  status=$?
[948]758
759  cat /tmp/out_command.$$
[811]760  if [ ${status} -gt 0 ] ; then
[923]761    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
[623]762    IGCM_debug_Exit "IGCM_sys_QsubPost"
[948]763  else
764    JobID=$( gawk {'print $4'} /tmp/out_command.$$ )
[623]765  fi
766  IGCM_debug_PopStack "IGCM_sys_QsubPost"
[622]767}
768
769#D-*************************
[623]770#D- File transfer functions
[622]771#D-*************************
772#D-
773
774#D-#==================================================
775#D-function IGCM_sys_Rsync_out
776#D-* Purpose: treat return val of rsync
777#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
778#D-  Error values and explanations can depend on your system version.
779function IGCM_sys_Rsync_out {
[719]780  status=$1
781  if [ ! $status ] ; then
[623]782    echo "rsync error !"
783  fi
[622]784
[623]785  if [ $MYLANG = "fr" ]; then
[719]786    case $status in
[623]787    0)  return ;;
788    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
789      echo "Erreur de syntaxe ou d'utilisation."
790      return;;
791    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
792      echo "Incompatibilité de protocole."
793      return;;
794    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
795      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
796      echo "répertoires"
797      return;;
798    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
799      echo "Action demandée non supportée : une tentative de manipulation de"
800      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
801      echo "été faite ; ou une option qui est supportée par le  client  mais"
802      echo "pas par le serveur a été spécifiée."
803      return;;
804    10) echo "Erreur de rsync ; RERR_SOCKETIO"
805      echo "Erreur dans le socket d'entrée sortie"
806      return;;
807    11) echo "Erreur de rsync ; RERR_FILEIO"
808      echo "Erreur d'entrée sortie fichier"
809      return;;
810    12) echo "Erreur de rsync ; RERR_STREAMIO"
811      echo "Erreur dans flux de donnée du protocole rsync"
812      return;;
813    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
814      echo "Erreur avec les diagnostics du programme"
815      return;;
816    14) echo "Erreur de rsync ; RERR_IPC"
817      echo "Erreur dans le code IPC"
818      return;;
819    20) echo "Erreur de rsync ; RERR_SIGNAL"
820      echo "SIGUSR1 ou SIGINT reçu"
821      return;;
822    21) echo "Erreur de rsync ; RERR_WAITCHILD"
823      echo "Une erreur retournée par waitpid()"
824      return;;
825    22) echo "Erreur de rsync ; RERR_MALLOC"
826      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
827      return;;
828    23) echo ""
829      echo "Erreur fichier inexistant"
830      return;;
831    30) echo "Erreur de rsync ; RERR_TIMEOUT"
832      echo "Temps d'attente écoulé dans l'envoi/réception de données"
833      return;;
[719]834    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $status
[623]835      return;;
836    esac
837  elif [ $MYLANG = "en" ] ; then
[719]838    case $status in
[623]839    0)  return;;
840    1)  echo "rsync error : Syntax or usage error "
841      return;;
842    2)  echo "rsync error : Protocol incompatibility "
843      return;;
844    3)  echo "rsync error : Errors selecting input/output files, dirs"
845      return;;
846    4)  echo "rsync error : Requested action not supported: an attempt"
847      echo "was made to manipulate 64-bit files on a platform that cannot support"
848      echo "them; or an option was specified that is supported by the client and"
849      echo "not by the server."
850      return;;
851    5)  echo "rsync error : Error starting client-server protocol"
852      return;;
853    10) echo "rsync error : Error in socket I/O "
854      return;;
855    11) echo "rsync error : Error in file I/O "
856      return;;
857    12) echo "rsync error : Error in rsync protocol data stream "
858      return;;
859    13) echo "rsync error : Errors with program diagnostics "
860      return;;
861    14) echo "rsync error : Error in IPC code "
862      return;;
863    20) echo "rsync error : Received SIGUSR1 or SIGINT "
864      return;;
865    21) echo "rsync error : Some error returned by waitpid() "
866      return;;
867    22) echo "rsync error : Error allocating core memory buffers "
868      return;;
869    23) echo "rsync error : Partial transfer due to error"
870      return;;
871    24) echo "rsync error : Partial transfer due to vanished source files"
872      return;;
873    30) echo "rsync error : Timeout in data send/receive "
874      return;;
[719]875    *)  echo "rsync error : return code of rsync unknown :" $status
[623]876      return;;
877    esac
878  else
879    echo "unknown language $MYLANG."
880    return
881  fi
[622]882}
883
884#D-#==================================================
885#D-function IGCM_sys_Miror_libIGCM
[811]886#D-* Purpose: Mirror libIGCM PATH and lib to frontend
[622]887#D-* Examples:
888#D-
889function IGCM_sys_Mirror_libIGCM {
[623]890  IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
891  if ( $DEBUG_sys ) ; then
892    echo "IGCM_sys_Mirror_libIGCM"
893  fi
[622]894
[773]895  typeset status
[622]896
[623]897  mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
[622]898
[938]899  echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > /tmp/out_command.$$ 2>&1
900  ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> /tmp/out_command.$$ 2>&1
[715]901  status=$?
[622]902
[715]903  if [ ${status} -gt 0 ] ; then
[811]904    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
[938]905    cat /tmp/out_command.$$
[623]906  fi
907  IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
[622]908}
909
910#D-#==================================================
911#D-function IGCM_sys_Cp
912#D-* Purpose: generic cp
913#D-* Examples:
914#D-
915function IGCM_sys_Cp {
[623]916  IGCM_debug_PushStack "IGCM_sys_Cp" $@
917  if ( $DEBUG_sys ) ; then
918    echo "IGCM_sys_Cp :" $@
919  fi
[622]920
[715]921  typeset status
[622]922
[938]923  echo cp $@ > /tmp/out_command.$$ 2>&1
924  \cp $@ >> /tmp/out_command.$$ 2>&1
[715]925  status=$?
[622]926
[715]927  if [ ${status} -gt 0 ] ; then
928    echo "IGCM_sys_Cp : error code ${status}"
[938]929    cat /tmp/out_command.$$
[623]930    IGCM_debug_Exit "IGCM_sys_Cp"
931  else
[938]932    \rm /tmp/out_command.$$
[623]933  fi
934  IGCM_debug_PopStack "IGCM_sys_Cp"
[622]935}
936
937#D-#==================================================
938#D-function IGCM_sys_Rm
939#D-* Purpose: generic rm
940#D-* Examples:
941#D-
942function IGCM_sys_Rm {
[1057]943  IGCM_debug_PushStack "IGCM_sys_Rm" $@
[623]944  if ( $DEBUG_sys ) ; then
945    echo "IGCM_sys_Rm :" $@
946  fi
[622]947
[715]948  typeset status
[622]949
[938]950  echo rm $@ > /tmp/out_command.$$ 2>&1
951  \rm $@ >> /tmp/out_command.$$ 2>&1
[715]952  status=$?
[622]953
[715]954  if [ ${status} -gt 0 ] ; then
955    echo "IGCM_sys_Rm : error code ${status}"
[938]956    cat /tmp/out_command.$$
[623]957    IGCM_debug_Exit "IGCM_sys_Rm"
958  else
[938]959    \rm /tmp/out_command.$$
[623]960  fi
961  IGCM_debug_PopStack "IGCM_sys_Rm"
[622]962}
963
964#D-#==================================================
965#D-function IGCM_sys_RmRunDir
966#D-* Purpose: rm tmpdir (dummy function most of the time batch
967#D-                      scheduler will do the job)
968#D-* Examples:
969#D-
970function IGCM_sys_RmRunDir {
[1057]971  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
[623]972  if ( $DEBUG_sys ) ; then
973    echo "IGCM_sys_RmRunDir :" $@
974  fi
[622]975
[715]976  typeset status
[622]977
[938]978  echo rm $@ > /tmp/out_command.$$ 2>&1
979  \rm $@ >> /tmp/out_command.$$ 2>&1
[715]980  status=$?
[622]981
[715]982  if [ ${status} -gt 0 ] ; then
[811]983    echo "IGCM_sys_RmRunDir : rm error code is ${status}."
[938]984    cat /tmp/out_command.$$
[623]985    IGCM_debug_Exit "IGCM_sys_RmRunDir"
986  else
[938]987    \rm /tmp/out_command.$$
[623]988  fi
989  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
[622]990}
991
992#D-#==================================================
993#D-function IGCM_sys_Mv
994#D-* Purpose: generic move
995#D-* Examples:
996#D-
997function IGCM_sys_Mv {
[623]998  IGCM_debug_PushStack "IGCM_sys_Mv" $@
999  if ( $DEBUG_sys ) ; then
1000    echo "IGCM_sys_Mv :" $@
1001  fi
[622]1002
[623]1003  if [ $DRYRUN = 0 ]; then
[622]1004
[715]1005    typeset status
[623]1006
[938]1007    echo mv $@ > /tmp/out_command.$$ 2>&1
1008    \mv $@ >> /tmp/out_command.$$ 2>&1
[715]1009    status=$?
[623]1010
[715]1011    if [ ${status} -gt 0 ] ; then
1012      echo "IGCM_sys_Mv : error code ${status}"
[938]1013      cat /tmp/out_command.$$
[623]1014      IGCM_debug_Exit "IGCM_sys_Mv"
[622]1015    else
[938]1016      \rm /tmp/out_command.$$
[622]1017    fi
[623]1018  fi
[622]1019
[623]1020  IGCM_debug_PopStack "IGCM_sys_Mv"
[622]1021}
1022
1023#D-#==================================================
1024#D-function IGCM_sys_Put_Dir
1025#D-* Purpose: Copy a complete directory on $(ARCHIVE)
1026#D-* Examples:
1027#D-
1028function IGCM_sys_Put_Dir {
[623]1029  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
1030  if ( $DEBUG_sys ) ; then
1031    echo "IGCM_sys_Put_Dir :" $@
1032  fi
1033  if [ $DRYRUN = 0 ]; then
1034    if [ ! -d ${1} ] ; then
1035      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
1036      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
1037      return
[622]1038    fi
1039
[715]1040    typeset status
[622]1041
[623]1042    # Only if we use rsync
1043    #IGCM_sys_TestDirArchive $( dirname $2 )
1044    #
1045    #USUAL WAY
[938]1046    \cp -r $1 $2 > /tmp/out_command.$$ 2>&1
[715]1047    status=$?
[622]1048
[715]1049    if [ ${status} -gt 0 ] ; then
[811]1050      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
[938]1051      cat /tmp/out_command.$$
[623]1052      IGCM_debug_Exit "IGCM_sys_Put_Dir"
[622]1053    else
[938]1054      \rm /tmp/out_command.$$
[622]1055    fi
[623]1056  fi
1057  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
[622]1058}
1059
1060#D-#==================================================
1061#D-function IGCM_sys_Get_Dir
[623]1062#D-* Purpose: Copy a complete directory from ${ARCHIVE}
[622]1063#D-* Examples:
1064#D-
1065function IGCM_sys_Get_Dir {
[623]1066  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
1067  if ( $DEBUG_sys ) ; then
1068    echo "IGCM_sys_Get_Dir :" $@
1069  fi
1070  if [ $DRYRUN = 0 ]; then
1071#    if [ ! -d ${1} ] ; then
1072#      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
1073#      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1074#      return
1075#    fi
[622]1076
[708]1077    typeset NB_ESSAI DELAI status i
1078    # number of tentative
1079    NB_ESSAI=3
1080    # time delay between tentative
1081    DELAI=2
[622]1082
[676]1083    # Only if we use rsync
1084    #IGCM_sys_TestDirArchive $( dirname $2 )
1085    #
[708]1086    # USUAL WAY
1087    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
[713]1088    ccc_hsm get -r $1
[622]1089
[709]1090    i=0
[708]1091    while [ $i -lt $NB_ESSAI ] ; do
[938]1092      \cp -ur $1 $2 >> /tmp/out_command.$$ 2>&1
[708]1093      status=$?
[811]1094      if [ ${status} -gt 0 ] ; then
[713]1095        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
[708]1096        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
1097        sleep $DELAI
1098      else
1099        break
1100      fi
1101      (( i = i + 1 ))
1102    done
1103
1104    if [ ${status} -gt 0 ] ; then
[623]1105      echo "IGCM_sys_Get_Dir : error."
[938]1106      cat /tmp/out_command.$$
[623]1107      IGCM_debug_Exit "IGCM_sys_Get_Dir"
[622]1108    else
[938]1109      \rm /tmp/out_command.$$
[622]1110    fi
[623]1111  fi
1112  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
[622]1113}
1114
1115#D-#==================================================
1116#D-function IGCM_sys_Get_Master
1117#D-* Purpose: Copy a complete directory from MASTER filesystem
1118#D-* Examples:
1119#D-
1120function IGCM_sys_Get_Master {
[623]1121  IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
1122  if ( $DEBUG_sys ) ; then
1123    echo "IGCM_sys_Get_Master :" $@
1124  fi
1125  if [ $DRYRUN = 0 ]; then
[640]1126    if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
[623]1127      echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
1128      IGCM_debug_PopStack "IGCM_sys_Get_Master"
1129      return
[622]1130    fi
1131
[708]1132    typeset NB_ESSAI DELAI status i
1133    # number of tentative
1134    NB_ESSAI=3
1135    # time delay between tentative
1136    DELAI=2
[622]1137
[709]1138    i=0
[708]1139    while [ $i -lt $NB_ESSAI ] ; do
[938]1140      \cp -urL $1 $2 > /tmp/out_command.$$ 2>&1
[708]1141      status=$?
1142      if [ ${status} -gt 0 ]; then
[713]1143        IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
[708]1144        IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
1145        sleep $DELAI
1146      else
1147        break
1148      fi
1149      (( i = i + 1 ))
1150    done
[622]1151
[708]1152    if [ ${status} -gt 0 ] ; then
[623]1153      echo "IGCM_sys_Get_Master : error."
[938]1154      cat /tmp/out_command.$$
[623]1155      IGCM_debug_Exit "IGCM_sys_Get_Master"
[622]1156    else
[938]1157      \rm /tmp/out_command.$$
[622]1158    fi
[623]1159  fi
1160  IGCM_debug_PopStack "IGCM_sys_Get_Master"
[622]1161}
1162
[689]1163#====================================================
1164#- Call IGCM_sys_Mirror_libIGCM now !
1165if ( $MirrorlibIGCM ) ; then
1166  IGCM_sys_Mirror_libIGCM
1167fi
1168
[622]1169#D-#==================================================
1170#D-function IGCM_sys_Put_Rest
[623]1171#D-* Purpose: Put computied restarts on ${ARCHIVE}.
[622]1172#D-           File and target directory must exist.
1173#D-* Examples:
1174#D-
1175function IGCM_sys_Put_Rest {
[623]1176  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1177  if ( $DEBUG_sys ) ; then
1178    echo "IGCM_sys_Put_Rest :" $@
1179  fi
1180  if [ $DRYRUN = 0 ]; then
1181    if [ ! -f ${1} ] ; then
1182      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1183      IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]1184    fi
1185
[715]1186    typeset status
[623]1187    #
1188    # USUAL WAY
[938]1189    \cp $1 $2 > /tmp/out_command.$$ 2>&1
[715]1190    status=$?
[622]1191
[623]1192#       #RSYNC WITH NETWORK SSH CALL
[938]1193#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > /tmp/out_command.$$ 2>&1
1194#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> /tmp/out_command.$$ 2>&1
[622]1195
[623]1196#       #RSYNC WITH NFS USE
[938]1197#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > /tmp/out_command.$$ 2>&1
1198#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> /tmp/out_command.$$ 2>&1
[622]1199
[715]1200#       status=$?
1201#       IGCM_sys_Rsync_out $status
[622]1202
[938]1203#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk /tmp/out_command.$$
[715]1204#       (( status=status+$? ))
[623]1205
[715]1206    if [ ${status} -gt 0 ] ; then
1207      echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
[765]1208      [ -f ${1} ] && ls -l ${1}
[759]1209      [ -f ${2} ] && ls -l ${2}
1210      [ -f ${2}/${1} ] && ls -l ${2}/${1}
[938]1211      cat /tmp/out_command.$$
[623]1212      IGCM_debug_Exit "IGCM_sys_Put_Rest"
[622]1213    else
[759]1214
1215      if [ X${JobType} = XRUN ] ; then
1216        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1217        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1218      fi
1219
[938]1220      \rm /tmp/out_command.$$
[622]1221    fi
[623]1222  fi
1223  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
[622]1224}
1225
1226#D-#==================================================
[623]1227#D-function IGCM_sys_PutBuffer_Rest
1228#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1229#D-           File and target directory must exist.
1230#D-* Examples:
1231#D-
1232function IGCM_sys_PutBuffer_Rest {
1233  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1234  if ( $DEBUG_sys ) ; then
1235    echo "IGCM_sys_PutBuffer_Rest :" $@
1236  fi
1237  if [ $DRYRUN = 0 ]; then
1238    if [ ! -f ${1} ] ; then
1239      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1240      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1241    fi
1242
[715]1243    typeset status
[623]1244    #
1245    # USUAL WAY
[938]1246    \cp $1 $2 > /tmp/out_command.$$ 2>&1
[715]1247    status=$?
[623]1248
[715]1249    if [ ${status} -gt 0 ] ; then
1250      echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
[759]1251      [ -f ${2} ] && ls -l ${2}
1252      [ -f ${2}/${1} ] && ls -l ${2}/${1}
[938]1253      cat /tmp/out_command.$$
[623]1254      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1255    else
[759]1256
1257      if [ X${JobType} = XRUN ] ; then
1258        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1259        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1260      fi
1261
[938]1262      \rm /tmp/out_command.$$
[623]1263    fi
1264  fi
1265  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1266}
1267
1268#D-#==================================================
[801]1269#D-function IGCM_sys_PrepareTaredRestart
1270#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
1271#D-* Examples:
1272#D-
1273function IGCM_sys_PrepareTaredRestart {
1274  IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
[802]1275  if [ $DRYRUN = 0 ]; then
[918]1276    [ ! -f $( basename $1 ) ] && IGCM_sys_Get $1 .
[802]1277  fi
[801]1278  IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
1279}
1280
1281#D-#==================================================
[622]1282#D-function IGCM_sys_Put_Out
[623]1283#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
[622]1284#D-* Examples:
1285#D-
1286function IGCM_sys_Put_Out {
[623]1287  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1288  if ( $DEBUG_sys ) ; then
1289    echo "IGCM_sys_Put_Out :" $@
1290  fi
[667]1291
[711]1292  typeset NB_ESSAI DELAI status i exist skip
[667]1293
1294  # number of tentative
[668]1295  NB_ESSAI=3
[667]1296  # time delay between tentative
[668]1297  DELAI=2
[667]1298
[623]1299  if [ $DRYRUN = 0 ]; then
1300    if [ ! -f ${1} ] ; then
1301      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1302      IGCM_debug_PopStack "IGCM_sys_Put_Out"
1303      return 1
[622]1304    fi
[623]1305    #
1306    IGCM_sys_MkdirArchive $( dirname $2 )
1307    #
[622]1308
[623]1309    #=====================================================
1310    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1311    #=====================================================
[622]1312
[623]1313    #echo ${2} | grep "${R_OUT}" > /dev/null 2>&1
1314    #if [ $? -eq 0 ] ; then
1315    #    typeset WORKPATH FILEPATH
1316    #    WORKPATH=$( dirname $2 | sed -e "s|${R_OUT}|${R_BUF}|" )
1317    #    IGCM_sys_MkdirWork ${WORKPATH}
1318    #    FILEPATH=${WORKPATH}/$( basename $2 )
1319    #    #
1320    #    IGCM_sys_Cp ${1} ${FILEPATH}
1321    #fi
[622]1322
[711]1323    exist=false
1324    skip=false
1325    if [ -f $2 ] ; then
1326      IGCM_debug_Print 1 "$2 already exist"
1327      ccc_hsm get $2
1328      exist=true
1329      if [ "X$( diff $1 $2 )" = X ] ; then
1330        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1331        skip=true
[667]1332      else
[711]1333        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1334        skip=false
[667]1335      fi
[711]1336    fi
1337    #
1338    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1339      IGCM_sys_Chmod u+w $2
1340    fi
[622]1341
[711]1342    if [ X${skip} = Xfalse ] ; then
[719]1343      i=0
[711]1344      while [ $i -lt $NB_ESSAI ] ; do
[1037]1345        if [ $( stat -c %d $1 ) -ne $( stat -c %d $( dirname $2 ) ) ] ; then
1346          # USUAL WAY
1347          \cp $1 $2 > /tmp/out_command.$$ 2>&1
1348          status=$?
1349        else
1350          # NOT SO USUAL WAY
1351          \mv $1 $2 > /tmp/out_command.$$ 2>&1
1352          status=$?
1353        fi
[711]1354        if [ ${status} -gt 0 ]; then
[713]1355          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
[711]1356          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
[765]1357          [ -f ${1} ] && ls -l ${1}
[759]1358          [ -f ${2} ] && ls -l ${2}
1359          [ -f ${2}/${1} ] && ls -l ${2}/${1}
[711]1360          sleep $DELAI
1361        else
1362          break
1363        fi
1364        (( i = i + 1 ))
1365      done
1366    fi
1367
[623]1368#       #RSYNC WITH NETWORK SSH CALL
[938]1369#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > /tmp/out_command.$$ 2>&1
1370#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> /tmp/out_command.$$ 2>&1
[622]1371
[623]1372#       #RSYNC WITH NFS USE
[938]1373#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > /tmp/out_command.$$ 2>&1
1374#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> /tmp/out_command.$$ 2>&1
[622]1375
[711]1376#       status=$?
1377#       IGCM_sys_Rsync_out $status
[622]1378
[938]1379#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk /tmp/out_command.$$
[711]1380#       (( status=status+$? ))
[622]1381
[711]1382    if [ ${status} -gt 0 ] ; then
[623]1383      echo "IGCM_sys_Put_Out : error."
[765]1384      [ -f ${1} ] && ls -l ${1}
[759]1385      [ -f ${2} ] && ls -l ${2}
1386      [ -f ${2}/${1} ] && ls -l ${2}/${1}
[938]1387      cat /tmp/out_command.$$
[623]1388      IGCM_debug_Exit "IGCM_sys_Put_Out"
[622]1389    else
[759]1390
1391      if [ X${JobType} = XRUN ] ; then
1392        if [ X${3} = X ] ; then
1393          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1394          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1395        fi
1396      fi
1397
[938]1398      \rm /tmp/out_command.$$
[622]1399    fi
[623]1400  fi
1401  IGCM_debug_PopStack "IGCM_sys_Put_Out"
1402  return 0
[622]1403}
1404
1405#D-#==================================================
[623]1406#D-function IGCM_sys_PutBuffer_Out
1407#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1408#D-* Examples:
1409#D-
1410function IGCM_sys_PutBuffer_Out {
1411  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1412  if ( $DEBUG_sys ) ; then
1413    echo "IGCM_sys_PutBuffer_Out :" $@
1414  fi
[711]1415
[773]1416  typeset NB_ESSAI DELAI status i exist skip
[711]1417
1418  # number of tentative
1419  NB_ESSAI=3
1420  # time delay between tentative
1421  DELAI=2
1422
[623]1423  if [ $DRYRUN = 0 ]; then
1424    if [ ! -f ${1} ] ; then
1425      echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1426      IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1427      return 1
1428    fi
1429    #
1430    IGCM_sys_Mkdir $( dirname $2 )
1431    #
1432
[711]1433    exist=false
1434    skip=false
1435    if [ -f $2 ] ; then
1436      IGCM_debug_Print 1 "$2 already exist"
1437      exist=true
1438      if [ "X$( diff $1 $2 )" = X ] ; then
1439        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
[811]1440        status=0
[711]1441        skip=true
1442      else
1443        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1444        skip=false
1445      fi
1446    fi
[623]1447    #
[711]1448    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1449      IGCM_sys_Chmod u+w $2
1450    fi
[623]1451
[711]1452    if [ X${skip} = Xfalse ] ; then
1453      i=0
1454      while [ $i -lt $NB_ESSAI ] ; do
[1037]1455        if [ $( stat -c %d $1 ) -ne $( stat -c %d $( dirname $2 ) ) ] ; then
1456          # USUAL WAY
1457          \cp $1 $2 > /tmp/out_command.$$ 2>&1
1458          status=$?
1459        else
1460          # NOT SO USUAL WAY
1461          \mv $1 $2 > /tmp/out_command.$$ 2>&1
1462          status=$?
1463        fi
[711]1464        if [ ${status} -gt 0 ]; then
[713]1465          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
[711]1466          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
[759]1467          [ -f ${2} ] && ls -l ${2}
1468          [ -f ${2}/${1} ] && ls -l ${2}/${1}
[711]1469          sleep $DELAI
1470        else
1471          break
1472        fi
1473        (( i = i + 1 ))
1474      done
1475    fi
1476
1477    if [ ${status} -gt 0 ] ; then
[623]1478      echo "IGCM_sys_PutBuffer_Out : error."
[759]1479      [ -f ${2} ] && ls -l ${2}
1480      [ -f ${2}/${1} ] && ls -l ${2}/${1}
[938]1481      cat /tmp/out_command.$$
[623]1482      IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1483    else
[759]1484
1485      if [ X${JobType} = XRUN ] ; then
1486        if [ X${3} = X ] ; then
1487          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1488          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1489        fi
1490      fi
1491
[938]1492      \rm /tmp/out_command.$$
[623]1493    fi
1494  fi
1495  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1496  return 0
1497}
1498
1499#D-#==================================================
[622]1500#D-function IGCM_sys_Get
1501#D-* Purpose: Get a file from ${ARCHIVE}
1502#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1503#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
1504function IGCM_sys_Get {
[623]1505  IGCM_debug_PushStack "IGCM_sys_Get" $@
[622]1506
[773]1507  typeset DEST dm_liste target file_work
[667]1508  typeset NB_ESSAI DELAI status i
[622]1509
[623]1510  if ( $DEBUG_sys ) ; then
1511    echo "IGCM_sys_Get :" $@
1512  fi
[667]1513
1514  # number of tentative
[668]1515  NB_ESSAI=3
[667]1516  # time delay between tentative
[668]1517  DELAI=2
[667]1518
[623]1519  if [ $DRYRUN -le 2 ]; then
1520    if [ X${1} = X'/l' ] ; then
1521      eval set +A dm_liste \${${2}}
1522    else
1523      eval set +A dm_liste ${1}
[622]1524    fi
[623]1525    eval DEST=\${${#}}
[622]1526
[623]1527    #=====================================================
1528    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1529    #=====================================================
[622]1530
[623]1531    # Is it an R_OUT file (not R_IN) ?
1532    #echo ${dm_liste[0]} | grep "${R_OUT}" > /dev/null 2>&1
1533    #if [ $? -eq 0 ] ; then
[785]1534    #  # Yes  ? then we try to get it in SCRATCHDIR
1535    #  set +A file_work $( echo ${dm_liste[*]} | sed -e "s|${R_OUT}|${R_BUF}|g" )
1536    #  if [ -f ${file_work[0]} ] ; then
1537    #    IGCM_sys_Cp ${file_work[*]} ${DEST}
1538    #    IGCM_debug_PopStack "IGCM_sys_Get"
1539    #    return
1540    #  fi
[623]1541    #fi
[622]1542
[938]1543    ccc_hsm get ${dm_liste[*]} > /tmp/out_command.$$ 2>&1
[715]1544    status=$?
1545    if [ ${status} -gt 0 ] ; then
1546      echo "WARNING IGCM_sys_Get : error code ${status}"
[938]1547      cat /tmp/out_command.$$
[623]1548      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1549    fi
[622]1550
[715]1551    #if [ ${status} -gt 0 ] ; then
[938]1552    #  if [ ! "X$( grep "Lost dmusrcmd connection" /tmp/out_command.$$ )" = "X" ] ; then
1553    #    cat /tmp/out_command.$$
[785]1554    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1555    #    sleep 30
1556    #    echo "We try another time"
[938]1557    ##    dmget ${dm_liste[*]} > /tmp/out_command.$$ 2>&1
1558    #    ccc_hsm get ${dm_liste[*]} > /tmp/out_command.$$ 2>&1
[785]1559    #    status=$?
1560    #    if [ ${status} -gt 0 ] ; then
1561    #      echo "ERROR IGCM_sys_Get : again demigration error :"
[938]1562    #      cat /tmp/out_command.$$
[785]1563    #      IGCM_debug_Exit "IGCM_sys_Get"
[623]1564    #    fi
[785]1565    #  else
1566    #    echo "ERROR IGCM_sys_Get : demigration error :"
[938]1567    #    cat /tmp/out_command.$$
[785]1568    #    IGCM_debug_Exit "IGCM_sys_Get"
1569    #  fi
[623]1570    #fi
[622]1571
[719]1572    #   #RSYNC WITH NETWORK SSH CALL
[938]1573    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > /tmp/out_command.$$ 2>&1
1574    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> /tmp/out_command.$$ 2>&1
[719]1575
1576    #   #RSYNC WITH NFS USE
[938]1577    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > /tmp/out_command.$$ 2>&1
1578    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> /tmp/out_command.$$ 2>&1
[719]1579
1580    #   status=$?
1581    #   IGCM_sys_Rsync_out $status
1582
[938]1583    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk /tmp/out_command.$$
[719]1584    #   (( status=status+$? ))
1585
[623]1586    #USUAL WAY
1587    if [ X${1} = X'/l' ] ; then
1588      for target in ${dm_liste[*]} ; do
1589        local_file=$( basename ${target} )
[668]1590        # test if the target file is present before the loop
1591        IGCM_sys_TestFileArchive ${target}
[715]1592        status=$?
1593        if [ ${status} -gt 0 ] ; then
[701]1594          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
[668]1595          IGCM_debug_Exit "IGCM_sys_Get"
1596        else
[719]1597          i=0
[668]1598          while [ $i -lt $NB_ESSAI ] ; do
[1037]1599            #if [ X${DoLink} = Xtrue ] ; then
1600            #  \ln -s ${target} ${DEST}/${local_file} >> /tmp/out_command.$$ 2>&1
1601            #  status=$?
1602            #  else
1603            #  \cp ${target} ${DEST}/${local_file} >> /tmp/out_command.$$ 2>&1
1604            #  status=$?
1605            #fi
1606            \ln -s ${target} ${DEST}/${local_file} >> /tmp/out_command.$$ 2>&1
[668]1607            status=$?
1608            if [ ${status} -gt 0 ]; then
[713]1609              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
[676]1610              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1611              sleep $DELAI
[668]1612            else
[676]1613              break
[668]1614            fi
1615            (( i = i + 1 ))
1616          done
[719]1617          if [ ${status} -gt 0 ] ; then
1618            echo "IGCM_sys_Get : error"
[938]1619            cat /tmp/out_command.$$
1620            \rm /tmp/out_command.$$
[719]1621            IGCM_debug_Exit "IGCM_sys_Get"
1622          else
[938]1623            \rm /tmp/out_command.$$
[719]1624          fi
[668]1625        fi
[623]1626      done
1627    else
[719]1628      i=0
1629      while [ $i -lt $NB_ESSAI ] ; do
[938]1630        \cp ${dm_liste} ${DEST} >> /tmp/out_command.$$ 2>&1
[719]1631        status=$?
1632        if [ ${status} -gt 0 ]; then
1633          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1634          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1635          sleep $DELAI
1636        else
1637          break
1638        fi
1639        (( i = i + 1 ))
1640      done
1641      if [ ${status} -gt 0 ] ; then
1642        echo "IGCM_sys_Get : error"
[938]1643        cat /tmp/out_command.$$
1644        \rm /tmp/out_command.$$
[719]1645        IGCM_debug_Exit "IGCM_sys_Get"
1646      else
[938]1647        \rm /tmp/out_command.$$
[719]1648      fi
[623]1649    fi
1650  fi
1651  IGCM_debug_PopStack "IGCM_sys_Get"
1652}
[622]1653
[623]1654#D-#==================================================
1655#D-function IGCM_sys_GetBuffer
1656#D-* Purpose: Get a file from ${SCRATCHDIR}
1657#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1658#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1659function IGCM_sys_GetBuffer {
1660  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1661
[773]1662  typeset DEST buf_liste target file_work
[719]1663  typeset NB_ESSAI DELAI status i
[623]1664
1665  if ( $DEBUG_sys ) ; then
1666    echo "IGCM_sys_GetBuffer :" $@
1667  fi
[719]1668
1669  # number of tentative
1670  NB_ESSAI=3
1671  # time delay between tentative
1672  DELAI=2
1673
[623]1674  if [ $DRYRUN -le 2 ]; then
1675    if [ X${1} = X'/l' ] ; then
1676      eval set +A buf_liste \${${2}}
[622]1677    else
[623]1678      eval set +A buf_liste ${1}
[622]1679    fi
[623]1680    eval DEST=\${${#}}
1681
1682    #USUAL WAY
1683    if [ X${1} = X'/l' ] ; then
1684      for target in ${buf_liste[*]} ; do
1685        local_file=$( basename ${target} )
[719]1686        i=0
1687        while [ $i -lt $NB_ESSAI ] ; do
[938]1688          \cp ${target} ${DEST}/${local_file} >> /tmp/out_command.$$ 2>&1
[719]1689          status=$?
1690          if [ ${status} -gt 0 ]; then
1691            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1692            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1693            sleep $DELAI
1694          else
1695            break
1696          fi
1697          (( i = i + 1 ))
1698        done
1699        if [ ${status} -gt 0 ] ; then
1700          echo "IGCM_sys_Get : error"
[938]1701          cat /tmp/out_command.$$
1702          \rm /tmp/out_command.$$
[719]1703          IGCM_debug_Exit "IGCM_sys_GetBuffer"
1704        else
[938]1705          \rm /tmp/out_command.$$
[719]1706        fi
[623]1707      done
1708    else
[719]1709      i=0
1710      while [ $i -lt $NB_ESSAI ] ; do
[938]1711        \cp ${buf_liste} ${DEST} >> /tmp/out_command.$$ 2>&1
[719]1712        status=$?
1713        if [ ${status} -gt 0 ]; then
1714          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1715          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1716          sleep $DELAI
1717        else
1718          break
1719        fi
1720        (( i = i + 1 ))
1721      done
1722      if [ ${status} -gt 0 ] ; then
1723        echo "IGCM_sys_Get : error"
[938]1724        cat /tmp/out_command.$$
1725        \rm /tmp/out_command.$$
[719]1726        IGCM_debug_Exit "IGCM_sys_GetBuffer"
1727      else
[938]1728        \rm /tmp/out_command.$$
[719]1729      fi
[623]1730    fi
1731  fi
1732  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
[622]1733}
1734
1735#D-#==================================================
1736#D-function IGCM_sys_GetDate_FichWork
1737#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1738#D-* Examples:
1739#D-
1740function IGCM_sys_GetDate_FichWork {
[623]1741  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1742  if ( $DEBUG_sys ) ; then
1743    echo "IGCM_sys_GetDate_FichWork :" $@
1744  fi
[955]1745
1746  if [ $# -ge 3 ] ; then
1747    mode=$3
1748    TimeStyle=$4
1749  else
1750    mode="default"
1751    TimeStyle="%Y%m%d%H%M%S"
1752  fi
1753
[623]1754  typeset dateF
[955]1755  set +A dateF -- $( ls -l --full-time --time-style=+"${TimeStyle}" ${1} )
[623]1756
[955]1757  case $mode in
1758    "default")
1759      eval ${2}=${dateF[5]}
1760      ;;
1761    "SplitFields")
1762      eval ${2}="${dateF[5]}\ ${dateF[6]}"
1763      ;;
1764  esac
1765
1766
[773]1767  # donne la date filesys d'un fichier sur la machine work
[623]1768  IGCM_debug_PopStack "IGCM_sys_FichWork"
[622]1769}
1770
1771#D-#==================================================
1772#D-function IGCM_sys_GetDate_FichArchive
[623]1773#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
[622]1774#D-* Examples:
1775#D-
1776function IGCM_sys_GetDate_FichArchive {
[623]1777  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1778  if ( $DEBUG_sys ) ; then
1779    echo "IGCM_sys_GetDate_FichArchive :" $@
1780  fi
1781  typeset dateF
1782  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1783  eval ${2}=${dateF[5]}
1784
1785  IGCM_debug_PopStack "IGCM_sys_FichArchive"
1786}
1787
1788#D-#==================================================
[835]1789#D-function IGCM_sys_GetDate_Monitoring
1790#D-* Purpose: get the last year for which the monitoring has been computed
1791#D-* Examples:
1792#D-
1793function IGCM_sys_GetDate_Monitoring {
1794  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
1795  if ( $DEBUG_sys ) ; then
1796    echo "IGCM_sys_GetDate_Monitoring :" $@
1797  fi
1798
1799  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
1800
1801  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
1802}
1803
1804#D-#==================================================
[623]1805#D-function IGCM_sys_Dods_Rm
1806#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1807#D-* Examples:
1808#D-
1809function IGCM_sys_Dods_Rm {
1810  if ( $DEBUG_sys ) ; then
1811    echo "IGCM_sys_Dods_Rm :" $@
1812  fi
[715]1813  typeset status
[623]1814  if [ $DRYRUN = 0 ]; then
1815
1816#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
1817#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
1818#      echo "Nothing has been done."
1819#      return
1820#    fi
1821
[649]1822    /ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
[715]1823    status=$?
[623]1824
[785]1825#    if [ ${status} -gt 0 ] ; then
1826#      echo "IGCM_sys_Dods_Rm : error."
1827#      cat out_dods_rm
1828#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
1829#    else
1830#      rm out_dods_rm
1831#    fi
[623]1832
1833  fi
[715]1834  return $status
[623]1835}
1836
1837#D-#==================================================
1838#D-function IGCM_sys_Dods_Cp
1839#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1840#D-* Examples:
1841#D-
1842function IGCM_sys_Dods_Cp {
1843  if ( $DEBUG_sys ) ; then
1844    echo "IGCM_sys_Dods_Cp :" $@
1845  fi
[715]1846  typeset status
[623]1847  if [ $DRYRUN = 0 ]; then
1848
1849#    if [ ! -d ${R_SAVE}/${1} ] ; then
1850#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1851#      echo "Nothing has been done."
1852#      return
1853#    fi
1854
[649]1855    /ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
[715]1856    status=$?
[623]1857
[715]1858#       if [ ${status} -gt 0 ] ; then
[623]1859#           echo "IGCM_sys_Dods_Cp : error."
1860#           cat out_dods_cp
1861#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1862#       else
1863#           rm out_dods_cp
1864#       fi
1865
1866  fi
[715]1867  return $status
[623]1868}
1869
1870#D-#==================================================
1871#D-function IGCM_sys_Put_Dods
1872#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
1873#D-* Examples:
1874#D-
1875function IGCM_sys_Put_Dods {
1876  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1877  if ( $DEBUG_sys ) ; then
1878    echo "IGCM_sys_Put_Dods :" $@
1879  fi
[711]1880  #set -vx
[715]1881  typeset status
[623]1882  if [ $DRYRUN = 0 ]; then
1883    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1884      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1885      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1886      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1887      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1888      return
[622]1889    fi
[623]1890
1891    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1892      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1893      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1894      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1895      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1896      return
1897    fi
1898    #
1899    if [ -d ${R_SAVE}/${1} ] ; then
1900      cd ${R_SAVE}
1901    elif [ -d ${R_FIGR}/${1} ] ; then
1902      cd ${R_FIGR}
1903    fi
1904
1905    IGCM_sys_Dods_Rm ${1}
1906    IGCM_sys_Dods_Cp ${1}
[715]1907    status=0
[623]1908
[715]1909    if [ ${status} -gt 0 ] ; then
[623]1910      echo "IGCM_sys_Put_Dods : error."
1911      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1912    fi
1913  fi
1914  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
[622]1915}
1916
1917##############################################################
1918# REBUILD OPERATOR
1919
[811]1920#D-#==================================================
1921#D-function IGCM_sys_rebuild
1922#D-* Purpose: rebuild parallel files
1923#D-* Examples:
1924#D-
[622]1925function IGCM_sys_rebuild {
[1057]1926  IGCM_debug_PushStack "IGCM_sys_rebuild" $@
[623]1927  if ( $DEBUG_sys ) ; then
1928    echo "IGCM_sys_rebuild :" $@
1929  fi
[716]1930
[734]1931  typeset NB_ESSAI DELAI status i firstArg
[716]1932  # number of tentative
1933  NB_ESSAI=3
1934  # time delay between tentative
1935  DELAI=2
1936
1937  i=0
1938  while [ $i -lt $NB_ESSAI ] ; do
[977]1939    ~p86ipsl/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/rebuild -f -o $@ > /tmp/out_command.$$ 2>&1
[716]1940    status=$?
1941    if [ ${status} -gt 0 ] ; then
1942      IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
[938]1943      cat /tmp/out_command.$$
1944      \rm /tmp/out_command.$$
[716]1945      IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[734]1946      firstArg=${1}
1947      \rm ${firstArg}
[733]1948      sleep $DELAI
[716]1949    else
[938]1950      \rm /tmp/out_command.$$
[716]1951      break
1952    fi
1953    (( i = i + 1 ))
1954  done
1955
[725]1956  if [ ${status} -gt 0 ] ; then
[773]1957    echo "IGCM_sys_rebuild : rebuild error code is ${status}"
[623]1958    IGCM_debug_Exit "rebuild"
1959  fi
[622]1960
[623]1961  IGCM_debug_PopStack "IGCM_sys_rebuild"
[622]1962}
1963
[811]1964#D-#==================================================
1965#D-function IGCM_sys_rebuild_station
[912]1966#D-* Purpose: rebuild parallel files describing station
[811]1967#D-* Examples:
1968#D-
[697]1969function IGCM_sys_rebuild_station {
[1057]1970  IGCM_debug_PushStack "IGCM_sys_rebuild_station" $@
[697]1971  typeset i list_opt file_in file_out prefix_invert list_invert
1972  if ( $DEBUG_sys ) ; then
1973    echo "IGCM_sys_rebuild_station :" $@
1974  fi
1975  list_opt=$@
1976
[773]1977  # Invert Axis : t,x -> x,t
[697]1978  #               t,pres,x -> x,t,pres
1979  # So that we can concatenate along x
1980  i=0
1981  for file_in in ${list_opt} ; do
1982    (( i = i + 1))
1983    [ ${i} = 1 ] && file_out=${file_in} && continue
[1046]1984    # detect time counter and do the job only if present
1985    var_unlim=$(ncdump -h ${file_in} | grep UNLIMITED | cut -d ' ' -f 1)
1986    if [ X${var_unlim} = Xtime_counter ] ; then
1987      prefix_invert=$( basename ${file_in} .nc )
1988      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1989      list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1990    fi
[697]1991  done
1992
1993  # Concatenate
1994  IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1995
1996  # Re-ivert file
1997  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1998
[922]1999  # Station re-ordering is too expansive to be run within libIGCM
[697]2000  # This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
2001  # This re-ordering must be done "in memory" by the cmorization process
2002  # Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
2003  # BEGIN reordering
2004
2005  # Only LMDZ text output contains the exact ordering of the station.
2006  # We isolate this in the code below:
2007  #  0  38  -157.5000000000000  70.98591549295774
2008  #  0  54  27.49999999999999   67.18309859154928
2009  #  0  56  -62.50000000000001  82.39436619718309
2010  #  0  79  12.49999999999999   78.59154929577466
2011  #  0  116 -165.0000000000000  76.05633802816901
2012  #  0  117 130.0000000000000   70.98591549295774
2013  #  0  118 110.0000000000000   87.46478873239437
2014  #  1  40  4.999999999999995   51.97183098591550
[1056]2015
[697]2016#  typeset iStation iProc list_opt file_in file_out prefix_invert
2017#  typeset -Z4 j4
2018#  typeset -Z3 j3
2019
2020#  unset list_opt
2021#  set +A list_opt $@
2022
2023  # Filename after rebuild
2024#  file_out=${list_opt[0]}
2025  # Prefix of output files
2026#  prefix_invert=$( basename ${file_out} .nc )
2027  # Number of procs
2028#  num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} | wc -l )
2029
2030#  iProc=0
2031#  while [ ${iProc} -lt ${num_proc} ] ; do
2032    # Array containing Station as a number
2033#    unset proc_stn
[912]2034#    set +A proc_stn $( grep "iophy_mpi rank ip lon lat  $iProc" ${PREFIX}_${Exe_Output} | sed -e "s/iophy_mpi rank ip lon lat //g" | gawk ' {print $2}' )
[697]2035    # Number of stations produced by processor proc
2036#    stationLast=${#proc_stn[*]}
2037    # Proc number on 4 digits
2038#    j4=${iProc}
2039    # Init
2040#    iStation=0
2041#    while [ ${iStation} -lt ${stationLast} ] ; do
2042      # Station number on 3 digits
2043#      j3=${proc_stn[${iStation}]}
2044      # Extract station
[773]2045      # Invert Axis : t,x -> x,t
[697]2046      #               t,pres,x -> x,t,pres
2047      # So that we can concatenate along x
2048#      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
2049#      (( iStation = iStation + 1 ))
2050#    done
2051#    (( iProc = iProc + 1 ))
2052#  done
2053
2054  # Concatenate all station along x
2055#  IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
2056
2057  # Re-invert file
2058#  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
2059
2060  # END reordering
2061
2062  IGCM_debug_PopStack "IGCM_sys_rebuild_station"
2063}
2064
[622]2065############################################################
2066# Activate Running Environnment Variables
2067
[811]2068#D-#==================================================
2069#D-function IGCM_sys_desactiv_variables
2070#D-* Purpose: set environement variables prior to execution
2071#D-* Examples:
2072#D-
[622]2073function IGCM_sys_activ_variables {
[623]2074  IGCM_debug_PushStack "IGCM_sys_activ_variables"
2075  if ( $DEBUG_sys ) ; then
2076    echo "IGCM_sys_activ_variables"
2077  fi
[622]2078
[811]2079# --------------------------------------------------------------------
2080#D- MPI specifications
2081# --------------------------------------------------------------------
2082
2083# --------------------------------------------------------------------
2084#D- Other specifications
2085# --------------------------------------------------------------------
2086
[623]2087  ulimit -s unlimited
[622]2088
[623]2089  IGCM_debug_PopStack "IGCM_sys_activ_variables"
[622]2090}
2091
2092############################################################
2093# Desactivate Running Environnment Variables
2094
[811]2095#D-#==================================================
2096#D-function IGCM_sys_desactiv_variables
2097#D-* Purpose: unset environement variables after execution
2098#D-* Examples:
2099#D-
[622]2100function IGCM_sys_desactiv_variables {
[623]2101  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
2102  if ( $DEBUG_sys ) ; then
2103    echo "IGCM_sys_desactiv_variables"
2104  fi
[811]2105# --------------------------------------------------------------------
2106#D- MPI specifications
2107# --------------------------------------------------------------------
2108
2109# --------------------------------------------------------------------
2110#D- Other specifications
2111# --------------------------------------------------------------------
2112
[623]2113  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
[622]2114}
2115
[623]2116############################################################
2117# Build MPI/OMP scripts run file (dummy function)
[622]2118
[811]2119#D-#==================================================
2120#D-function IGCM_sys_build_run_file
2121#D-* Purpose: build run file (deprecated)
2122#D-* Examples:
2123#D-
[622]2124function IGCM_sys_build_run_file {
2125
[811]2126  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
[622]2127
2128}
2129
2130############################################################
2131# Build MPI/OMP scripts
[811]2132
2133#D-#==================================================
2134#D-function IGCM_sys_build_execution_scripts
2135#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
2136#D-* Examples:
2137#D-
[623]2138function IGCM_sys_build_execution_scripts
[622]2139{
[623]2140  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
2141  if ( $DEBUG_sys ) ; then
2142    echo "IGCM_sys_build_execution_scripts " $@
2143  fi
[622]2144
[623]2145  EXECUTION=${HOST_MPIRUN_COMMAND}
[622]2146
[1078]2147  # MPMD mode
[623]2148  if ( ${OK_PARA_MPMD} ) ; then
[622]2149
[1078]2150    # Only MPI (MPMD)
2151    if  ( ! ${OK_PARA_OMP} ) ; then
[622]2152
[1078]2153      if [ -f run_file ] ; then
2154        IGCM_sys_Rm -f run_file
2155      fi
2156      touch run_file
[622]2157
[1078]2158      # Build run_file
[622]2159
[1078]2160      # First loop on the components for the coupler ie oasis (only if oasis3)
2161      # the coupler ie oasis3 must be the first one
2162      for comp in ${config_ListOfComponents[*]} ; do
[622]2163
[1078]2164        eval ExeNameIn=\${config_Executable_${comp}[0]}
2165        eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]2166
[1078]2167        # for CPL component only
2168        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
[1000]2169
[1078]2170          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2171          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2172          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
2173        fi
2174      done
[622]2175
[1078]2176      # Then second loop on the components
2177      for comp in ${config_ListOfComponents[*]} ; do
[622]2178
[1078]2179        eval ExeNameIn=\${config_Executable_${comp}[0]}
2180        eval ExeNameOut=\${config_Executable_${comp}[1]}
[622]2181
[1078]2182        # Only if we really have an executable for the component and not the coupler ie oasis:
2183        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
[622]2184
[1078]2185          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2186          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2187          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
2188        fi
2189      done
[622]2190
[1078]2191      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
[622]2192
[1078]2193      IGCM_sys_Chmod u+x run_file
2194      if ( $DEBUG_sys ) ; then
2195        echo "run_file contains : "
2196        cat run_file
2197      fi
[622]2198
[1078]2199    # MPI-OpenMP (MPMD)
2200    else
[622]2201
[1078]2202      # Use of mpirun instead of ccc_mprun
2203      EXECUTION="time mpirun"
[622]2204
[1078]2205      #  Hosts treatment
2206      ${EXECUTION} hostname | sort | uniq > hosts.tmp
[622]2207
[1078]2208      i=0
2209      rm -f hosts rankfile
2210      IGCM_debug_Print 1 "sys Curie, Hosts available :"
2211      for nodes in `cat hosts.tmp`
2212      do
2213        host[$i]=$nodes
2214        echo "${host[$i]}" >> hosts
2215        IGCM_debug_Print 1 ${host[$i]}
2216        i=$((i+1))
2217      done
2218      rm -f hosts.tmp
[1000]2219
[1078]2220      listnodes=${host[*]}
[1000]2221
[1078]2222      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
[1000]2223
[1078]2224      # Initialisation
2225      rank=0
2226      current_core=0
2227      core_per_node=16
2228      init_exec=n
[1000]2229
[1078]2230      # Loop on the components
2231      for comp in ${config_ListOfComponents[*]} ; do
[1000]2232
[1078]2233        eval ExeNameIn=\${config_Executable_${comp}[0]}
2234        eval ExeNameOut=\${config_Executable_${comp}[1]}
[1000]2235
[1078]2236        # Not possible if oasis has an executable (i.e old version of oasis3)
2237        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
2238          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
2239          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
2240          IGCM_debug_Verif_Exit
2241        fi
[1000]2242
[1078]2243        # Only if we really have an executable for the component :
2244        if [ "X${ExeNameOut}" != X\"\" ] ; then
[1000]2245
[1078]2246          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
2247          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
[1000]2248
[1078]2249          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
2250          echo ""  >> script_${ExeNameOut}.ksh
2251          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
2252
2253            # Check if the number of threads is correct
2254            case ${comp_proc_omp_loc} in
2255            2|4|8|16)
2256              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
2257              ;;
2258            *)
2259              IGCM_debug_Exit "ERROR with OMP parameters !"
2260              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
2261              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
2262              IGCM_debug_Verif_Exit
2263              ;;
2264            esac
2265            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
2266            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
2267            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
2268            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
2269          fi
2270          #echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
2271          #echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
2272          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
2273          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
2274
2275          if [ ${init_exec} = y ] ; then
2276            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
2277          else
2278            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
2279            init_exec=y
2280          fi
2281
[1000]2282# Build rankfile : method used to assign cores and nodes for the MPI process
[1078]2283# Ex :
2284#rank 0=curie5296 slot=0,1,2,3
[1000]2285#rank 1=curie5296 slot=4,5,6,7
2286# Example of final command :
2287# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
2288# with script_lmdz.x.ksh :
2289# #!/bin/ksh
2290#export KMP_STACKSIZE=3g
2291#export KMP_LIBRARY=turnaround
2292#export MKL_SERIAL=YES
2293#OMP_NUM_THREADS=4
2294#./lmdz.x
2295#
2296
[1078]2297          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
2298            (( index_host = current_core / core_per_node ))
2299            host_value=${host[${index_host}]}
2300            (( slot =  current_core % core_per_node ))
2301            virg=","
2302            string_final=""
2303            for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
2304              string=$index$virg
2305              string_final=$string_final$string
2306            done
2307            string_final=$( echo $string_final | sed "s/.$//" )
2308            echo "rank $rank=$host_value slot=$string_final" >> rankfile
2309            (( rank = rank + 1 ))
2310            (( current_core = current_core + comp_proc_omp_loc ))
2311          done
2312        fi
[1000]2313
[1078]2314      done
2315    fi
[1000]2316
[1078]2317  # Only one executable (SPMD mode).
2318  else
[1000]2319
[622]2320    for comp in ${config_ListOfComponents[*]} ; do
2321
[685]2322      # Only if we really have an executable for the component :
[623]2323      eval ExeNameOut=\${config_Executable_${comp}[1]}
[685]2324      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
[622]2325
[685]2326        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
2327        echo ""  >> script_${ExeNameOut}.ksh
2328        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
2329
2330        if ( ${OK_PARA_OMP} ) ; then
2331          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
2332          echo ""  >> script_${ExeNameOut}.ksh
2333          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
2334          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
2335          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
2336          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
2337        fi
2338
2339        if  ( ${OK_PARA_MPI} ) ; then
[623]2340          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
[685]2341          # Default : ccc_mprun used if nb_proc gt 1
2342          # to have out/err per process on different files
2343          # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
2344          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
2345          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
[623]2346        else
[685]2347          # Default : ccc_mprun is NOT used if nb_proc eq 1
2348          # to have out/err per process on different files
2349          # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
2350          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
2351          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
[622]2352        fi
[685]2353
2354        IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
2355        cat script_${ExeNameOut}.ksh
2356
[623]2357      fi
[622]2358
2359    done
2360
[623]2361  fi
[622]2362
[623]2363  IGCM_debug_Print 1 "sys Curie : execution command is "
2364  IGCM_debug_Print 1 "$EXECUTION"
[622]2365
[623]2366  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
[622]2367}
2368
[933]2369#D-#==================================================
2370#D-function IGCM_sys_check_path
2371#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
2372#D-* do not point to an important use directory. Stop immediately in that case.
2373#D-* Examples:
2374#D-
2375function IGCM_sys_check_path {
2376  IGCM_debug_PushStack "IGCM_sys_check_path"
2377  if ( $DEBUG_sys ) ; then
2378    echo "IGCM_sys_check_path"
2379  fi
[811]2380
[933]2381  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORKDIR} ] || [ X${RUN_DIR_PATH} = X${SCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
2382    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
2383    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
[936]2384    IGCM_debug_Exit "This will stop the job"
[933]2385  fi
2386  IGCM_debug_PopStack "IGCM_sys_check_path"
2387}
2388
[811]2389#D-#==================================================
2390#D-function IGCM_sys_check_quota
2391#D-* Purpose: check user quota. Stop the simulation if quota above 90%
2392#D-* Examples:
2393#D-
[714]2394function IGCM_sys_check_quota {
[776]2395  IGCM_debug_PushStack "IGCM_sys_check_quota"
2396  if ( $DEBUG_sys ) ; then
2397    echo "IGCM_sys_check_quota"
2398  fi
2399  # Limit of quota (in %)
2400  limit_quota=90
2401
2402  # Check of the volume
[912]2403  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
2404  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
[776]2405
2406  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
2407
2408    unit_avail=${volume_avail: -1}
2409    unit_quota=${volume_quota: -1}
2410
2411    if [ "${unit_quota}" = "*" ] ; then
[811]2412      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2413      IGCM_debug_Print 1 "More than 100% of your quota is used"
2414      IGCM_debug_Print 1 "Use the ccc_quota command to check"
2415      IGCM_debug_Print 1 "You must have more than 10% available to run"
2416      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2417      IGCM_debug_Verif_Exit
[714]2418    fi
[776]2419
2420    temp_avail=${volume_avail%%${volume_avail: -1}*}
2421    temp_quota=${volume_quota%%${volume_quota: -1}*}
2422
2423    if [ ! ${unit_avail} = ${unit_quota} ] ; then
2424
2425    # Convertion
2426      if [ ${volume_avail: -1} = "T" ] ; then
2427        (( temp_avail = temp_avail * 1000000000000 ))
2428      elif [ ${volume_avail: -1} = "G" ] ; then
2429        (( temp_avail = temp_avail * 1000000000 ))
2430      elif [ ${volume_avail: -1} = "M" ] ; then
2431        (( temp_avail = temp_avail * 1000000 ))
2432      elif [ ${volume_avail: -1} = "k" ] ; then
2433        (( temp_avail = temp_avail * 1000 ))
2434      else
2435        (( temp_avail = volume_avail ))
2436      fi
2437      if [ ${volume_quota: -1} = "T" ] ; then
2438        (( temp_quota = temp_quota * 1000000000000 ))
2439      elif [ ${volume_quota: -1} = "G" ] ; then
2440        (( temp_quota = temp_quota * 1000000000 ))
2441      elif [ ${volume_quota: -1} = "M" ] ; then
2442        (( temp_quota = temp_quota * 1000000 ))
2443      elif [ ${volume_quota: -1} = "k" ] ; then
2444        (( temp_quota = temp_quota * 1000 ))
2445      else
2446        (( temp_quota = volume_quota ))
2447      fi
2448    fi
2449
2450    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
2451#    echo "volume ratio is " $quota_volume
2452
2453    if [ ${quota_volume} -ge ${limit_quota} ] ; then
2454      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2455      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
2456      IGCM_debug_Print 1 "Use the ccc_quota command to check"
2457      IGCM_debug_Print 1 "You must have more than 10% available to run"
2458      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2459      IGCM_debug_Verif_Exit
2460    fi
2461
2462  fi
2463
2464# Check of the number of inodes
2465
[912]2466  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
2467  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
[776]2468
2469  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
2470
2471    unit_avail=${inode_avail: -1}
2472    unit_quota=${inode_quota: -1}
2473
2474    if [ "${unit_quota}" = "*" ] ; then
[811]2475      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2476      IGCM_debug_Print 1 "More than 100% of your quota is used"
2477      IGCM_debug_Print 1 "Use the ccc_quota command to check"
2478      IGCM_debug_Print 1 "You must have more than 10% available to run"
2479      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2480      IGCM_debug_Verif_Exit
[776]2481    fi
2482
2483    temp_avail=${inode_avail%%${inode_avail: -1}*}
2484    temp_quota=${inode_quota%%${inode_quota: -1}*}
2485
2486    if [ ! ${unit_avail} = ${unit_quota} ] ; then
2487
2488    # Convertion
2489      if [ ${inode_avail: -1} = "T" ] ; then
2490        (( temp_avail = temp_avail * 1000000000000 ))
2491      elif [ ${inode_avail: -1} = "G" ] ; then
2492        (( temp_avail = temp_avail * 1000000000 ))
2493      elif [ ${inode_avail: -1} = "M" ] ; then
2494        (( temp_avail = temp_avail * 1000000 ))
2495      elif [ ${inode_avail: -1} = "k" ] ; then
2496        (( temp_avail = temp_avail * 1000 ))
2497      else
2498        (( temp_avail = inode_avail ))
2499      fi
2500
2501      if [ ${inode_quota: -1} = "T" ] ; then
2502        (( temp_quota = temp_quota * 1000000000000 ))
2503      elif [ ${inode_quota: -1} = "G" ] ; then
2504        (( temp_quota = temp_quota * 1000000000 ))
2505      elif [ ${inode_quota: -1} = "M" ] ; then
2506        (( temp_quota = temp_quota * 1000000 ))
2507      elif [ ${inode_quota: -1} = "k" ] ; then
2508        (( temp_quota = temp_quota * 1000 ))
2509      else
2510        (( temp_quota = inode_quota ))
2511      fi
2512    fi
2513    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
2514#    echo "inode ratio is " $quota_inode
2515
2516    if [ ${quota_inode} -ge ${limit_quota} ] ; then
2517      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
2518      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
2519      IGCM_debug_Print 1 "Use the ccc_quota command to check"
2520      IGCM_debug_Print 1 "You must have more than 10% available to run"
2521      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2522      IGCM_debug_Verif_Exit
2523    fi
2524  fi
2525  IGCM_debug_PopStack "IGCM_sys_check_quota"
[714]2526}
2527
[773]2528#D-#==================================================
[958]2529#D-function IGCM_sys_GetJobID
2530#D-* Purpose: Check if job_name is currently
2531#D-  running or in queue
2532#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
2533#D-
2534function IGCM_sys_GetJobID {
2535  IGCM_debug_PushStack "IGCM_sys_GetJobID"
2536  if ( $DEBUG_sys ) ; then
2537    echo "IGCM_sys_GetJobID"
2538  fi
2539
2540  # With -f option, the full job name is given in the last column
2541  ID=$( ccc_mstat -f -u $2 | \
2542        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
2543
2544  eval ${3}=${ID}
2545
2546  IGCM_debug_PopStack "IGCM_sys_GetJobID"
2547}
2548
2549#D-#==================================================
[773]2550#D-function IGCM_sys_CountJobInQueue
2551#D-* Purpose: Check if job_name is currently
2552#D-  running or in queue
2553#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
2554#D-
2555function IGCM_sys_CountJobInQueue {
2556  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
2557  if ( $DEBUG_sys ) ; then
2558    echo "IGCM_sys_CountJobInQueue"
2559  fi
2560
2561  #NbRun=$( ccc_mstat -f | grep -c ${JobName} )
2562
2563  # With -f option, the full job name is given in the last column
[834]2564  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
[773]2565
2566  eval ${2}=${NbRun}
2567
2568  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
2569}
2570
[955]2571#D-#==================================================
2572#D-function IGCM_sys_ListJobInQueue
2573#D-* Purpose: Check if job_name is currently
2574#D-  running or in queue
2575#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
2576#D-
2577function IGCM_sys_ListJobInQueue {
2578  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
2579  if ( $DEBUG_sys ) ; then
2580    echo "IGCM_sys_ListJobInQueue"
2581  fi
2582
2583  # With -f option, the full job name is given in the last column
2584  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
2585                                        '( $2  == User      && \
2586                                           $NF != /TS/      && \
2587                                           $NF !~ /PACK/    && \
2588                                           $NF !~ /REBUILD/ && \
2589                                           $NF !~ /pack/ )     \
2590                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
2591
2592  eval set -A ${2} ${JobList[*]}
2593
2594  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
2595}
2596
[622]2597##############################################################
2598# NCO OPERATOR
2599
[811]2600#D-#==================================================
2601#D-function IGCM_sys_ncap2
2602#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
2603#D-* Examples:
2604#D-
[623]2605function IGCM_sys_ncap2 {
[1057]2606  IGCM_debug_PushStack "IGCM_sys_ncap2" $@
[623]2607  if ( $DEBUG_sys ) ; then
2608    echo "IGCM_sys_ncap2 :" $@
2609  fi
[713]2610
2611  typeset NB_ESSAI DELAI status i
2612  # number of tentative
2613  NB_ESSAI=3
2614  # time delay between tentative
2615  DELAI=2
2616
2617  i=0
2618  while [ $i -lt $NB_ESSAI ] ; do
[1091]2619    ncap2 -C "$@" > /tmp/out_command.$$ 2>&1
[713]2620    status=$?
2621    if [ ${status} -gt 0 ] ; then
2622      IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
[938]2623      cat /tmp/out_command.$$
2624      \rm /tmp/out_command.$$
[713]2625      IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]2626      sleep $DELAI
[713]2627    else
[938]2628      \rm /tmp/out_command.$$
[713]2629      break
2630    fi
2631    (( i = i + 1 ))
2632  done
2633
2634  if [ ${status} -gt 0 ] ; then
[811]2635    echo "IGCM_sys_ncap2 : ncap2 error"
2636    IGCM_debug_Exit "ncap2"
[623]2637  fi
2638
2639  IGCM_debug_PopStack "IGCM_sys_ncap2"
2640}
2641
[811]2642#D-#==================================================
2643#D-function IGCM_sys_ncatted
2644#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
2645#D-* Examples:
2646#D-
[622]2647function IGCM_sys_ncatted {
[1057]2648  IGCM_debug_PushStack "IGCM_sys_ncatted" $@
[623]2649  if ( $DEBUG_sys ) ; then
2650    echo "IGCM_sys_ncatted :" $@
2651  fi
[713]2652
2653  typeset NB_ESSAI DELAI status i
2654  # number of tentative
2655  NB_ESSAI=3
2656  # time delay between tentative
2657  DELAI=2
2658
2659  i=0
2660  while [ $i -lt $NB_ESSAI ] ; do
[938]2661    ncatted "$@" > /tmp/out_command.$$ 2>&1
[713]2662    status=$?
2663    if [ ${status} -gt 0 ] ; then
2664      IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
[938]2665      cat /tmp/out_command.$$
2666      \rm /tmp/out_command.$$
[713]2667      IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]2668      sleep $DELAI
[713]2669    else
[938]2670      \rm /tmp/out_command.$$
[713]2671      break
2672    fi
2673    (( i = i + 1 ))
2674  done
2675
2676  if [ ${status} -gt 0 ] ; then
[811]2677    echo "IGCM_sys_ncatted : ncatted error"
2678    IGCM_debug_Exit "ncatted"
[623]2679  fi
[622]2680
[623]2681  IGCM_debug_PopStack "IGCM_sys_ncatted"
[622]2682}
2683
[811]2684#D-#==================================================
2685#D-function IGCM_sys_ncbo
2686#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
2687#D-* Examples:
2688#D-
[622]2689function IGCM_sys_ncbo {
[1057]2690  IGCM_debug_PushStack "IGCM_sys_ncbo" $@
[623]2691  if ( $DEBUG_sys ) ; then
2692    echo "IGCM_sys_ncbo :" $@
2693  fi
[713]2694
2695  typeset NB_ESSAI DELAI status i
2696  # number of tentative
2697  NB_ESSAI=3
2698  # time delay between tentative
2699  DELAI=2
2700
2701  i=0
2702  while [ $i -lt $NB_ESSAI ] ; do
[1091]2703    ncbo -C $@ > /tmp/out_command.$$ 2>&1
[713]2704    status=$?
2705    if [ ${status} -gt 0 ] ; then
2706      IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
[938]2707      cat /tmp/out_command.$$
2708      \rm /tmp/out_command.$$
[713]2709      IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]2710      sleep $DELAI
[713]2711    else
[938]2712      \rm /tmp/out_command.$$
[713]2713      break
2714    fi
2715    (( i = i + 1 ))
2716  done
2717
2718  if [ ${status} -gt 0 ] ; then
[811]2719    echo "IGCM_sys_ncbo : ncbo error"
2720    IGCM_debug_Exit "ncbo"
[623]2721  fi
[622]2722
[623]2723  IGCM_debug_PopStack "IGCM_sys_ncbo"
[622]2724}
2725
[811]2726#D-#==================================================
2727#D-function IGCM_sys_ncdif
2728#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
2729#D-* Examples:
2730#D-
[622]2731function IGCM_sys_ncdiff {
[1057]2732  IGCM_debug_PushStack "IGCM_sys_ncdiff" $@
[623]2733  if ( $DEBUG_sys ) ; then
2734    echo "IGCM_sys_ncdiff :" $@
2735  fi
[713]2736
2737  typeset NB_ESSAI DELAI status i
2738  # number of tentative
2739  NB_ESSAI=3
2740  # time delay between tentative
2741  DELAI=2
2742
2743  i=0
2744  while [ $i -lt $NB_ESSAI ] ; do
[1091]2745    ncdiff -C $@ > /tmp/out_command.$$ 2>&1
[713]2746    status=$?
2747    if [ ${status} -gt 0 ] ; then
2748      IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
[938]2749      cat /tmp/out_command.$$
2750      \rm /tmp/out_command.$$
[713]2751      IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]2752      sleep $DELAI
[713]2753    else
[938]2754      \rm /tmp/out_command.$$
[713]2755      break
2756    fi
2757    (( i = i + 1 ))
2758  done
2759
2760  if [ ${status} -gt 0 ] ; then
[811]2761    echo "IGCM_sys_ncdiff : ncdiff error"
2762    IGCM_debug_Exit "ncdiff"
[623]2763  fi
[622]2764
[623]2765  IGCM_debug_PopStack "IGCM_sys_ncdiff"
[622]2766}
2767
[811]2768#D-#==================================================
2769#D-function IGCM_sys_ncea
2770#D-* Purpose: encapsulate ncea call so as to manage error code and retry
2771#D-* Examples:
2772#D-
[622]2773function IGCM_sys_ncea {
[1057]2774  IGCM_debug_PushStack "IGCM_sys_ncea" $@
[623]2775  if ( $DEBUG_sys ) ; then
2776    echo "IGCM_sys_ncea :" $@
2777  fi
[713]2778
2779  typeset NB_ESSAI DELAI status i
2780  # number of tentative
2781  NB_ESSAI=3
2782  # time delay between tentative
2783  DELAI=2
2784
2785  i=0
2786  while [ $i -lt $NB_ESSAI ] ; do
[1091]2787    ncea -C $@ > /tmp/out_command.$$ 2>&1
[713]2788    status=$?
2789    if [ ${status} -gt 0 ] ; then
2790      IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
[938]2791      cat /tmp/out_command.$$
2792      \rm /tmp/out_command.$$
[713]2793      IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]2794      sleep $DELAI
[713]2795    else
[938]2796      \rm /tmp/out_command.$$
[713]2797      break
2798    fi
2799    (( i = i + 1 ))
2800  done
2801
2802  if [ ${status} -gt 0 ] ; then
[811]2803    echo "IGCM_sys_ncea : ncea error"
2804    IGCM_debug_Exit "ncea"
[623]2805  fi
[622]2806
[623]2807  IGCM_debug_PopStack "IGCM_sys_ncea"
[622]2808}
2809
[811]2810#D-#==================================================
2811#D-function IGCM_sys_ncecat
2812#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
2813#D-* Examples:
2814#D-
[622]2815function IGCM_sys_ncecat {
[1057]2816  IGCM_debug_PushStack "IGCM_sys_ncecat" $@
[623]2817  if ( $DEBUG_sys ) ; then
2818    echo "IGCM_sys_ncecat :" $@
2819  fi
[713]2820
2821  typeset NB_ESSAI DELAI status i
2822  # number of tentative
2823  NB_ESSAI=3
2824  # time delay between tentative
2825  DELAI=2
2826
2827  i=0
2828  while [ $i -lt $NB_ESSAI ] ; do
[1091]2829    ncecat -C $@ > /tmp/out_command.$$ 2>&1
[713]2830    status=$?
2831    if [ ${status} -gt 0 ] ; then
2832      IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
[938]2833      cat /tmp/out_command.$$
2834      \rm /tmp/out_command.$$
[713]2835      IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]2836      sleep $DELAI
[713]2837    else
[938]2838      \rm /tmp/out_command.$$
[713]2839      break
2840    fi
2841    (( i = i + 1 ))
2842  done
2843
2844  if [ ${status} -gt 0 ] ; then
[811]2845    echo "IGCM_sys_ncecat : ncecat error"
2846    IGCM_debug_Exit "ncecat"
[623]2847  fi
[622]2848
[623]2849  IGCM_debug_PopStack "IGCM_sys_ncecat"
[622]2850}
2851
[811]2852#D-#==================================================
2853#D-function IGCM_sys_ncflint
2854#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
2855#D-* Examples:
2856#D-
[622]2857function IGCM_sys_ncflint {
[1057]2858  IGCM_debug_PushStack "IGCM_sys_ncflint" $@
[623]2859  if ( $DEBUG_sys ) ; then
2860    echo "IGCM_sys_ncflint :" $@
2861  fi
[713]2862
2863  typeset NB_ESSAI DELAI status i
2864  # number of tentative
2865  NB_ESSAI=3
2866  # time delay between tentative
2867  DELAI=2
2868
2869  i=0
2870  while [ $i -lt $NB_ESSAI ] ; do
[1091]2871    ncflint -C $@ > /tmp/out_command.$$ 2>&1
[713]2872    status=$?
2873    if [ ${status} -gt 0 ] ; then
2874      IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
[938]2875      cat /tmp/out_command.$$
2876      \rm /tmp/out_command.$$
[713]2877      IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]2878      sleep $DELAI
[713]2879    else
[938]2880      \rm /tmp/out_command.$$
[713]2881      break
2882    fi
2883    (( i = i + 1 ))
2884  done
2885
2886  if [ ${status} -gt 0 ] ; then
[811]2887    echo "IGCM_sys_ncflint : ncflint error"
2888    IGCM_debug_Exit "ncflint"
[623]2889  fi
[622]2890
[623]2891  IGCM_debug_PopStack "IGCM_sys_ncflint"
[622]2892}
2893
[811]2894#D-#==================================================
2895#D-function IGCM_sys_ncks
2896#D-* Purpose: encapsulate ncks call so as to manage error code and retry
2897#D-* Examples:
2898#D-
[622]2899function IGCM_sys_ncks {
[1057]2900  IGCM_debug_PushStack "IGCM_sys_ncks" $@
[623]2901  if ( $DEBUG_sys ) ; then
2902    echo "IGCM_sys_ncks :" $@
2903  fi
[713]2904
2905  typeset NB_ESSAI DELAI status i
2906  # number of tentative
2907  NB_ESSAI=3
2908  # time delay between tentative
2909  DELAI=2
2910
2911  i=0
2912  while [ $i -lt $NB_ESSAI ] ; do
[1091]2913    ncks -C $@ > /tmp/out_command.$$ 2>&1
[713]2914    status=$?
2915    if [ ${status} -gt 0 ] ; then
2916      IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
[938]2917      cat /tmp/out_command.$$
2918      \rm /tmp/out_command.$$
[713]2919      IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]2920      sleep $DELAI
[713]2921    else
[938]2922      \rm /tmp/out_command.$$
[713]2923      break
2924    fi
2925    (( i = i + 1 ))
2926  done
2927
2928  if [ ${status} -gt 0 ] ; then
[811]2929    echo "IGCM_sys_ncks : ncks error"
2930    IGCM_debug_Exit "ncks"
[623]2931  fi
[622]2932
[623]2933  IGCM_debug_PopStack "IGCM_sys_ncks"
[622]2934}
2935
[811]2936#D-#==================================================
2937#D-function IGCM_sys_ncpdq
2938#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
2939#D-* Examples:
2940#D-
[622]2941function IGCM_sys_ncpdq {
[1057]2942  IGCM_debug_PushStack "IGCM_sys_ncpdq" $@
[623]2943  if ( $DEBUG_sys ) ; then
2944    echo "IGCM_sys_ncpdq :" $@
2945  fi
[713]2946
2947  typeset NB_ESSAI DELAI status i
2948  # number of tentative
2949  NB_ESSAI=3
2950  # time delay between tentative
2951  DELAI=2
2952
2953  i=0
2954  while [ $i -lt $NB_ESSAI ] ; do
[1091]2955    ncpdq -C $@ > /tmp/out_command.$$ 2>&1
[713]2956    status=$?
2957    if [ ${status} -gt 0 ] ; then
2958      IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
[938]2959      cat /tmp/out_command.$$
2960      \rm /tmp/out_command.$$
[713]2961      IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]2962      sleep $DELAI
[713]2963    else
[938]2964      \rm /tmp/out_command.$$
[713]2965      break
2966    fi
2967    (( i = i + 1 ))
2968  done
2969
2970  if [ ${status} -gt 0 ] ; then
[811]2971    echo "IGCM_sys_ncpdq : ncpdq error"
2972    IGCM_debug_Exit "ncpdq"
[623]2973  fi
[622]2974
[623]2975  IGCM_debug_PopStack "IGCM_sys_ncpdq"
[622]2976}
2977
[811]2978#D-#==================================================
2979#D-function IGCM_sys_ncra
2980#D-* Purpose: encapsulate ncra call so as to manage error code and retry
2981#D-* Examples:
2982#D-
[622]2983function IGCM_sys_ncra {
[1057]2984  IGCM_debug_PushStack "IGCM_sys_ncra" $@
[623]2985  if ( $DEBUG_sys ) ; then
2986    echo "IGCM_sys_ncra :" $@
2987  fi
[713]2988
2989  typeset NB_ESSAI DELAI status i
2990  # number of tentative
2991  NB_ESSAI=3
2992  # time delay between tentative
2993  DELAI=2
2994
2995  i=0
2996  while [ $i -lt $NB_ESSAI ] ; do
[1091]2997    ncra -C $@ > /tmp/out_command.$$ 2>&1
[713]2998    status=$?
2999    if [ ${status} -gt 0 ] ; then
3000      IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
[938]3001      cat /tmp/out_command.$$
3002      \rm /tmp/out_command.$$
[713]3003      IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]3004      sleep $DELAI
[713]3005    else
[938]3006      \rm /tmp/out_command.$$
[713]3007      break
3008    fi
3009    (( i = i + 1 ))
3010  done
3011
3012  if [ ${status} -gt 0 ] ; then
[811]3013    echo "IGCM_sys_ncra : ncra error"
3014    IGCM_debug_Exit "ncra"
[623]3015  fi
[622]3016
[623]3017  IGCM_debug_PopStack "IGCM_sys_ncra"
[622]3018}
3019
[811]3020#D-#==================================================
3021#D-function IGCM_sys_ncrcat
3022#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
3023#D-* Examples:
3024#D-
[622]3025function IGCM_sys_ncrcat {
[1057]3026  IGCM_debug_PushStack "IGCM_sys_ncrcat" $@
[623]3027  if ( $DEBUG_sys ) ; then
3028    echo "IGCM_sys_ncrcat :" $@
3029  fi
[713]3030
[725]3031  typeset NB_ESSAI DELAI status i lastArg
[713]3032  # number of tentative
3033  NB_ESSAI=3
3034  # time delay between tentative
3035  DELAI=2
3036
3037  i=0
3038  while [ $i -lt $NB_ESSAI ] ; do
[1091]3039    ncrcat -C $@ > /tmp/out_command.$$ 2>&1
[713]3040    status=$?
3041    if [ ${status} -gt 0 ] ; then
3042      IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
[938]3043      cat /tmp/out_command.$$
3044      \rm /tmp/out_command.$$
[713]3045      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]3046      sleep $DELAI
[938]3047    elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" /tmp/out_command.$$ )" = "X" ] ; then
[721]3048      IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
[938]3049      cat /tmp/out_command.$$
[725]3050      # remove files having corrupted time axis
[734]3051      eval lastArg=\${$#}
[725]3052      IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
3053      \rm ${lastArg}
[938]3054      \rm /tmp/out_command.$$
[721]3055      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]3056      sleep $DELAI
[713]3057    else
[938]3058      \rm /tmp/out_command.$$
[713]3059      break
3060    fi
3061    (( i = i + 1 ))
3062  done
3063
3064  if [ ${status} -gt 0 ] ; then
[811]3065    echo "IGCM_sys_ncrcat : ncrcat error"
3066    #IGCM_debug_Exit "ncrcat"
[623]3067  fi
[622]3068
[623]3069  IGCM_debug_PopStack "IGCM_sys_ncrcat"
[622]3070}
3071
[811]3072#D-#==================================================
3073#D-function IGCM_sys_ncrename
3074#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
3075#D-* Examples:
3076#D-
[622]3077function IGCM_sys_ncrename {
[1057]3078  IGCM_debug_PushStack "IGCM_sys_ncrename" $@
[623]3079  if ( $DEBUG_sys ) ; then
3080    echo "IGCM_sys_ncrename :" $@
3081  fi
[713]3082
3083  typeset NB_ESSAI DELAI status i
3084  # number of tentative
3085  NB_ESSAI=3
3086  # time delay between tentative
3087  DELAI=2
3088
3089  i=0
3090  while [ $i -lt $NB_ESSAI ] ; do
[938]3091    ncrename $@ > /tmp/out_command.$$ 2>&1
[713]3092    status=$?
3093    if [ ${status} -gt 0 ] ; then
3094      IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
[938]3095      cat /tmp/out_command.$$
3096      \rm /tmp/out_command.$$
[713]3097      IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]3098      sleep $DELAI
[713]3099    else
[938]3100      \rm /tmp/out_command.$$
[713]3101      break
3102    fi
3103    (( i = i + 1 ))
3104  done
3105
3106  if [ ${status} -gt 0 ] ; then
[811]3107    echo "IGCM_sys_ncrename : ncrename error"
3108    IGCM_debug_Exit "ncrename"
[623]3109  fi
[622]3110
[623]3111  IGCM_debug_PopStack "IGCM_sys_ncrename"
[622]3112}
3113
[811]3114#D-#==================================================
3115#D-function IGCM_sys_ncwa
3116#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
3117#D-* Examples:
3118#D-
[622]3119function IGCM_sys_ncwa {
[1057]3120  IGCM_debug_PushStack "IGCM_sys_ncwa" $@
[623]3121  if ( $DEBUG_sys ) ; then
3122    echo "IGCM_sys_ncwa :" $@
3123  fi
[713]3124
3125  typeset NB_ESSAI DELAI status i
3126  # number of tentative
3127  NB_ESSAI=3
3128  # time delay between tentative
3129  DELAI=2
3130
3131  i=0
3132  while [ $i -lt $NB_ESSAI ] ; do
[1091]3133    ncwa -C $@ > /tmp/out_command.$$ 2>&1
[713]3134    status=$?
3135    if [ ${status} -gt 0 ] ; then
3136      IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
[938]3137      cat /tmp/out_command.$$
3138      \rm /tmp/out_command.$$
[713]3139      IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
[733]3140      sleep $DELAI
[713]3141    else
[938]3142      \rm /tmp/out_command.$$
[713]3143      break
3144    fi
3145    (( i = i + 1 ))
3146  done
3147
3148  if [ ${status} -gt 0 ] ; then
[811]3149    echo "IGCM_sys_ncwa : ncwa error"
3150    IGCM_debug_Exit "ncwa"
[623]3151  fi
[622]3152
[623]3153  IGCM_debug_PopStack "IGCM_sys_ncwa"
[622]3154}
[623]3155
3156##############################################################
3157# CDO OPERATOR
3158
[811]3159#D-#==================================================
3160#D-function IGCM_sys_cdo
3161#D-* Purpose: encapsulate cdo call so as to manage error code and retry
3162#D-* Examples:
3163#D-
[623]3164function IGCM_sys_cdo {
[1057]3165  IGCM_debug_PushStack "IGCM_sys_cdo" $@
[623]3166  if ( $DEBUG_sys ) ; then
3167    echo "IGCM_sys_cdo :" $@
3168  fi
[713]3169
3170  typeset status
3171
[938]3172  \cdo $@ > /tmp/out_command.$$ 2>&1
[713]3173  status=$?
3174  if [ ${status} -gt 0 ] ; then
3175    echo "IGCM_sys_cdo : error code ${status}"
[938]3176    cat /tmp/out_command.$$
3177    \rm /tmp/out_command.$$
[623]3178    IGCM_debug_PopStack "IGCM_sys_cdo"
3179    return 1
3180  else
3181    IGCM_debug_PopStack "IGCM_sys_cdo"
3182    return 0
3183  fi
3184
3185  IGCM_debug_PopStack "IGCM_sys_cdo"
3186}
Note: See TracBrowser for help on using the repository browser.