source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_default.ksh @ 1147

Last change on this file since 1147 was 1147, checked in by sdipsl, 9 years ago

broken ensemble. Tentative fix for Agathe. Ensemble are broken since r1057...
broken rebuild station due to changes in LMDZ station files. Tentative fix for Lidia.
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 71.8 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip
5# Contact: Martial.Mancip__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys Default host
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset  HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset  LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset -r MASTER=${HOST}
71
72#D-
73#D-#==================================================
74#D-Program used in libIGCM
75#D-#==================================================
76
77# Submit command
78typeset SUBMIT=${SUBMIT:=qsub}
79# rsync with path
80typeset -r RSYNC=/usr/bin/rsync
81# RSYNC_opt args to rsync
82typeset -r RSYNC_opt="-va"
83# ie storage filesystem
84typeset -r STOREHOST=${MASTER}
85
86#====================================================
87# Set environment tools (ferret, nco, cdo)
88#====================================================
89#. /home/${LOGIN}/.atlas_env_${HOST}_ksh
90
91export PATH=${PATH}:/home/jripsl/snapshot/Monitoring/CNClient/
92
93#====================================================
94# Host specific DIRECTORIES
95#====================================================
96
97#====================================================
98#- MirrorlibIGCM for frontend
99typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
100
101#====================================================
102#- libIGCM_POST for frontend
103typeset -r libIGCM_POST=${libIGCM}
104
105#====================================================
106#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
107typeset -r R_EXE="${MODIPSL}/bin"
108
109#====================================================
110#- SUBMIT_DIR : submission dir
111typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PWD}}
112
113#====================================================
114#- IN
115typeset -r R_IN=${R_IN:=/home/${LOGIN}/IGCM}
116typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/home/${LOGIN}/IGCM}
117
118#====================================================
119#- RUN_DIR_PATH : Temporary working directory (=> TMP)
120typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/home/${LOGIN}/RUN_DIR/tmp$$}
121
122#====================================================
123#- HOST_MPIRUN_COMMAND
124typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
125
126#====================================================
127#- Max number of arguments passed to nco operator or demigration command
128UNIX_MAX_LIMIT=360
129
130#====================================================
131#- Number of core per node (max number of OpenMP task)
132NUM_COREPERNODE=1
133
134#====================================================
135#- Default number of MPI task for IPSL coupled model
136#- required for backward compatibility
137#-
138DEFAULT_NUM_PROC_OCE=1
139DEFAULT_NUM_PROC_CPL=1
140(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
141DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
142
143#D-#==================================================
144#D-function IGCM_sys_defineArchives
145#D-* Purpose:
146#D-* Define ARCHIVE : Dedicated to large files
147#D-* Define STORAGE : Dedicated to small/medium files
148#D-* Define R_OUT   : Output tree located on ARCHIVE
149#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
150#D-* Define R_BUF   : Output tree located on STORAGE hosting files waiting for rebuild or pack processes
151#D-* if SpaceName=TEST nothing special will hapen
152#D-* Examples:
153#D-
154function IGCM_sys_defineArchives {
155  IGCM_debug_PushStack "IGCM_sys_defineArchives"
156
157  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
158    #====================================================
159    #- ARCHIVE (dedicated to large files)
160    ARCHIVE=${config_UserChoices_ARCHIVE}
161  else
162    #====================================================
163    #- ARCHIVE (dedicated to large files)
164    ARCHIVE=${ARCHIVE:=/home/${LOGIN}}
165  fi
166
167  if [ ! X${config_UserChoices_STORAGE} = X ]; then
168    #====================================================
169    #- STORAGE (dedicated to small/medium files)
170    STORAGE=${config_UserChoices_STORAGE}
171  else
172    #====================================================
173    #- STORAGE (dedicated to small/medium files)
174    STORAGE=${ARCHIVE}
175  fi
176
177  # ON OBELIX NO SPECIAL CASE WHEN X${config_UserChoices_SpaceName} = XTEST
178
179  #====================================================
180  #- R_OUT
181  R_OUT=${ARCHIVE}/IGCM_OUT
182
183  #====================================================
184  #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
185  R_FIG=${STORAGE}/IGCM_OUT
186
187  #====================================================
188  #- R_BUF (ONLY FOR double copy an scratch)
189  R_BUF=${STORAGE}/IGCM_OUT
190
191  #====================================================
192  #- R_TMP (point a space acting as /tmp)
193  R_TMP=/tmp
194
195  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
196  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
197  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
198
199  IGCM_debug_PopStack "IGCM_sys_defineArchives"
200}
201
202#D-#==================================================
203#D-function IGCM_sys_RshMaster
204#D-* Purpose: Connection to frontend machine.
205#D-* Examples:
206#D-
207function IGCM_sys_RshMaster {
208  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
209  /bin/ksh <<-EOF
210    export libIGCM=${libIGCM}
211    export DEBUG_debug=${DEBUG_debug}
212    . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
213    . ${libIGCM}/libIGCM_card/libIGCM_card.ksh
214    ${@}
215EOF
216  if [ $? -gt 0 ] ; then
217    echo "IGCM_sys_RshMaster : erreur."
218    IGCM_debug_Exit "IGCM_sys_RshMaster"
219  fi
220  IGCM_debug_PopStack "IGCM_sys_RshMaster"
221}
222
223#D-#==================================================
224#D-function IGCM_sys_RshArchive
225#D-* Purpose: Archive rsh command
226#D-* Examples:
227#D-
228function IGCM_sys_RshArchive {
229  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
230  /bin/ksh <<-EOF
231    ${@}
232EOF
233  status=$?
234  if [ ${status} -gt 0 ] ; then
235    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
236    IGCM_debug_Exit "IGCM_sys_RshArchive"
237  fi
238  IGCM_debug_PopStack "IGCM_sys_RshArchive"
239}
240
241#D-#==================================================
242#D-function IGCM_sys_RshPost
243#D-* Purpose: Post-process rsh command
244#D-* Examples:
245#D-
246function IGCM_sys_RshPost {
247  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
248  if ( $DEBUG_sys ) ; then
249    echo "IGCM_sys_RshPost :" $@
250  fi
251
252  # keep standard input (stdin) for the loop onto temporary file
253  cat >tmp_IGCM_sys_RshPost_$$
254
255  /bin/ksh <tmp_IGCM_sys_RshPost_$$
256  if [ $? -gt 0 ] ; then
257    echo "IGCM_sys_RshPost : erreur."
258    IGCM_debug_Exit "IGCM_sys_RshPost"
259  fi
260  # delete temporary file
261  \rm tmp_IGCM_sys_RshPost_$$
262  IGCM_debug_PopStack "IGCM_sys_RshPost"
263}
264
265#D-#==================================================
266#D-function IGCM_sys_SendMail
267#D-* Purpose: Send mail when simulation is over
268#D-* Examples:
269#D-
270function IGCM_sys_SendMail {
271  IGCM_debug_PushStack "IGCM_sys_SendMail" $@
272  if ( $DEBUG_sys ) ; then
273    echo "IGCM_sys_SendMail :" $@
274  fi
275
276  if [ X${1} = XAccounting ] ; then
277    status=Accounting
278    mailText=jobAccounting.mail
279  elif ( ${ExitFlag} ) ; then
280    status=failed
281    mailText=jobEnd.mail
282  else
283    status=completed
284    mailText=jobEnd.mail
285  fi
286
287  # Update selected mail template
288  while read -r line; do
289    eval echo $line >> mail.txt ;
290  done < ${libIGCM}/libIGCM_sys/${mailText}
291
292  if [ ! -z ${config_UserChoices_MailName} ] ; then
293    mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} <  mail.txt
294  elif [ -f ~/.forward ] ; then
295    mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < mail.txt
296  fi
297
298  sleep 10
299  rm -f mail.txt
300
301  IGCM_debug_PopStack "IGCM_sys_SendMail"
302}
303
304#D-#==================================================
305#D-function IGCM_sys_Mkdir
306#D-* Purpose: Master locale mkdir command
307#D-* Examples:
308#D-
309function IGCM_sys_Mkdir {
310  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
311  if ( $DEBUG_sys ) ; then
312    echo "IGCM_sys_Mkdir :" $@
313  fi
314  if [ ! -d ${1} ]; then
315    \mkdir -p $1
316    if [ $? -gt 0 ] ; then
317      echo "IGCM_sys_Mkdir : erreur."
318      IGCM_debug_Exit "IGCM_sys_Mkdir"
319    fi
320  fi
321  # vérification :
322  if [ ! -d ${1} ] ; then
323    echo "IGCM_sys_Mkdir : erreur."
324    IGCM_debug_Exit "IGCM_sys_Mkdir"
325  fi
326  IGCM_debug_PopStack "IGCM_sys_Mkdir"
327}
328
329#D-#==================================================
330#D-function IGCM_sys_MkdirArchive
331#D-* Purpose: Mkdir on Archive
332#D-* Examples:
333#D-
334function IGCM_sys_MkdirArchive {
335  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
336  if ( $DEBUG_sys ) ; then
337    echo "IGCM_sys_MkdirArchive :" $@
338  fi
339  #- creation de repertoire sur le serveur fichier
340  mkdir -p $1
341  status=$?
342
343  if [ ${status} -gt 0 ] ; then
344    IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
345    IGCM_debug_Exit "IGCM_sys_MkdirArchive"
346  fi
347  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
348}
349
350#D-#==================================================
351#D-function IGCM_sys_MkdirWork
352#D-* Purpose: Mkdir on Work
353#D-* Examples:
354#D-
355function IGCM_sys_MkdirWork {
356  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
357  if ( $DEBUG_sys ) ; then
358    echo "IGCM_sys_MkdirWork :" $@
359  fi
360  #- creation de repertoire sur le serveur fichier
361  if [ ! -d ${1} ]; then
362    \mkdir -p $1
363    if [ $? -gt 0 ] ; then
364      echo "IGCM_sys_MkdirWork : erreur."
365      IGCM_debug_Exit "IGCM_sys_MkdirWork"
366    fi
367  fi
368  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
369}
370
371#D-#==================================================
372#D-function IGCM_sys_Cd
373#D-* Purpose: master cd command
374#D-* Examples:
375#D-
376function IGCM_sys_Cd {
377  IGCM_debug_PushStack "IGCM_sys_Cd" $@
378  if ( $DEBUG_sys ) ; then
379    echo "IGCM_sys_Cd :" $@
380  fi
381  \cd $1
382  if [ $? -gt 0 ] ; then
383    echo "IGCM_sys_Cd : erreur."
384    IGCM_debug_Exit "IGCM_sys_Cd"
385  fi
386  IGCM_debug_PopStack "IGCM_sys_Cd"
387}
388
389#D-#==================================================
390#D-function IGCM_sys_Chmod
391#D-* Purpose: Chmod
392#D-* Examples:
393#D-
394function IGCM_sys_Chmod {
395  IGCM_debug_PushStack "IGCM_sys_Chmod" $@
396  if ( $DEBUG_sys ) ; then
397    echo "IGCM_sys_Chmod :" $@
398  fi
399  \chmod $@
400  if [ $? -gt 0 ] ; then
401    echo "IGCM_sys_Chmod : erreur."
402    IGCM_debug_Exit "IGCM_sys_Chmod"
403  fi
404  IGCM_debug_PopStack "IGCM_sys_Chmod"
405}
406
407#D-#==================================================
408#D-function IGCM_sys_FileSize
409#D-* Purpose: Filesize
410#D-* Examples:
411#D-
412function IGCM_sys_FileSize {
413  IGCM_debug_PushStack "IGCM_sys_FileSize" $@
414
415  typeset sizeF
416  set +A sizeF -- $( ls -la ${1} )
417  if [ $? -gt 0 ] ; then
418    IGCM_debug_Exit "IGCM_sys_FileSize"
419  fi
420  eval ${2}=${sizeF[4]}
421
422  IGCM_debug_PopStack "IGCM_sys_FileSize"
423}
424
425#D-#==================================================
426#D-function IGCM_sys_TestDir
427#D-* Purpose: Test Directory that must exists
428#D-* Examples:
429#D-
430function IGCM_sys_TestDir {
431  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
432  if ( $DEBUG_sys ) ; then
433    echo "IGCM_sys_TestDir :" $@
434  fi
435  typeset ExistFlag
436  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
437  IGCM_debug_PopStack "IGCM_sys_TestDir"
438
439  return ${ExistFlag}
440}
441
442#D-#==================================================
443#D-function IGCM_sys_TestDirArchive
444#D-* Purpose: Test Directory that must exists on Archive
445#D-* Examples:
446#D-
447function IGCM_sys_TestDirArchive {
448  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
449  if ( $DEBUG_sys ) ; then
450    echo "IGCM_sys_TestDirArchive :" $@
451  fi
452  typeset ExistFlag
453  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
454  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
455  return ${ExistFlag}
456}
457
458#D-#==================================================
459#D-function IGCM_sys_IsFileArchived
460#D-* Purpose: Test file that must NOT EXISTS on Archive
461#D-* Examples:
462#D-
463function IGCM_sys_IsFileArchived {
464  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
465  if ( $DEBUG_sys ) ; then
466    echo "IGCM_sys_IsFileArchived :" $@
467  fi
468  typeset IsArchivedFlag
469  # Never archived for this system
470  IsArchivedFlag=1
471  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
472
473  return ${IsArchivedFlag}
474}
475
476#D-#==================================================
477#D-function IGCM_sys_TestFileArchive
478#D-* Purpose: Test file that must NOT EXISTS on Archive
479#D-* Examples:
480#D-
481function IGCM_sys_TestFileArchive {
482  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
483  typeset ExistFlag
484  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
485  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
486
487  return ${ExistFlag}
488}
489
490#D-#==================================================
491#D-function IGCM_sys_TestFileBuffer
492#D-* Purpose: Test file that must NOT EXISTS on Buffer
493#D-* Examples:
494#D-
495function IGCM_sys_TestFileBuffer {
496  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
497  typeset ExistFlag
498  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
499  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
500
501  return ${ExistFlag}
502}
503
504#D-#==================================================
505#D-function IGCM_sys_CountFileArchive
506#D-* Purpose: Count files on Archive filesystem
507#D-* Examples:
508#D-
509function IGCM_sys_CountFileArchive {
510  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
511  ls ${@} 2>/dev/null | wc -l
512  if [ $? -gt 0 ] ; then
513    echo "IGCM_sys_CountFileArchive : erreur."
514  fi
515  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
516}
517
518#D-#==================================================
519#D-function IGCM_sys_CountFileBuffer
520#D-* Purpose: Count files on Scratch filesystem
521#D-* Examples:
522#D-
523function IGCM_sys_CountFileBuffer {
524  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
525  ls ${@} 2>/dev/null | wc -l
526  if [ $? -gt 0 ] ; then
527    echo "IGCM_sys_CountFileBuffer : erreur."
528  fi
529  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
530}
531
532#D-#==================================================
533#D-function IGCM_sys_Tree
534#D-* Purpose: Tree directories with files on ${ARCHIVE}
535#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
536#D-
537function IGCM_sys_Tree {
538  IGCM_debug_PushStack "IGCM_sys_Tree" $@
539  if ( $DEBUG_sys ) ; then
540    echo "IGCM_sys_Tree :" $@
541  fi
542
543  \tree -f $@
544
545  IGCM_debug_PopStack "IGCM_sys_Tree"
546}
547
548#D-#==================================================
549#D-function IGCM_sys_Tar
550#D-* Purpose: master tar command
551#D-* Examples:
552#D-
553function IGCM_sys_Tar {
554  IGCM_debug_PushStack "IGCM_sys_Tar" $@
555  if ( $DEBUG_sys ) ; then
556    echo "IGCM_sys_Tar :" $@
557  fi
558  \tar cf $@
559  if [ $? -gt 0 ] ; then
560    echo "IGCM_sys_Tar : erreur."
561    IGCM_debug_Exit "IGCM_sys_Tar"
562  fi
563  IGCM_debug_PopStack "IGCM_sys_Tar"
564}
565
566#D-#==================================================
567#D-function IGCM_sys_UnTar
568#D-* Purpose: master un-tar command
569#D-* Examples:
570#D-
571function IGCM_sys_UnTar {
572  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
573  if ( $DEBUG_sys ) ; then
574    echo "IGCM_sys_UnTar :" $@
575  fi
576  \tar xvf $1
577  if [ $? -gt 0 ] ; then
578    echo "IGCM_sys_UnTar : erreur."
579    IGCM_debug_Exit "IGCM_sys_UnTar"
580  fi
581  IGCM_debug_PopStack "IGCM_sys_UnTar"
582}
583
584#D-#==================================================
585#D-function IGCM_sys_Qsub
586#D-* Purpose: Qsub new job
587#D-* Examples:
588#D-
589function IGCM_sys_Qsub {
590  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
591  if ( $DEBUG_sys ) ; then
592    echo "IGCM_sys_Qsub :" $@
593  fi
594  echo "${@} > ${SUBMIT_DIR}/${Script_Output} 2>&1 &"
595  ${@} > ${SUBMIT_DIR}/${Script_Output} 2>&1 &
596  if [ $? -gt 0 ] ; then
597    echo "IGCM_sys_Qsub : erreur ${options} $1"
598    IGCM_debug_Exit "IGCM_sys_Qsub"
599  fi
600  IGCM_debug_PopStack "IGCM_sys_Qsub"
601}
602
603#D-#==================================================
604#D-function IGCM_sys_QsubPost
605#D-* Purpose: Qsub new job on scalaire
606#D-* Examples:
607#D-
608function IGCM_sys_QsubPost {
609  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
610  if ( $DEBUG_sys ) ; then
611    echo "IGCM_sys_QsubPost :" $@
612  fi
613  echo "${libIGCM_POST}/$1.job > ${POST_DIR}/$1.${PeriodDateEnd}.out 2>&1 &"
614  ${libIGCM_POST}/$1.job > ${POST_DIR}/${Script_Post_Output}.out 2>&1 &
615  if [ $? -gt 0 ] ; then
616    echo "IGCM_sys_QsubPost : erreur " $@
617    IGCM_debug_Exit "IGCM_sys_QsubPost"
618  fi
619  IGCM_debug_PopStack "IGCM_sys_QsubPost"
620}
621
622#D-*************************
623#D- File transfer functions
624#D-*************************
625#D-
626
627#D-#==================================================
628#D-function IGCM_sys_Rsync_out
629#D-* Purpose: treat return val of rsync
630#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
631#D-  Error values and explanations can depend on your system version.
632function IGCM_sys_Rsync_out {
633  status=$1
634  if [ ! $status ] ; then
635    echo "rsync error !"
636  fi
637
638  if [ $MYLANG = "fr" ]; then
639    case $status in
640    0)  return ;;
641    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
642      echo "Erreur de syntaxe ou d'utilisation."
643      return;;
644    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
645      echo "Incompatibilité de protocole."
646      return;;
647    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
648      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
649      echo "répertoires"
650      return;;
651    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
652      echo "Action demandée non supportée : une tentative de manipulation de"
653      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
654      echo "été faite ; ou une option qui est supportée par le  client  mais"
655      echo "pas par le serveur a été spécifiée."
656      return;;
657    10) echo "Erreur de rsync ; RERR_SOCKETIO"
658      echo "Erreur dans le socket d'entrée sortie"
659      return;;
660    11) echo "Erreur de rsync ; RERR_FILEIO"
661      echo "Erreur d'entrée sortie fichier"
662      return;;
663    12) echo "Erreur de rsync ; RERR_STREAMIO"
664      echo "Erreur dans flux de donnée du protocole rsync"
665      return;;
666    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
667      echo "Erreur avec les diagnostics du programme"
668      return;;
669    14) echo "Erreur de rsync ; RERR_IPC"
670      echo "Erreur dans le code IPC"
671      return;;
672    20) echo "Erreur de rsync ; RERR_SIGNAL"
673      echo "SIGUSR1 ou SIGINT reçu"
674      return;;
675    21) echo "Erreur de rsync ; RERR_WAITCHILD"
676      echo "Une erreur retournée par waitpid()"
677      return;;
678    22) echo "Erreur de rsync ; RERR_MALLOC"
679      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
680      return;;
681    23) echo ""
682      echo "Erreur fichier inexistant"
683      return;;
684    30) echo "Erreur de rsync ; RERR_TIMEOUT"
685      echo "Temps d'attente écoulé dans l'envoi/réception de données"
686      return;;
687    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $status
688      return;;
689    esac
690  elif [ $MYLANG = "en" ] ; then
691    case $status in
692    0)  return;;
693    1)  echo "rsync error : Syntax or usage error "
694      return;;
695    2)  echo "rsync error : Protocol incompatibility "
696      return;;
697    3)  echo "rsync error : Errors selecting input/output files, dirs"
698      return;;
699    4)  echo "rsync error : Requested action not supported: an attempt"
700      echo "was made to manipulate 64-bit files on a platform that cannot support"
701      echo "them; or an option was specified that is supported by the client and"
702      echo "not by the server."
703      return;;
704    5)  echo "rsync error : Error starting client-server protocol"
705      return;;
706    10) echo "rsync error : Error in socket I/O "
707      return;;
708    11) echo "rsync error : Error in file I/O "
709      return;;
710    12) echo "rsync error : Error in rsync protocol data stream "
711      return;;
712    13) echo "rsync error : Errors with program diagnostics "
713      return;;
714    14) echo "rsync error : Error in IPC code "
715      return;;
716    20) echo "rsync error : Received SIGUSR1 or SIGINT "
717      return;;
718    21) echo "rsync error : Some error returned by waitpid() "
719      return;;
720    22) echo "rsync error : Error allocating core memory buffers "
721      return;;
722    23) echo "rsync error : Partial transfer due to error"
723      return;;
724    24) echo "rsync error : Partial transfer due to vanished source files"
725      return;;
726    30) echo "rsync error : Timeout in data send/receive "
727      return;;
728    *)  echo "rsync error : return code of rsync unknown :" $status
729      return;;
730    esac
731  else
732    echo "unknown language $MYLANG."
733    return
734  fi
735}
736
737#D-#==================================================
738#D-function IGCM_sys_Miror_libIGCM
739#D-* Purpose: Mirror libIGCM PATH and lib to frontend
740#D-* Examples:
741#D-
742function IGCM_sys_Mirror_libIGCM {
743  IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
744  if ( $DEBUG_sys ) ; then
745    echo "IGCM_sys_Mirror_libIGCM"
746  fi
747
748  typeset status
749
750  mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
751
752  echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > /tmp/out_command.$$ 2>&1
753  ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> /tmp/out_command.$$ 2>&1
754  status=$?
755
756  if [ ${status} -gt 0 ] ; then
757    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on frontend."
758    cat /tmp/out_command.$$
759  fi
760  IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
761}
762
763#D-#==================================================
764#D-function IGCM_sys_Cp
765#D-* Purpose: generic cp
766#D-* Examples:
767#D-
768function IGCM_sys_Cp {
769  IGCM_debug_PushStack "IGCM_sys_Cp" $@
770  if ( $DEBUG_sys ) ; then
771    echo "IGCM_sys_Cp :" $@
772  fi
773
774  typeset status
775
776  echo cp $@ > /tmp/out_command.$$ 2>&1
777  \cp $@ >> /tmp/out_command.$$ 2>&1
778  status=$?
779
780  if [ ${status} -gt 0 ] ; then
781    echo "IGCM_sys_Cp : error code ${status}"
782    cat /tmp/out_command.$$
783    IGCM_debug_Exit "IGCM_sys_Cp"
784  else
785    \rm /tmp/out_command.$$
786  fi
787  IGCM_debug_PopStack "IGCM_sys_Cp"
788}
789
790#D-#==================================================
791#D-function IGCM_sys_Rm
792#D-* Purpose: generic rm
793#D-* Examples:
794#D-
795function IGCM_sys_Rm {
796  IGCM_debug_PushStack "IGCM_sys_Rm" $@
797  if ( $DEBUG_sys ) ; then
798    echo "IGCM_sys_Rm :" $@
799  fi
800
801  typeset status
802
803  echo rm $@ > /tmp/out_command.$$ 2>&1
804  \rm $@ >> /tmp/out_command.$$ 2>&1
805  status=$?
806
807  if [ ${status} -gt 0 ] ; then
808    echo "IGCM_sys_Rm : error code ${status}"
809    cat /tmp/out_command.$$
810    IGCM_debug_Exit "IGCM_sys_Rm"
811  else
812    \rm /tmp/out_command.$$
813  fi
814  IGCM_debug_PopStack "IGCM_sys_Rm"
815}
816
817#D-#==================================================
818#D-function IGCM_sys_RmRunDir
819#D-* Purpose: rm tmpdir (dummy function most of the time batch
820#D-                      scheduler will do the job)
821#D-* Examples:
822#D-
823function IGCM_sys_RmRunDir {
824  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
825  if ( $DEBUG_sys ) ; then
826    echo "IGCM_sys_RmRunDir :" $@
827    echo "Dummy call, let the user do that."
828#    IGCM_sys_Rm -rf ${RUN_DIR}
829  fi
830  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
831}
832
833#D-#==================================================
834#D-function IGCM_sys_Mv
835#D-* Purpose: generic move
836#D-* Examples:
837#D-
838function IGCM_sys_Mv {
839  IGCM_debug_PushStack "IGCM_sys_Mv" $@
840  if ( $DEBUG_sys ) ; then
841    echo "IGCM_sys_Mv :" $@
842  fi
843
844  if [ $DRYRUN = 0 ]; then
845
846    typeset status
847
848    echo mv $@ > /tmp/out_command.$$ 2>&1
849    \mv $@ >> /tmp/out_command.$$ 2>&1
850    status=$?
851
852    if [ ${status} -gt 0 ] ; then
853      echo "IGCM_sys_Mv : error code ${status}"
854      cat /tmp/out_command.$$
855      IGCM_debug_Exit "IGCM_sys_Mv"
856    else
857      \rm /tmp/out_command.$$
858    fi
859  fi
860
861  IGCM_debug_PopStack "IGCM_sys_Mv"
862}
863
864#D-#==================================================
865#D-function IGCM_sys_Put_Dir
866#D-* Purpose: Copy a complete directory on $(ARCHIVE)
867#D-* Examples:
868#D-
869function IGCM_sys_Put_Dir {
870  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
871  if ( $DEBUG_sys ) ; then
872    echo "IGCM_sys_Put_Dir :" $@
873  fi
874  if [ $DRYRUN = 0 ]; then
875    if [ ! -d ${1} ] ; then
876      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
877      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
878      return
879    fi
880
881    typeset status
882
883    # Only if we use rsync
884    #IGCM_sys_TestDirArchive $( dirname $2 )
885    #
886    #USUAL WAY
887    \cp -r $1 $2 > /tmp/out_command.$$ 2>&1
888    status=$?
889
890    if [ ${status} -gt 0 ] ; then
891      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
892      cat /tmp/out_command.$$
893      IGCM_debug_Exit "IGCM_sys_Put_Dir"
894    else
895      \rm /tmp/out_command.$$
896    fi
897  fi
898  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
899}
900
901#D-#==================================================
902#D-function IGCM_sys_Get_Dir
903#D-* Purpose: Copy a complete directory from ${ARCHIVE}
904#D-* Examples:
905#D-
906function IGCM_sys_Get_Dir {
907  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
908  if ( $DEBUG_sys ) ; then
909    echo "IGCM_sys_Get_Dir :" $@
910  fi
911  if [ $DRYRUN = 0 ]; then
912    if [ ! -d ${1} ] ; then
913      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
914      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
915      return
916    fi
917
918    typeset status
919
920    #USUAL WAY
921    \cp -ur $1 $2 > /tmp/out_command.$$ 2>&1
922    status=$?
923
924    if [ ${status} -gt 0 ] ; then
925      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
926      cat /tmp/out_command.$$
927      IGCM_debug_Exit "IGCM_sys_Get_Dir"
928    else
929      \rm /tmp/out_command.$$
930    fi
931  fi
932  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
933}
934
935#D-#==================================================
936#D-function IGCM_sys_Get_Master
937#D-* Purpose: Copy a complete directory from MASTER filesystem
938#D-* Examples:
939#D-
940function IGCM_sys_Get_Master {
941  IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
942  if ( $DEBUG_sys ) ; then
943    echo "IGCM_sys_Get_Master :" $@
944  fi
945  if [ $DRYRUN = 0 ]; then
946    if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
947      echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
948      IGCM_debug_PopStack "IGCM_sys_Get_Master"
949      return
950    fi
951
952    typeset NB_ESSAI DELAI status i
953    # number of tentative
954    NB_ESSAI=3
955    # time delay between tentative
956    DELAI=2
957
958    i=0
959    while [ $i -lt $NB_ESSAI ] ; do
960      \cp -urL $1 $2 > /tmp/out_command.$$ 2>&1
961      status=$?
962      if [ ${status} -gt 0 ]; then
963        IGCM_debug_Print 2 "IGCM_sys_Get_Master : cp failed error code ${status} ${i}/${NB_ESSAI}"
964        IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
965        sleep $DELAI
966      else
967        break
968      fi
969      (( i = i + 1 ))
970    done
971
972    if [ ${status} -gt 0 ] ; then
973      echo "IGCM_sys_Get_Master : error."
974      cat /tmp/out_command.$$
975      IGCM_debug_Exit "IGCM_sys_Get_Master"
976    else
977      \rm /tmp/out_command.$$
978    fi
979  fi
980  IGCM_debug_PopStack "IGCM_sys_Get_Master"
981}
982
983#====================================================
984#- Call IGCM_sys_Mirror_libIGCM now !
985if ( $MirrorlibIGCM ) ; then
986  IGCM_sys_Mirror_libIGCM
987fi
988
989#D-#==================================================
990#D-function IGCM_sys_Put_Rest
991#D-* Purpose: Put computied restarts on ${ARCHIVE}.
992#D-           File and target directory must exist.
993#D-* Examples:
994#D-
995function IGCM_sys_Put_Rest {
996  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
997  if ( $DEBUG_sys ) ; then
998    echo "IGCM_sys_Put_Rest :" $@
999  fi
1000  if [ $DRYRUN = 0 ]; then
1001    if [ ! -f ${1} ] ; then
1002      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1003      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1004    fi
1005
1006    typeset status
1007    #
1008    if [ X${JobType} = XRUN ] ; then
1009      IGCM_sys_Chmod 444 ${1}
1010    fi
1011    #
1012    # Only if we use rsync
1013    #IGCM_sys_MkdirArchive $( dirname $2 )
1014    #
1015    #USUAL WAY
1016    \cp $1 $2 > /tmp/out_command.$$ 2>&1
1017    status=$?
1018
1019#       #RSYNC WITH NETWORK SSH CALL
1020#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > /tmp/out_command.$$ 2>&1
1021#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> /tmp/out_command.$$ 2>&1
1022
1023#       #RSYNC WITH NFS USE
1024#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > /tmp/out_command.$$ 2>&1
1025#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> /tmp/out_command.$$ 2>&1
1026
1027#       status=$?
1028#       IGCM_sys_Rsync_out $status
1029
1030#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk /tmp/out_command.$$
1031#       (( status=status+$? ))
1032
1033    if [ ${status} -gt 0 ] ; then
1034      echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
1035      cat /tmp/out_command.$$
1036      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1037    else
1038      \rm /tmp/out_command.$$
1039    fi
1040  fi
1041  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1042}
1043
1044#D-#==================================================
1045#D-function IGCM_sys_PutBuffer_Rest
1046#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1047#D-           File and target directory must exist.
1048#D-* Examples:
1049#D-
1050function IGCM_sys_PutBuffer_Rest {
1051  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1052  if ( $DEBUG_sys ) ; then
1053    echo "IGCM_sys_PutBuffer_Rest :" $@
1054  fi
1055  if [ $DRYRUN = 0 ]; then
1056    if [ ! -f ${1} ] ; then
1057      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1058      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1059    fi
1060
1061    typeset status
1062    #
1063    if [ X${JobType} = XRUN ] ; then
1064      IGCM_sys_Chmod 444 ${1}
1065    fi
1066
1067    #
1068    # USUAL WAY
1069    \cp $1 $2 > /tmp/out_command.$$ 2>&1
1070    status=$?
1071
1072    if [ ${status} -gt 0 ] ; then
1073      echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
1074      [ -f ${2} ] && ls -l ${2}
1075      [ -f ${2}/${1} ] && ls -l ${2}/${1}
1076      cat /tmp/out_command.$$
1077      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1078    else
1079      \rm /tmp/out_command.$$
1080    fi
1081  fi
1082  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1083}
1084
1085#D-#==================================================
1086#D-function IGCM_sys_PrepareTaredRestart
1087#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
1088#D-* Examples:
1089#D-
1090function IGCM_sys_PrepareTaredRestart {
1091  IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
1092
1093  IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
1094}
1095
1096#D-#==================================================
1097#D-function IGCM_sys_Put_Out
1098#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1099#D-* Examples:
1100#D-
1101function IGCM_sys_Put_Out {
1102  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1103  if ( $DEBUG_sys ) ; then
1104    echo "IGCM_sys_Put_Out :" $@
1105  fi
1106
1107  typeset status
1108
1109  if [ $DRYRUN = 0 ]; then
1110    if [ ! -f ${1} ] ; then
1111      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1112      IGCM_debug_PopStack "IGCM_sys_Put_Out"
1113      return 1
1114    fi
1115    #
1116    IGCM_sys_MkdirArchive $( dirname $2 )
1117    #
1118    if [ X${JobType} = XRUN ] ; then
1119      if [ X${3} = X ] ; then
1120        IGCM_sys_Chmod 444 ${1}
1121      fi
1122    fi
1123    #
1124
1125    echo ${RSYNC} ${RSYNC_opt} $1 $2 > /tmp/out_command.$$ 2>&1
1126    ${RSYNC} ${RSYNC_opt} $1 $2 >> /tmp/out_command.$$ 2>&1
1127    status=$?
1128    IGCM_sys_Rsync_out $status
1129
1130    ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk /tmp/out_command.$$
1131    (( status=status+$? ))
1132
1133    if [ ${status} -gt 0 ] ; then
1134      IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
1135      cat /tmp/out_command.$$
1136      IGCM_debug_Exit "IGCM_sys_Put_Out"
1137    else
1138      \rm /tmp/out_command.$$
1139    fi
1140  fi
1141  IGCM_debug_PopStack "IGCM_sys_Put_Out"
1142  return 0
1143}
1144
1145#D-#==================================================
1146#D-function IGCM_sys_PutBuffer_Out
1147#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1148#D-* Examples:
1149#D-
1150function IGCM_sys_PutBuffer_Out {
1151  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1152  if ( $DEBUG_sys ) ; then
1153    echo "IGCM_sys_PutBuffer_Out :" $@
1154  fi
1155
1156  typeset NB_ESSAI DELAI status i exist skip
1157
1158  # number of tentative
1159  NB_ESSAI=3
1160  # time delay between tentative
1161  DELAI=2
1162
1163  if [ $DRYRUN = 0 ]; then
1164    if [ ! -f ${1} ] ; then
1165      echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1166      IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1167      return 1
1168    fi
1169    #
1170    IGCM_sys_Mkdir $( dirname $2 )
1171    #
1172
1173    exist=false
1174    skip=false
1175    if [ -f $2 ] ; then
1176      IGCM_debug_Print 1 "$2 already exist"
1177      exist=true
1178      if [ "X$( diff $1 $2 )" = X ] ; then
1179        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1180        status=0
1181        skip=true
1182      else
1183        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1184        skip=false
1185      fi
1186    fi
1187    #
1188    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1189      IGCM_sys_Chmod u+w $2
1190    fi
1191
1192    if [ X${skip} = Xfalse ] ; then
1193      i=0
1194      while [ $i -lt $NB_ESSAI ] ; do
1195        # USUAL WAY
1196        \cp $1 $2 > /tmp/out_command.$$ 2>&1
1197        status=$?
1198        if [ ${status} -gt 0 ]; then
1199          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1200          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
1201          [ -f ${2} ] && ls -l ${2}
1202          [ -f ${2}/${1} ] && ls -l ${2}/${1}
1203          sleep $DELAI
1204        else
1205          break
1206        fi
1207        (( i = i + 1 ))
1208      done
1209    fi
1210
1211    if [ ${status} -gt 0 ] ; then
1212      echo "IGCM_sys_PutBuffer_Out : error."
1213      [ -f ${2} ] && ls -l ${2}
1214      [ -f ${2}/${1} ] && ls -l ${2}/${1}
1215      cat /tmp/out_command.$$
1216      IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1217    else
1218
1219      if [ X${JobType} = XRUN ] ; then
1220        if [ X${3} = X ] ; then
1221          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
1222          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
1223        fi
1224      fi
1225
1226      \rm /tmp/out_command.$$
1227    fi
1228  fi
1229  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1230  return 0
1231}
1232
1233#D-#==================================================
1234#D-function IGCM_sys_Get
1235#D-* Purpose: Get a file from ${ARCHIVE}
1236#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1237#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
1238function IGCM_sys_Get {
1239  IGCM_debug_PushStack "IGCM_sys_Get" $@
1240
1241  typeset DEST status dm_liste
1242
1243  if ( $DEBUG_sys ) ; then
1244    echo "IGCM_sys_Get :" $@
1245  fi
1246  if [ $DRYRUN -le 2 ]; then
1247    if [ X${1} = X'/l' ] ; then
1248      eval set +A dm_liste \${${2}}
1249    else
1250      dm_liste=${1}
1251    fi
1252    eval DEST=\${${#}}
1253
1254    # test if the (first) file is present in the old computation :
1255    IGCM_sys_TestFileArchive ${dm_liste[0]}
1256    status=$?
1257    if [ ${status} -gt 0 ] ; then
1258      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
1259      IGCM_debug_Exit "IGCM_sys_Get"
1260      return
1261    fi
1262
1263    #USUAL WAY
1264    \cp ${dm_liste[*]} ${DEST} > /tmp/out_command.$$ 2>&1
1265    status=$?
1266
1267    if [ ${status} -gt 0 ] ; then
1268      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
1269      cat /tmp/out_command.$$
1270      IGCM_debug_Exit "IGCM_sys_Get"
1271    else
1272      \rm /tmp/out_command.$$
1273    fi
1274  fi
1275  IGCM_debug_PopStack "IGCM_sys_Get"
1276}
1277
1278#D-#==================================================
1279#D-function IGCM_sys_GetBuffer
1280#D-* Purpose: Get a file from ${SCRATCHDIR}
1281#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1282#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1283function IGCM_sys_GetBuffer {
1284  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1285
1286  typeset DEST buf_liste target file_work
1287  typeset NB_ESSAI DELAI status i
1288
1289  if ( $DEBUG_sys ) ; then
1290    echo "IGCM_sys_GetBuffer :" $@
1291  fi
1292
1293  # number of tentative
1294  NB_ESSAI=3
1295  # time delay between tentative
1296  DELAI=2
1297
1298  if [ $DRYRUN -le 2 ]; then
1299    if [ X${1} = X'/l' ] ; then
1300      eval set +A buf_liste \${${2}}
1301    else
1302      eval set +A buf_liste ${1}
1303    fi
1304    eval DEST=\${${#}}
1305
1306    #USUAL WAY
1307    if [ X${1} = X'/l' ] ; then
1308      for target in ${buf_liste[*]} ; do
1309        local_file=$( basename ${target} )
1310        i=0
1311        while [ $i -lt $NB_ESSAI ] ; do
1312          \cp ${target} ${DEST}/${local_file} >> /tmp/out_command.$$ 2>&1
1313          status=$?
1314          if [ ${status} -gt 0 ]; then
1315            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1316            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1317            sleep $DELAI
1318          else
1319            break
1320          fi
1321          (( i = i + 1 ))
1322        done
1323        if [ ${status} -gt 0 ] ; then
1324          echo "IGCM_sys_Get : error"
1325          cat /tmp/out_command.$$
1326          \rm /tmp/out_command.$$
1327          IGCM_debug_Exit "IGCM_sys_GetBuffer"
1328        else
1329          \rm /tmp/out_command.$$
1330        fi
1331      done
1332    else
1333      i=0
1334      while [ $i -lt $NB_ESSAI ] ; do
1335        \cp ${buf_liste} ${DEST} >> /tmp/out_command.$$ 2>&1
1336        status=$?
1337        if [ ${status} -gt 0 ]; then
1338          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1339          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1340          sleep $DELAI
1341        else
1342          break
1343        fi
1344        (( i = i + 1 ))
1345      done
1346      if [ ${status} -gt 0 ] ; then
1347        echo "IGCM_sys_Get : error"
1348        cat /tmp/out_command.$$
1349        \rm /tmp/out_command.$$
1350        IGCM_debug_Exit "IGCM_sys_GetBuffer"
1351      else
1352        \rm /tmp/out_command.$$
1353      fi
1354    fi
1355  fi
1356  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1357}
1358
1359#D-#==================================================
1360#D-function IGCM_sys_GetDate_FichWork
1361#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1362#D-* Examples:
1363#D-
1364function IGCM_sys_GetDate_FichWork {
1365  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1366  if ( $DEBUG_sys ) ; then
1367    echo "IGCM_sys_GetDate_FichWork :" $@
1368  fi
1369  typeset dateF
1370  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1371  eval ${2}=${dateF[5]}
1372
1373  # donne la date filesys d'un fichier sur la machine work
1374  IGCM_debug_PopStack "IGCM_sys_FichWork"
1375}
1376
1377#D-#==================================================
1378#D-function IGCM_sys_GetDate_FichArchive
1379#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1380#D-* Examples:
1381#D-
1382function IGCM_sys_GetDate_FichArchive {
1383  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1384  if ( $DEBUG_sys ) ; then
1385    echo "IGCM_sys_GetDate_FichArchive :" $@
1386  fi
1387  typeset dateF
1388  set +A dateF -- $( ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1} )
1389  eval ${2}=${dateF[5]}
1390
1391  IGCM_debug_PopStack "IGCM_sys_FichArchive"
1392}
1393
1394#D-#==================================================
1395#D-function IGCM_sys_Dods_Rm
1396#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1397#D-* Examples:
1398#D-
1399function IGCM_sys_Dods_Rm {
1400  if ( $DEBUG_sys ) ; then
1401    echo "IGCM_sys_Dods_Rm :" $@
1402  fi
1403  return 0
1404}
1405
1406#D-#==================================================
1407#D-function IGCM_sys_Dods_Cp
1408#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1409#D-* Examples:
1410#D-
1411function IGCM_sys_Dods_Cp {
1412  if ( $DEBUG_sys ) ; then
1413    echo "IGCM_sys_Dods_Cp :" $@
1414  fi
1415  return 0
1416}
1417
1418#D-#==================================================
1419#D-function IGCM_sys_Put_Dods
1420#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
1421#D-* Examples:
1422#D-
1423function IGCM_sys_Put_Dods {
1424  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1425  if ( $DEBUG_sys ) ; then
1426    echo "IGCM_sys_Put_Dods :" $@
1427  fi
1428  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1429}
1430
1431##############################################################
1432# REBUILD OPERATOR
1433
1434#D-#==================================================
1435#D-function IGCM_sys_rebuild
1436#D-* Purpose: rebuild parallel files
1437#D-* Examples:
1438#D-
1439function IGCM_sys_rebuild {
1440  IGCM_debug_PushStack "IGCM_sys_rebuild" $@
1441  if ( $DEBUG_sys ) ; then
1442    echo "IGCM_sys_rebuild :" $@
1443  fi
1444
1445  typeset NB_ESSAI DELAI status i firstArg
1446  # number of tentative
1447  NB_ESSAI=3
1448  # time delay between tentative
1449  DELAI=2
1450
1451  i=0
1452  while [ $i -lt $NB_ESSAI ] ; do
1453    rebuild -f -o $@ > /tmp/out_command.$$ 2>&1
1454    status=$?
1455    if [ ${status} -gt 0 ] ; then
1456      IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
1457      cat /tmp/out_command.$$
1458      \rm /tmp/out_command.$$
1459      IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1460      firstArg=${1}
1461      \rm ${firstArg}
1462      sleep $DELAI
1463    else
1464      \rm /tmp/out_command.$$
1465      break
1466    fi
1467    (( i = i + 1 ))
1468  done
1469
1470  if [ ${status} -gt 0 ] ; then
1471    echo "IGCM_sys_rebuild : rebuild error code is ${status}"
1472    IGCM_debug_Exit "rebuild"
1473  fi
1474
1475  IGCM_debug_PopStack "IGCM_sys_rebuild"
1476}
1477
1478#D-#==================================================
1479#D-function IGCM_sys_rebuild_station
1480#D-* Purpose: rebuild parallel files describing station
1481#D-* Examples:
1482#D-
1483function IGCM_sys_rebuild_station {
1484  IGCM_debug_PushStack "IGCM_sys_rebuild_station" $@
1485  typeset i list_opt file_in file_out prefix_invert list_invert
1486  if ( $DEBUG_sys ) ; then
1487    echo "IGCM_sys_rebuild_station :" $@
1488  fi
1489  list_opt=$@
1490
1491  # Invert Axis : t,x -> x,t
1492  #               t,pres,x -> x,t,pres
1493  # So that we can concatenate along x
1494  i=0
1495  for file_in in ${list_opt} ; do
1496    (( i = i + 1))
1497    [ ${i} = 1 ] && file_out=${file_in} && continue
1498    # detect time counter and do the job only if present
1499    var_unlim=$(ncdump -h ${file_in} | grep UNLIMITED | cut -d ' ' -f 1 | sed -e 's/^[ \t]*//' -e 's/[ \t]*$//')
1500    if [ X${var_unlim} = Xtime_counter ] ; then
1501      prefix_invert=$( basename ${file_in} .nc )
1502      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1503      list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1504    fi
1505  done
1506
1507  # Concatenate
1508  IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1509
1510  # Re-ivert file
1511  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1512
1513  # Station re-ordering is too expansive to be run within libIGCM
1514  # This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1515  # This re-ordering must be done "in memory" by the cmorization process
1516  # Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1517  # BEGIN reordering
1518
1519  # Only LMDZ text output contains the exact ordering of the station.
1520  # We isolate this in the code below:
1521  #  0  38  -157.5000000000000  70.98591549295774
1522  #  0  54  27.49999999999999   67.18309859154928
1523  #  0  56  -62.50000000000001  82.39436619718309
1524  #  0  79  12.49999999999999   78.59154929577466
1525  #  0  116 -165.0000000000000  76.05633802816901
1526  #  0  117 130.0000000000000   70.98591549295774
1527  #  0  118 110.0000000000000   87.46478873239437
1528  #  1  40  4.999999999999995   51.97183098591550
1529#  typeset iStation iProc list_opt file_in file_out prefix_invert
1530#  typeset -Z4 j4
1531#  typeset -Z3 j3
1532
1533#  unset list_opt
1534#  set +A list_opt $@
1535
1536  # Filename after rebuild
1537#  file_out=${list_opt[0]}
1538  # Prefix of output files
1539#  prefix_invert=$( basename ${file_out} .nc )
1540  # Number of procs
1541#  num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} | wc -l )
1542
1543#  iProc=0
1544#  while [ ${iProc} -lt ${num_proc} ] ; do
1545    # Array containing Station as a number
1546#    unset proc_stn
1547#    set +A proc_stn $( grep "iophy_mpi rank ip lon lat  $iProc" ${PREFIX}_${Exe_Output} | sed -e "s/iophy_mpi rank ip lon lat //g" | gawk ' {print $2}' )
1548    # Number of stations produced by processor proc
1549#    stationLast=${#proc_stn[*]}
1550    # Proc number on 4 digits
1551#    j4=${iProc}
1552    # Init
1553#    iStation=0
1554#    while [ ${iStation} -lt ${stationLast} ] ; do
1555      # Station number on 3 digits
1556#      j3=${proc_stn[${iStation}]}
1557      # Extract station
1558      # Invert Axis : t,x -> x,t
1559      #               t,pres,x -> x,t,pres
1560      # So that we can concatenate along x
1561#      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1562#      (( iStation = iStation + 1 ))
1563#    done
1564#    (( iProc = iProc + 1 ))
1565#  done
1566
1567  # Concatenate all station along x
1568#  IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1569
1570  # Re-invert file
1571#  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1572
1573  # END reordering
1574
1575  IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1576}
1577
1578############################################################
1579# Activate Running Environnment Variables
1580
1581#D-#==================================================
1582#D-function IGCM_sys_desactiv_variables
1583#D-* Purpose: set environement variables prior to execution
1584#D-* Examples:
1585#D-
1586function IGCM_sys_activ_variables {
1587  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1588  if ( $DEBUG_sys ) ; then
1589    echo "IGCM_sys_activ_variables"
1590  fi
1591
1592# --------------------------------------------------------------------
1593#D- MPI specifications
1594# --------------------------------------------------------------------
1595
1596# --------------------------------------------------------------------
1597#D- Other specifications
1598# --------------------------------------------------------------------
1599
1600  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1601}
1602
1603############################################################
1604# Desactivate Running Environnment Variables
1605
1606#D-#==================================================
1607#D-function IGCM_sys_desactiv_variables
1608#D-* Purpose: unset environement variables after execution
1609#D-* Examples:
1610#D-
1611function IGCM_sys_desactiv_variables {
1612  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1613  if ( $DEBUG_sys ) ; then
1614    echo "IGCM_sys_desactiv_variables"
1615  fi
1616# --------------------------------------------------------------------
1617#D- MPI specifications
1618# --------------------------------------------------------------------
1619
1620# --------------------------------------------------------------------
1621#D- Other specifications
1622# --------------------------------------------------------------------
1623
1624  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1625}
1626
1627############################################################
1628# Build MPI/OMP scripts run file (dummy function)
1629
1630#D-#==================================================
1631#D-function IGCM_sys_build_run_file
1632#D-* Purpose: build run file (deprecated)
1633#D-* Examples:
1634#D-
1635function IGCM_sys_build_run_file {
1636
1637  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1638
1639}
1640
1641############################################################
1642# Build MPI/OMP scripts
1643
1644#D-#==================================================
1645#D-function IGCM_sys_build_execution_scripts
1646#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1647#D-* Examples:
1648#D-
1649function IGCM_sys_build_execution_scripts
1650{
1651  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1652  if ( $DEBUG_sys ) ; then
1653    echo "IGCM_sys_build_execution_scripts " $@
1654  fi
1655  typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
1656  typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
1657  typeset num_corempi nombre_restant_node nombre_restant_comp
1658
1659  if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
1660    IGCM_debug_Exit "IGCM_sys_default build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
1661  fi
1662
1663  if ( ${OK_PARA_MPMD} ) ; then
1664
1665    if [ -f run_file ] ; then
1666      IGCM_sys_Rm -f run_file
1667    fi
1668    touch run_file
1669
1670    if ( ${OK_PARA_OMP} ) ; then
1671
1672      #  Hosts treatment
1673
1674      ${HOST_MPIRUN_COMMAND} hostname | sort | uniq > hosts.tmp
1675
1676      i=0
1677      rm -f hosts
1678      IGCM_debug_Print 1 "sys $( hostname ), Hosts avaible :"
1679      for nodes in `cat hosts.tmp` ; do
1680        host[$i]=$nodes
1681        echo "${host[$i]} slots=1 max_slots=1" >> hosts
1682        IGCM_debug_Print 1 ${host[$i]}
1683        i=$((i+1))
1684      done
1685      rm -f hosts.tmp
1686
1687      listnodes=${host[*]}
1688
1689      EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
1690
1691      # Initialisation
1692
1693      init_node=y
1694      node_num_current=0
1695      start_num=0
1696      init_exec=n
1697
1698      # Test : if oasis is there, we put it at the first position
1699
1700      for comp in ${config_ListOfComponents[*]} ; do
1701
1702        if [ "X${comp}" = "XCPL" ]  ; then
1703
1704          eval ExeNameIn=\${config_Executable_${comp}[0]}
1705          eval ExeNameOut=\${config_Executable_${comp}[1]}
1706
1707          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1708          echo ""  >> script_${ExeNameOut}.ksh
1709          #echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1710          #echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1711          #echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1712          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err"  >> script_${ExeNameOut}.ksh
1713          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1714
1715          init_node=n
1716
1717          (( nombre_restant_node = NUM_COREPERNODE - 1 ))
1718          node_num_current=0
1719          node_current=${host[${node_num_current}]}
1720
1721          EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
1722
1723          init_exec=y
1724          start_num=1
1725
1726        fi
1727
1728      done
1729
1730      # Then loop on the components (except for oasis)
1731
1732      for comp in ${config_ListOfComponents[*]} ; do
1733
1734        eval ExeNameIn=\${config_Executable_${comp}[0]}
1735        eval ExeNameOut=\${config_Executable_${comp}[1]}
1736
1737        # Only if we really have an executable for the component :
1738        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
1739
1740          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1741          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1742
1743          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1744          # echo "set -vx" >> script_${ExeNameOut}.ksh
1745          echo ""  >> script_${ExeNameOut}.ksh
1746          #echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1747          #echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1748          #echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1749          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1750          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
1751          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
1752          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1753
1754          node_num=0
1755
1756          # We define the number of MPI process to be assigned for the component
1757
1758          nombre_restant_comp=${comp_proc_mpi_loc}
1759
1760          # Loop on the allocated nodes
1761
1762          for node in ${listnodes} ; do
1763
1764            # We go to the current node
1765            if [ ${node_num} = ${node_num_current} ] ; then
1766
1767              node_current=${host[${node_num_current}]}
1768
1769              # If first time on the node : initialisation
1770
1771              if [ ${init_node} = y ] ; then
1772                nombre_restant_node=${NUM_COREPERNODE}
1773              fi
1774
1775              # Test on the number of OMP threads
1776
1777              if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
1778                (( node_num = node_num + 1 ))
1779                node_num_current=${node_num}
1780                init_node=y
1781                continue
1782              fi
1783
1784              # Number of MPI process to assign
1785
1786              (( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
1787
1788              if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
1789                num_corempi=${nombre_restant_comp}
1790              fi
1791
1792              (( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
1793              (( nombre_restant_comp = nombre_restant_comp - num_corempi ))
1794
1795              if [ ${init_exec} = y ] ; then
1796                EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1797              else
1798                EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
1799                init_exec=y
1800              fi
1801
1802              ((  start_num = num_corempi + start_num ))
1803
1804            else
1805
1806              (( node_num = node_num + 1 ))
1807              continue
1808            fi
1809
1810            # Test on the number of core/process remaining on the node/component
1811
1812            if [ ${nombre_restant_node} = 0 ] ; then
1813              (( node_num = node_num + 1 ))
1814              node_num_current=${node_num}
1815              init_node=y
1816
1817              if [ ${nombre_restant_comp} = 0 ] ; then
1818                break 1
1819              fi
1820            else
1821
1822              node_num_current=${node_num}
1823              init_node=n
1824
1825              if [ ${nombre_restant_comp} = 0 ] ; then
1826                break 1
1827              fi
1828            fi
1829          done
1830        fi
1831      done
1832
1833    else
1834
1835      # Then first loop on the components for the coupler ie oasis
1836
1837      ## the coupler ie oasis must be the first one
1838      for comp in ${config_ListOfComponents[*]} ; do
1839
1840        eval ExeNameOut=\${config_Executable_${comp}[1]}
1841
1842        # for CPL component only
1843        if [ "X${comp}" = "XCPL" ] ; then
1844          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1845          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
1846        fi
1847      done
1848
1849      # Then second loop on the components
1850
1851      for comp in ${config_ListOfComponents[*]} ; do
1852
1853        eval ExeNameOut=\${config_Executable_${comp}[1]}
1854
1855        # Only if we really have an executable for the component and not the coupler ie oasis:
1856        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1857          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1858          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1859        fi
1860      done
1861      IGCM_sys_Chmod u+x run_file
1862
1863      EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
1864
1865    fi
1866
1867  else # Only one executable. launch it.
1868
1869    for comp in ${config_ListOfComponents[*]} ; do
1870
1871      # Only if we really have an executable for the component :
1872      eval ExeNameOut=\${config_Executable_${comp}[1]}
1873      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1874
1875        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1876        echo ""  >> script_${ExeNameOut}.ksh
1877        if ( ${OK_PARA_OMP} ) ; then
1878          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1879          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1880        fi
1881        if  ( ${OK_PARA_MPI} ) ; then
1882          # Default : mpirun used if nb_proc gt 1
1883          # pour sortie out/err par process
1884          # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}"  >> script_${ExeNameOut}.ksh
1885          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1886          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1887          EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1888        else
1889          # Default : mpirun is NOT used if nb_proc eq 1
1890          # pour sortie out/err par process
1891          # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1892          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1893          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1894          EXECUTION="time ./script_${ExeNameOut}.ksh"
1895        fi
1896      fi
1897    done
1898
1899  fi
1900
1901  IGCM_debug_Print 1 "sys $( hostname ) : execution command is"
1902  IGCM_debug_Print 1 "$EXECUTION"
1903
1904  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1905}
1906
1907#D-#==================================================
1908#D-function IGCM_sys_check_path
1909#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1910#D-* do not point to an important use directory. Stop immediately in that case.
1911#D-* Examples:
1912#D-
1913function IGCM_sys_check_path {
1914  IGCM_debug_PushStack "IGCM_sys_check_path"
1915  if ( $DEBUG_sys ) ; then
1916    echo "IGCM_sys_check_path"
1917  fi
1918
1919  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1920    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1921    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1922    IGCM_debug_Exit "This will stop the job"
1923  fi
1924  IGCM_debug_PopStack "IGCM_sys_check_path"
1925}
1926
1927#D-#==================================================
1928#D-function IGCM_sys_check_quota. Dummy call here
1929#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1930#D-* Examples:
1931#D-
1932function IGCM_sys_check_quota {
1933  IGCM_debug_PushStack "IGCM_sys_check_quota"
1934  if ( $DEBUG_sys ) ; then
1935    echo "IGCM_sys_check_quota"
1936  fi
1937  IGCM_debug_PopStack "IGCM_sys_check_quota"
1938}
1939
1940#D-#==================================================
1941#D-function IGCM_sys_CountJobInQueue
1942#D-* Purpose: Check if job_name is currently
1943#D-  running or in queue
1944#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1945#D-
1946function IGCM_sys_CountJobInQueue {
1947  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1948  if ( $DEBUG_sys ) ; then
1949    echo "IGCM_sys_CountJobInQueue"
1950  fi
1951  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1952}
1953
1954##############################################################
1955# NCO OPERATOR
1956
1957#D-#==================================================
1958#D-function IGCM_sys_ncap2
1959#D-* Purpose: encapsulate ncap2 call so as to manage error code and retry
1960#D-* Examples:
1961#D-
1962function IGCM_sys_ncap2 {
1963  IGCM_debug_PushStack "IGCM_sys_ncap2" $@
1964  if ( $DEBUG_sys ) ; then
1965    echo "IGCM_sys_ncap2 :" $@
1966  fi
1967
1968  typeset NB_ESSAI DELAI status i
1969  # number of tentative
1970  NB_ESSAI=3
1971  # time delay between tentative
1972  DELAI=2
1973
1974  i=0
1975  while [ $i -lt $NB_ESSAI ] ; do
1976    ncap2 -C "$@" > /tmp/out_command.$$ 2>&1
1977    status=$?
1978    if [ ${status} -gt 0 ] ; then
1979      IGCM_debug_Print 2 "IGCM_sys_ncap2 : error code ${status}"
1980      cat /tmp/out_command.$$
1981      \rm /tmp/out_command.$$
1982      IGCM_debug_Print 2 "IGCM_sys_ncap2 : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1983      sleep $DELAI
1984    else
1985      \rm /tmp/out_command.$$
1986      break
1987    fi
1988    (( i = i + 1 ))
1989  done
1990
1991  if [ ${status} -gt 0 ] ; then
1992    echo "IGCM_sys_ncap2 : ncap2 error"
1993    IGCM_debug_Exit "ncap2"
1994  fi
1995
1996  IGCM_debug_PopStack "IGCM_sys_ncap2"
1997}
1998
1999#D-#==================================================
2000#D-function IGCM_sys_ncatted
2001#D-* Purpose: encapsulate ncatted call so as to manage error code and retry
2002#D-* Examples:
2003#D-
2004function IGCM_sys_ncatted {
2005  IGCM_debug_PushStack "IGCM_sys_ncatted" $@
2006  if ( $DEBUG_sys ) ; then
2007    echo "IGCM_sys_ncatted :" $@
2008  fi
2009
2010  typeset NB_ESSAI DELAI status i
2011  # number of tentative
2012  NB_ESSAI=3
2013  # time delay between tentative
2014  DELAI=2
2015
2016  i=0
2017  while [ $i -lt $NB_ESSAI ] ; do
2018    ncatted "$@" > /tmp/out_command.$$ 2>&1
2019    status=$?
2020    if [ ${status} -gt 0 ] ; then
2021      IGCM_debug_Print 2 "IGCM_sys_ncatted : error code ${status}"
2022      cat /tmp/out_command.$$
2023      \rm /tmp/out_command.$$
2024      IGCM_debug_Print 2 "IGCM_sys_ncatted : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2025      sleep $DELAI
2026    else
2027      \rm /tmp/out_command.$$
2028      break
2029    fi
2030    (( i = i + 1 ))
2031  done
2032
2033  if [ ${status} -gt 0 ] ; then
2034    echo "IGCM_sys_ncatted : ncatted error"
2035    IGCM_debug_Exit "ncatted"
2036  fi
2037
2038  IGCM_debug_PopStack "IGCM_sys_ncatted"
2039}
2040
2041#D-#==================================================
2042#D-function IGCM_sys_ncbo
2043#D-* Purpose: encapsulate ncbo call so as to manage error code and retry
2044#D-* Examples:
2045#D-
2046function IGCM_sys_ncbo {
2047  IGCM_debug_PushStack "IGCM_sys_ncbo" $@
2048  if ( $DEBUG_sys ) ; then
2049    echo "IGCM_sys_ncbo :" $@
2050  fi
2051
2052  typeset NB_ESSAI DELAI status i
2053  # number of tentative
2054  NB_ESSAI=3
2055  # time delay between tentative
2056  DELAI=2
2057
2058  i=0
2059  while [ $i -lt $NB_ESSAI ] ; do
2060    ncbo -C $@ > /tmp/out_command.$$ 2>&1
2061    status=$?
2062    if [ ${status} -gt 0 ] ; then
2063      IGCM_debug_Print 2 "IGCM_sys_ncbo : error code ${status}"
2064      cat /tmp/out_command.$$
2065      \rm /tmp/out_command.$$
2066      IGCM_debug_Print 2 "IGCM_sys_ncbo : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2067      sleep $DELAI
2068    else
2069      \rm /tmp/out_command.$$
2070      break
2071    fi
2072    (( i = i + 1 ))
2073  done
2074
2075  if [ ${status} -gt 0 ] ; then
2076    echo "IGCM_sys_ncbo : ncbo error"
2077    IGCM_debug_Exit "ncbo"
2078  fi
2079
2080  IGCM_debug_PopStack "IGCM_sys_ncbo"
2081}
2082
2083#D-#==================================================
2084#D-function IGCM_sys_ncdif
2085#D-* Purpose: encapsulate ncdiff call so as to manage error code and retry
2086#D-* Examples:
2087#D-
2088function IGCM_sys_ncdiff {
2089  IGCM_debug_PushStack "IGCM_sys_ncdiff" $@
2090  if ( $DEBUG_sys ) ; then
2091    echo "IGCM_sys_ncdiff :" $@
2092  fi
2093
2094  typeset NB_ESSAI DELAI status i
2095  # number of tentative
2096  NB_ESSAI=3
2097  # time delay between tentative
2098  DELAI=2
2099
2100  i=0
2101  while [ $i -lt $NB_ESSAI ] ; do
2102    ncdiff -C $@ > /tmp/out_command.$$ 2>&1
2103    status=$?
2104    if [ ${status} -gt 0 ] ; then
2105      IGCM_debug_Print 2 "IGCM_sys_ncdiff : error code ${status}"
2106      cat /tmp/out_command.$$
2107      \rm /tmp/out_command.$$
2108      IGCM_debug_Print 2 "IGCM_sys_ncdiff : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2109      sleep $DELAI
2110    else
2111      \rm /tmp/out_command.$$
2112      break
2113    fi
2114    (( i = i + 1 ))
2115  done
2116
2117  if [ ${status} -gt 0 ] ; then
2118    echo "IGCM_sys_ncdiff : ncdiff error"
2119    IGCM_debug_Exit "ncdiff"
2120  fi
2121
2122  IGCM_debug_PopStack "IGCM_sys_ncdiff"
2123}
2124
2125#D-#==================================================
2126#D-function IGCM_sys_ncea
2127#D-* Purpose: encapsulate ncea call so as to manage error code and retry
2128#D-* Examples:
2129#D-
2130function IGCM_sys_ncea {
2131  IGCM_debug_PushStack "IGCM_sys_ncea" $@
2132  if ( $DEBUG_sys ) ; then
2133    echo "IGCM_sys_ncea :" $@
2134  fi
2135
2136  typeset NB_ESSAI DELAI status i
2137  # number of tentative
2138  NB_ESSAI=3
2139  # time delay between tentative
2140  DELAI=2
2141
2142  i=0
2143  while [ $i -lt $NB_ESSAI ] ; do
2144    ncea -C $@ > /tmp/out_command.$$ 2>&1
2145    status=$?
2146    if [ ${status} -gt 0 ] ; then
2147      IGCM_debug_Print 2 "IGCM_sys_ncea : error code ${status}"
2148      cat /tmp/out_command.$$
2149      \rm /tmp/out_command.$$
2150      IGCM_debug_Print 2 "IGCM_sys_ncea : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2151      sleep $DELAI
2152    else
2153      \rm /tmp/out_command.$$
2154      break
2155    fi
2156    (( i = i + 1 ))
2157  done
2158
2159  if [ ${status} -gt 0 ] ; then
2160    echo "IGCM_sys_ncea : ncea error"
2161    IGCM_debug_Exit "ncea"
2162  fi
2163
2164  IGCM_debug_PopStack "IGCM_sys_ncea"
2165}
2166
2167#D-#==================================================
2168#D-function IGCM_sys_ncecat
2169#D-* Purpose: encapsulate ncecat call so as to manage error code and retry
2170#D-* Examples:
2171#D-
2172function IGCM_sys_ncecat {
2173  IGCM_debug_PushStack "IGCM_sys_ncecat" $@
2174  if ( $DEBUG_sys ) ; then
2175    echo "IGCM_sys_ncecat :" $@
2176  fi
2177
2178  typeset NB_ESSAI DELAI status i
2179  # number of tentative
2180  NB_ESSAI=3
2181  # time delay between tentative
2182  DELAI=2
2183
2184  i=0
2185  while [ $i -lt $NB_ESSAI ] ; do
2186    ncecat -C $@ > /tmp/out_command.$$ 2>&1
2187    status=$?
2188    if [ ${status} -gt 0 ] ; then
2189      IGCM_debug_Print 2 "IGCM_sys_ncecat : error code ${status}"
2190      cat /tmp/out_command.$$
2191      \rm /tmp/out_command.$$
2192      IGCM_debug_Print 2 "IGCM_sys_ncecat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2193      sleep $DELAI
2194    else
2195      \rm /tmp/out_command.$$
2196      break
2197    fi
2198    (( i = i + 1 ))
2199  done
2200
2201  if [ ${status} -gt 0 ] ; then
2202    echo "IGCM_sys_ncecat : ncecat error"
2203    IGCM_debug_Exit "ncecat"
2204  fi
2205
2206  IGCM_debug_PopStack "IGCM_sys_ncecat"
2207}
2208
2209#D-#==================================================
2210#D-function IGCM_sys_ncflint
2211#D-* Purpose: encapsulate ncflint call so as to manage error code and retry
2212#D-* Examples:
2213#D-
2214function IGCM_sys_ncflint {
2215  IGCM_debug_PushStack "IGCM_sys_ncflint" $@
2216  if ( $DEBUG_sys ) ; then
2217    echo "IGCM_sys_ncflint :" $@
2218  fi
2219
2220  typeset NB_ESSAI DELAI status i
2221  # number of tentative
2222  NB_ESSAI=3
2223  # time delay between tentative
2224  DELAI=2
2225
2226  i=0
2227  while [ $i -lt $NB_ESSAI ] ; do
2228    ncflint -C $@ > /tmp/out_command.$$ 2>&1
2229    status=$?
2230    if [ ${status} -gt 0 ] ; then
2231      IGCM_debug_Print 2 "IGCM_sys_ncflint : error code ${status}"
2232      cat /tmp/out_command.$$
2233      \rm /tmp/out_command.$$
2234      IGCM_debug_Print 2 "IGCM_sys_ncflint : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2235      sleep $DELAI
2236    else
2237      \rm /tmp/out_command.$$
2238      break
2239    fi
2240    (( i = i + 1 ))
2241  done
2242
2243  if [ ${status} -gt 0 ] ; then
2244    echo "IGCM_sys_ncflint : ncflint error"
2245    IGCM_debug_Exit "ncflint"
2246  fi
2247
2248  IGCM_debug_PopStack "IGCM_sys_ncflint"
2249}
2250
2251#D-#==================================================
2252#D-function IGCM_sys_ncks
2253#D-* Purpose: encapsulate ncks call so as to manage error code and retry
2254#D-* Examples:
2255#D-
2256function IGCM_sys_ncks {
2257  IGCM_debug_PushStack "IGCM_sys_ncks" $@
2258  if ( $DEBUG_sys ) ; then
2259    echo "IGCM_sys_ncks :" $@
2260  fi
2261
2262  typeset NB_ESSAI DELAI status i
2263  # number of tentative
2264  NB_ESSAI=3
2265  # time delay between tentative
2266  DELAI=2
2267
2268  i=0
2269  while [ $i -lt $NB_ESSAI ] ; do
2270    ncks -C $@ > /tmp/out_command.$$ 2>&1
2271    status=$?
2272    if [ ${status} -gt 0 ] ; then
2273      IGCM_debug_Print 2 "IGCM_sys_ncks : error code ${status}"
2274      cat /tmp/out_command.$$
2275      \rm /tmp/out_command.$$
2276      IGCM_debug_Print 2 "IGCM_sys_ncks : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2277      sleep $DELAI
2278    else
2279      \rm /tmp/out_command.$$
2280      break
2281    fi
2282    (( i = i + 1 ))
2283  done
2284
2285  if [ ${status} -gt 0 ] ; then
2286    echo "IGCM_sys_ncks : ncks error"
2287    IGCM_debug_Exit "ncks"
2288  fi
2289
2290  IGCM_debug_PopStack "IGCM_sys_ncks"
2291}
2292
2293#D-#==================================================
2294#D-function IGCM_sys_ncpdq
2295#D-* Purpose: encapsulate ncpdq call so as to manage error code and retry
2296#D-* Examples:
2297#D-
2298function IGCM_sys_ncpdq {
2299  IGCM_debug_PushStack "IGCM_sys_ncpdq" $@
2300  if ( $DEBUG_sys ) ; then
2301    echo "IGCM_sys_ncpdq :" $@
2302  fi
2303
2304  typeset NB_ESSAI DELAI status i
2305  # number of tentative
2306  NB_ESSAI=3
2307  # time delay between tentative
2308  DELAI=2
2309
2310  i=0
2311  while [ $i -lt $NB_ESSAI ] ; do
2312    ncpdq -C $@ > /tmp/out_command.$$ 2>&1
2313    status=$?
2314    if [ ${status} -gt 0 ] ; then
2315      IGCM_debug_Print 2 "IGCM_sys_ncpdq : error code ${status}"
2316      cat /tmp/out_command.$$
2317      \rm /tmp/out_command.$$
2318      IGCM_debug_Print 2 "IGCM_sys_ncpdq : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2319      sleep $DELAI
2320    else
2321      \rm /tmp/out_command.$$
2322      break
2323    fi
2324    (( i = i + 1 ))
2325  done
2326
2327  if [ ${status} -gt 0 ] ; then
2328    echo "IGCM_sys_ncpdq : ncpdq error"
2329    IGCM_debug_Exit "ncpdq"
2330  fi
2331
2332  IGCM_debug_PopStack "IGCM_sys_ncpdq"
2333}
2334
2335#D-#==================================================
2336#D-function IGCM_sys_ncra
2337#D-* Purpose: encapsulate ncra call so as to manage error code and retry
2338#D-* Examples:
2339#D-
2340function IGCM_sys_ncra {
2341  IGCM_debug_PushStack "IGCM_sys_ncra" $@
2342  if ( $DEBUG_sys ) ; then
2343    echo "IGCM_sys_ncra :" $@
2344  fi
2345
2346  typeset NB_ESSAI DELAI status i
2347  # number of tentative
2348  NB_ESSAI=3
2349  # time delay between tentative
2350  DELAI=2
2351
2352  i=0
2353  while [ $i -lt $NB_ESSAI ] ; do
2354    ncra -C $@ > /tmp/out_command.$$ 2>&1
2355    status=$?
2356    if [ ${status} -gt 0 ] ; then
2357      IGCM_debug_Print 2 "IGCM_sys_ncra : error code ${status}"
2358      cat /tmp/out_command.$$
2359      \rm /tmp/out_command.$$
2360      IGCM_debug_Print 2 "IGCM_sys_ncra : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2361      sleep $DELAI
2362    else
2363      \rm /tmp/out_command.$$
2364      break
2365    fi
2366    (( i = i + 1 ))
2367  done
2368
2369  if [ ${status} -gt 0 ] ; then
2370    echo "IGCM_sys_ncra : ncra error"
2371    IGCM_debug_Exit "ncra"
2372  fi
2373
2374  IGCM_debug_PopStack "IGCM_sys_ncra"
2375}
2376
2377#D-#==================================================
2378#D-function IGCM_sys_ncrcat
2379#D-* Purpose: encapsulate ncrcat call so as to manage error code and retry
2380#D-* Examples:
2381#D-
2382function IGCM_sys_ncrcat {
2383  IGCM_debug_PushStack "IGCM_sys_ncrcat" $@
2384  if ( $DEBUG_sys ) ; then
2385    echo "IGCM_sys_ncrcat :" $@
2386  fi
2387
2388  typeset NB_ESSAI DELAI status i lastArg
2389  # number of tentative
2390  NB_ESSAI=3
2391  # time delay between tentative
2392  DELAI=2
2393
2394  i=0
2395  while [ $i -lt $NB_ESSAI ] ; do
2396    ncrcat -C $@ > /tmp/out_command.$$ 2>&1
2397    status=$?
2398    if [ ${status} -gt 0 ] ; then
2399      IGCM_debug_Print 2 "IGCM_sys_ncrcat : error code ${status}"
2400      cat /tmp/out_command.$$
2401      \rm /tmp/out_command.$$
2402      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2403      sleep $DELAI
2404    elif [ ! "X$( grep "WARNING Intra-file non-monotonicity" /tmp/out_command.$$ )" = "X" ] ; then
2405      IGCM_debug_Print 2 "IGCM_sys_ncrcat : WARNING Intra-file non-monotonicity"
2406      cat /tmp/out_command.$$
2407      # remove files having corrupted time axis
2408      eval lastArg=\${$#}
2409      IGCM_debug_Print 2 "IGCM_sys_ncrcat : Delete ${lastArg}"
2410      \rm ${lastArg}
2411      \rm /tmp/out_command.$$
2412      IGCM_debug_Print 2 "IGCM_sys_ncrcat : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2413      sleep $DELAI
2414    else
2415      \rm /tmp/out_command.$$
2416      break
2417    fi
2418    (( i = i + 1 ))
2419  done
2420
2421  if [ ${status} -gt 0 ] ; then
2422    echo "IGCM_sys_ncrcat : ncrcat error"
2423    #IGCM_debug_Exit "ncrcat"
2424  fi
2425
2426  IGCM_debug_PopStack "IGCM_sys_ncrcat"
2427}
2428
2429#D-#==================================================
2430#D-function IGCM_sys_ncrename
2431#D-* Purpose: encapsulate ncrename call so as to manage error code and retry
2432#D-* Examples:
2433#D-
2434function IGCM_sys_ncrename {
2435  IGCM_debug_PushStack "IGCM_sys_ncrename" $@
2436  if ( $DEBUG_sys ) ; then
2437    echo "IGCM_sys_ncrename :" $@
2438  fi
2439
2440  typeset NB_ESSAI DELAI status i
2441  # number of tentative
2442  NB_ESSAI=3
2443  # time delay between tentative
2444  DELAI=2
2445
2446  i=0
2447  while [ $i -lt $NB_ESSAI ] ; do
2448    ncrename $@ > /tmp/out_command.$$ 2>&1
2449    status=$?
2450    if [ ${status} -gt 0 ] ; then
2451      IGCM_debug_Print 2 "IGCM_sys_ncrename : error code ${status}"
2452      cat /tmp/out_command.$$
2453      \rm /tmp/out_command.$$
2454      IGCM_debug_Print 2 "IGCM_sys_ncrename : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2455      sleep $DELAI
2456    else
2457      \rm /tmp/out_command.$$
2458      break
2459    fi
2460    (( i = i + 1 ))
2461  done
2462
2463  if [ ${status} -gt 0 ] ; then
2464    echo "IGCM_sys_ncrename : ncrename error"
2465    IGCM_debug_Exit "ncrename"
2466  fi
2467
2468  IGCM_debug_PopStack "IGCM_sys_ncrename"
2469}
2470
2471#D-#==================================================
2472#D-function IGCM_sys_ncwa
2473#D-* Purpose: encapsulate ncwa call so as to manage error code and retry
2474#D-* Examples:
2475#D-
2476function IGCM_sys_ncwa {
2477  IGCM_debug_PushStack "IGCM_sys_ncwa" $@
2478  if ( $DEBUG_sys ) ; then
2479    echo "IGCM_sys_ncwa :" $@
2480  fi
2481
2482  typeset NB_ESSAI DELAI status i
2483  # number of tentative
2484  NB_ESSAI=3
2485  # time delay between tentative
2486  DELAI=2
2487
2488  i=0
2489  while [ $i -lt $NB_ESSAI ] ; do
2490    ncwa -C $@ > /tmp/out_command.$$ 2>&1
2491    status=$?
2492    if [ ${status} -gt 0 ] ; then
2493      IGCM_debug_Print 2 "IGCM_sys_ncwa : error code ${status}"
2494      cat /tmp/out_command.$$
2495      \rm /tmp/out_command.$$
2496      IGCM_debug_Print 2 "IGCM_sys_ncwa : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
2497      sleep $DELAI
2498    else
2499      \rm /tmp/out_command.$$
2500      break
2501    fi
2502    (( i = i + 1 ))
2503  done
2504
2505  if [ ${status} -gt 0 ] ; then
2506    echo "IGCM_sys_ncwa : ncwa error"
2507    IGCM_debug_Exit "ncwa"
2508  fi
2509
2510  IGCM_debug_PopStack "IGCM_sys_ncwa"
2511}
2512
2513##############################################################
2514# CDO OPERATOR
2515
2516#D-#==================================================
2517#D-function IGCM_sys_cdo
2518#D-* Purpose: encapsulate cdo call so as to manage error code and retry
2519#D-* Examples:
2520#D-
2521function IGCM_sys_cdo {
2522  IGCM_debug_PushStack "IGCM_sys_cdo" $@
2523  if ( $DEBUG_sys ) ; then
2524    echo "IGCM_sys_cdo :" $@
2525  fi
2526
2527  typeset status
2528
2529  \cdo $@ > /tmp/out_command.$$ 2>&1
2530  status=$?
2531  if [ ${status} -gt 0 ] ; then
2532    echo "IGCM_sys_cdo : error code ${status}"
2533    cat /tmp/out_command.$$
2534    \rm /tmp/out_command.$$
2535    IGCM_debug_PopStack "IGCM_sys_cdo"
2536    return 1
2537  else
2538    IGCM_debug_PopStack "IGCM_sys_cdo"
2539    return 0
2540  fi
2541
2542  IGCM_debug_PopStack "IGCM_sys_cdo"
2543}
Note: See TracBrowser for help on using the repository browser.