source: tags/libIGCM_v2.0/libIGCM_sys/libIGCM_sys_mercurex9.ksh @ 1170

Last change on this file since 1170 was 804, checked in by sdipsl, 11 years ago
  • Files waiting for rebuild are stored within the relevant simulation tree. see #87

--> /IGCM_OUT/TagName/SpaceName/ExperiementName/JobName/REBUILD

  • Using only 1 proc and having RebuildFrequency? != NONE won't be an issue. see #88
  • Homogenize RUN_DIR PATH during computing and post-processing. see #92
  • Save text files before restarts and outputs. (more chance to have a clue of what went wrong) see #103
  • Remove useless variable BIG_DIR
  • Cosmectics
  • Property svn:keywords set to Revision Author Date
File size: 58.1 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Mercure SX9
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34# gawk specific location on SX9
35export PATH=/applications/gawk-3.0.4/bin:${PATH}
36
37#====================================================
38# set DEBUG_sys to true to output calls of function
39typeset -r DEBUG_sys=${DEBUG_sys:=true}
40
41#====================================================
42# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
43typeset -r DRYRUN=${DRYRUN:=0}
44
45# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
46# -------------------------------------------------------------------------------------
47# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
48# |          |  Cp/Exe param files |            |  Chmod  |                           |
49# |          |      Qsub           |            |         |                           |
50# -------------------------------------------------------------------------------------
51# |    0     |       yes           |    yes     |  yes    |      yes                  |
52# -------------------------------------------------------------------------------------
53# |    1     |       yes           |    yes     |  yes    |      no                   |
54# -------------------------------------------------------------------------------------
55# |    2     |       yes           |    yes     |  no     |      no                   |
56# -------------------------------------------------------------------------------------
57# |    3     |       yes           |    no      |  no     |      no                   |
58# -------------------------------------------------------------------------------------
59
60#=====================================================
61# Global Variables :
62#=====================================================
63# Language : "fr" or "en"
64typeset -r MYLANG="fr"
65
66#=====================================================
67# Host and user names
68# $hostname ou hostname
69typeset  HOST=${HOST:=$( hostname )}
70# $username ou whoami
71typeset  LOGIN=${LOGIN:=$( whoami )}
72# $hostname of the MASTER job
73typeset -r MASTER=mercure
74
75#D-
76#D-#==================================================
77#D-Program used in libIGCM
78#D-#==================================================
79
80# Submit command
81typeset SUBMIT=${SUBMIT:=qsub}
82# rsync with path
83typeset -r RSYNC=/home/cont003/p86denv/SX_RSYNC/bin/rsync
84# RSYNC_opt args to rsync
85typeset -r RSYNC_opt="-va"
86# ie storage filesystem
87typeset -r STOREHOST=${MASTER}
88
89#====================================================
90# Host specific DIRECTORIES
91#====================================================
92
93# ============ CESIUM START ============ #
94
95#====================================================
96#- Mirror libIGCM from mercure to cesium if needed
97#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
98#if [ ! ${ROOTSYS} = "home" ] ; then
99#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
100#else
101#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
102#fi
103
104#====================================================
105#- libIGCM_POST
106#if ( ${MirrorlibIGCM} ) ; then
107#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
108#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
109#else
110#  typeset -r libIGCM_POST=${libIGCM}
111#fi
112
113# ============ CESIUM  END  ============ #
114
115#====================================================
116#- MirrorlibIGCM for frontend
117typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
118
119#====================================================
120#- libIGCM_POST for frontend
121typeset -r libIGCM_POST=${libIGCM}
122
123#====================================================
124#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
125typeset -r R_EXE="${MODIPSL}/bin"
126
127#====================================================
128#- SUBMIT_DIR : submission dir
129typeset SUBMIT_DIR=${SUBMIT_DIR:=${PBS_O_WORKDIR}}
130
131#====================================================
132#- IN
133typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
134typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
135
136#====================================================
137#- ARCHIVE (dedicated to large files)
138typeset ARCHIVE=${CCCSTOREDIR}
139
140#====================================================
141#- STORAGE (dedicated to small/medium files)
142typeset STORAGE=${CCCWORKDIR}
143
144#====================================================
145#- R_OUT
146typeset R_OUT=${ARCHIVE}/IGCM_OUT
147
148#====================================================
149#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
150typeset R_FIG=${STORAGE}/IGCM_OUT
151
152#====================================================
153#- R_BUF (ONLY FOR double copy an scratch)
154typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
155
156#====================================================
157#- RUN_DIR_PATH : Temporary working directory (=> TMP)
158typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${LOCALTMPDIR}}
159
160#====================================================
161#- HOST_MPIRUN_COMMAND
162typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="mpirun"}
163
164#====================================================
165#- Max number of arguments passed to nco operator or demigration command
166UNIX_MAX_LIMIT=360
167
168#====================================================
169#- set PackDefault true on NEC SX9
170PackDefault=true
171
172#====================================================
173#- Default number of MPI task for IPSL coupled model
174#- required for backward compatibility
175#-
176DEFAULT_NUM_PROC_OCE=1
177DEFAULT_NUM_PROC_CPL=1
178(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - 1 ))
179DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
180
181#D-#==================================================
182#D-function IGCM_sys_ChangeArchive
183#D-* Purpose: if SpaceName=TEST everything is stored on SCRATCHDIR
184#D-* Examples:
185#D-
186function IGCM_sys_ChangeArchive {
187  IGCM_debug_PushStack "IGCM_sys_ChangeArchive"
188
189  R_OUT=${SCRATCHDIR}/IGCM_OUT
190  R_FIG=${SCRATCHDIR}/IGCM_OUT
191
192  IGCM_debug_Print 1 "R_OUT   has been redefined = ${R_OUT}"
193  IGCM_debug_Print 1 "R_FIG   has been redefined = ${R_FIG}"
194
195  IGCM_debug_PopStack "IGCM_sys_ChangeArchive"
196}
197
198#D-#==================================================
199#D-function IGCM_sys_RshMaster
200#D-* Purpose: Connection to frontend machine.
201#D-* Examples:
202#D-
203function IGCM_sys_RshMaster {
204  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
205  ssh -t ${MASTER} /bin/ksh <<-EOF
206  export libIGCM=${libIGCM}
207  export DEBUG_debug=${DEBUG_debug}
208  . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
209  . ${libIGCM}/libIGCM_card/libIGCM_card.ksh
210  ${@}
211EOF
212  if [ $? -gt 0 ] ; then
213    echo "IGCM_sys_RshMaster : erreur."
214    IGCM_debug_Exit "IGCM_sys_RshMaster"
215  fi
216  IGCM_debug_PopStack "IGCM_sys_RshMaster"
217}
218
219#D-#==================================================
220#D-function IGCM_sys_RshArchive
221#D-* Purpose: Archive rsh command
222#D-* Examples:
223#D-
224function IGCM_sys_RshArchive {
225  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
226  /bin/ksh <<-EOF
227    ${@}
228EOF
229  if [ $? -gt 0 ] ; then
230    echo "IGCM_sys_RshArchive : erreur."
231    IGCM_debug_Exit "IGCM_sys_RshArchive"
232  fi
233  IGCM_debug_PopStack "IGCM_sys_RshArchive"
234}
235
236#D-#==================================================
237#D-function IGCM_sys_RshPost
238#D-* Purpose: Post-process rsh command
239#D-* Examples:
240#D-
241function IGCM_sys_RshPost {
242  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
243  if ( $DEBUG_sys ) ; then
244    echo "IGCM_sys_RshPost :" $@
245  fi
246
247  #echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
248  #cat tmp_IGCM_sys_RshPost_$$
249  # keep standard input (stdin) for the loop onto temporary file
250  cat >tmp_IGCM_sys_RshPost_$$
251
252# ============ FRONTEND START ============ #
253
254  /bin/ksh <tmp_IGCM_sys_RshPost_$$
255  if [ $? -gt 0 ] ; then
256    echo "IGCM_sys_RshPost : erreur."
257    IGCM_debug_Exit "IGCM_sys_RshPost"
258  fi
259  # delete temporary file
260  \rm tmp_IGCM_sys_RshPost_$$
261
262# ============ FRONTEND  END  ============ #
263
264# ============ CESIUM START ============ #
265#  typeset NB_ESSAI DELAI status i
266#  # number of tentative
267#  NB_ESSAI=10
268#  # time delay between tentative
269#  DELAI=10
270#  (( i = 0 ))
271#  while [ $i -lt $NB_ESSAI ] ; do
272#    ssh -t mercure01 ssh cesium /bin/ksh <tmp_IGCM_sys_RshPost_$$
273#    status=$?
274#    if [ ${status} -gt 0 ]; then
275#      IGCM_debug_Print 2 "IGCM_sys_RshPost : ssh failed ${i}/${NB_ESSAI}"
276#      IGCM_debug_Print 2 "IGCM_sys_RshPost : sleep ${DELAI} seconds and try again."
277#      sleep $DELAI
278#    else
279#      break
280#    fi
281#    (( i = i + 1 ))
282#  done
283#  # delete temporary file
284#  /bin/rm tmp_IGCM_sys_RshPost_$$
285# ============ CESIUM  END  ============ #
286
287  IGCM_debug_PopStack "IGCM_sys_RshPost"
288}
289
290#D-#==================================================
291#D-function IGCM_sys_SendMail
292#D-* Purpose: Send mail when simulation is over
293#D-* Examples:
294#D-
295function IGCM_sys_SendMail {
296  IGCM_debug_PushStack "IGCM_sys_SendMail" $@
297  if ( $DEBUG_sys ) ; then
298    echo "IGCM_sys_SendMail :" $@
299  fi
300
301  if ( ${ExitFlag} ) ; then
302    status=failed
303  else
304    status=completed
305  fi
306
307  cat  << END_MAIL > job_end.mail
308Dear ${LOGIN},
309
310  Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
311  Job started : ${DateBegin}
312  Job ended   : ${DateEnd}
313  Output files are available in ${R_SAVE}
314  Files to be rebuild are temporarily available in ${REBUILD_DIR}
315  Pre-packed files are temporarily available in ${R_BUFR}
316  Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
317END_MAIL
318
319  if  [ X"${config_UserChoices_MailName}" != X ] ; then
320    mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} <  job_end.mail
321  elif [ -f ~/.forward ] ; then
322    mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
323  else
324    mailx -s "${config_UserChoices_JobName} ${status}" ${LOGIN} < job_end.mail
325  fi
326
327  if [ $? -gt 0 ] ; then
328    echo "IGCM_sys_SendMail : erreur."
329    IGCM_debug_Exit "IGCM_sys_SendMail"
330  fi
331  IGCM_debug_PopStack "IGCM_sys_SendMail"
332}
333
334#D-#==================================================
335#D-function IGCM_sys_Mkdir
336#D-* Purpose: Master locale mkdir command
337#D-* Examples:
338#D-
339function IGCM_sys_Mkdir {
340  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
341  if ( $DEBUG_sys ) ; then
342    echo "IGCM_sys_Mkdir :" $@
343  fi
344
345  typeset NB_ESSAI DELAI status i
346  # number of tentative
347  NB_ESSAI=3
348  # time delay between tentative
349  DELAI=2
350
351  if [ ! -d ${1} ]; then
352    i=0
353    while [ $i -lt $NB_ESSAI ] ; do
354      \mkdir -p $1
355      status=$?
356      if [ ${status} -gt 0 ] ; then
357        IGCM_debug_Print 2 "IGCM_sys_Mkdir : mkdir failed error code ${status} ${i}/${NB_ESSAI}"
358        IGCM_debug_Print 2 "IGCM_sys_Mkdir : sleep ${DELAI} seconds and try again."
359        sleep $DELAI
360      else
361        break
362      fi
363      (( i = i + 1 ))
364    done
365  fi
366  # vérification :
367  if [ ! -d ${1} ] ; then
368    echo "IGCM_sys_Mkdir : erreur."
369    IGCM_debug_Exit "IGCM_sys_Mkdir"
370  fi
371  IGCM_debug_PopStack "IGCM_sys_Mkdir"
372}
373
374#D-#==================================================
375#D-function IGCM_sys_MkdirArchive
376#D-* Purpose: Mkdir on Archive
377#D-* Examples:
378#D-
379function IGCM_sys_MkdirArchive {
380  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
381  if ( $DEBUG_sys ) ; then
382    echo "IGCM_sys_MkdirArchive :" $@
383  fi
384
385  typeset NB_ESSAI DELAI status i
386  # number of tentative
387  NB_ESSAI=3
388  # time delay between tentative
389  DELAI=2
390
391  if [ ! -d ${1} ]; then
392    i=0
393    while [ $i -lt $NB_ESSAI ] ; do
394      \mkdir -p $1
395      status=$?
396      if [ ${status} -gt 0 ] ; then
397        IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status} ${i}/${NB_ESSAI}"
398        IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : sleep ${DELAI} seconds and try again."
399        sleep $DELAI
400      else
401        break
402      fi
403      (( i = i + 1 ))
404    done
405  fi
406
407  # vérification :
408  if [ ! -d ${1} ] ; then
409    echo "IGCM_sys_MkdirArchive : erreur."
410    IGCM_debug_Exit "IGCM_sys_MkdirArchive"
411  fi
412  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
413}
414
415#D-#==================================================
416#D-function IGCM_sys_MkdirWork
417#D-* Purpose: Mkdir on Work
418#D-* Examples:
419#D-
420function IGCM_sys_MkdirWork {
421  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
422  if ( $DEBUG_sys ) ; then
423    echo "IGCM_sys_MkdirWork :" $@
424  fi
425  #- creation de repertoire sur le serveur fichier
426  if [ ! -d ${1} ]; then
427    \mkdir -p $1
428    if [ $? -gt 0 ] ; then
429      echo "IGCM_sys_MkdirWork : erreur."
430      IGCM_debug_Exit "IGCM_sys_MkdirWork"
431    fi
432  fi
433  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
434}
435
436#D-#==================================================
437#D-function IGCM_sys_Cd
438#D-* Purpose: master cd command
439#D-* Examples:
440#D-
441function IGCM_sys_Cd {
442  IGCM_debug_PushStack "IGCM_sys_Cd" $@
443  if ( $DEBUG_sys ) ; then
444    echo "IGCM_sys_Cd :" $@
445  fi
446  \cd $1
447  if [ $? -gt 0 ] ; then
448    echo "IGCM_sys_Cd : erreur."
449    IGCM_debug_Exit "IGCM_sys_Cd"
450  fi
451  IGCM_debug_PopStack "IGCM_sys_Cd"
452}
453
454#D-#==================================================
455#D-function IGCM_sys_Chmod
456#D-* Purpose: Chmod
457#D-* Examples:
458#D-
459function IGCM_sys_Chmod {
460  IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
461  if ( $DEBUG_sys ) ; then
462    echo "IGCM_sys_Chmod :" $@
463  fi
464  if [ $DRYRUN -le 1 ]; then
465    \chmod $@
466    if [ $? -gt 0 ] ; then
467      echo "IGCM_sys_Chmod : erreur."
468      IGCM_debug_Exit "IGCM_sys_Chmod"
469    fi
470  else
471    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
472  fi
473  IGCM_debug_PopStack "IGCM_sys_Chmod"
474}
475
476#D-#==================================================
477#D-function IGCM_sys_FileSize
478#D-* Purpose: Filesize
479#D-* Examples:
480#D-
481function IGCM_sys_FileSize {
482  IGCM_debug_PushStack "IGCM_sys_FileSize" $@
483
484  typeset sizeF
485  set +A sizeF -- $( ls -la ${1} )
486  if [ $? -gt 0 ] ; then
487    IGCM_debug_Exit "IGCM_sys_FileSize"
488  fi
489  eval ${2}=${sizeF[4]}
490
491  IGCM_debug_PopStack "IGCM_sys_FileSize"
492}
493
494#D-#==================================================
495#D-function IGCM_sys_TestDir
496#D-* Purpose: Test Directory that must exists
497#D-* Examples:
498#D-
499function IGCM_sys_TestDir {
500  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
501  if ( $DEBUG_sys ) ; then
502    echo "IGCM_sys_TestDir :" $@
503  fi
504  typeset ExistFlag
505  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
506  IGCM_debug_PopStack "IGCM_sys_TestDir"
507
508  return ${ExistFlag}
509}
510
511#D-#==================================================
512#D-function IGCM_sys_TestDirArchive
513#D-* Purpose: Test Directory that must exists on Archive
514#D-* Examples:
515#D-
516function IGCM_sys_TestDirArchive {
517  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
518  if ( $DEBUG_sys ) ; then
519    echo "IGCM_sys_TestDirArchive :" $@
520  fi
521  typeset ExistFlag
522  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
523  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
524
525  return ${ExistFlag}
526}
527
528#D-#==================================================
529#D-function IGCM_sys_IsFileArchived
530#D-* Purpose: Test file that must NOT EXISTS on Archive
531#D-* Examples:
532#D-
533function IGCM_sys_IsFileArchived {
534  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
535  if ( $DEBUG_sys ) ; then
536    echo "IGCM_sys_IsFileArchived :" $@
537  fi
538  typeset IsArchivedFlag
539  IsArchivedFlag=$( [ X$( echo $1 | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
540  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
541
542  return ${IsArchivedFlag}
543}
544
545#D-#==================================================
546#D-function IGCM_sys_TestFileArchive
547#D-* Purpose: Test file that must NOT EXISTS on Archive
548#D-* Examples:
549#D-
550function IGCM_sys_TestFileArchive {
551  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
552  typeset ExistFlag
553  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
554  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
555
556  return ${ExistFlag}
557}
558
559#D-#==================================================
560#D-function IGCM_sys_TestFileBuffer
561#D-* Purpose: Test file that must NOT EXISTS on Buffer
562#D-* Examples:
563#D-
564function IGCM_sys_TestFileBuffer {
565  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
566  typeset ExistFlag
567  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
568  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
569
570  return ${ExistFlag}
571}
572
573#D-#==================================================
574#D-function IGCM_sys_CountFileArchive
575#D-* Purpose: Count files on Archive filesystem
576#D-* Examples:
577#D-
578function IGCM_sys_CountFileArchive {
579  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
580  ls ${@} 2>/dev/null | wc -l
581  if [ $? -gt 0 ] ; then
582    echo "IGCM_sys_CountFileArchive : erreur."
583  fi
584  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
585}
586
587#D-#==================================================
588#D-function IGCM_sys_CountFileBuffer
589#D-* Purpose: Count files on Scratch filesystem
590#D-* Examples:
591#D-
592function IGCM_sys_CountFileBuffer {
593  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
594  ls ${@} 2>/dev/null | wc -l
595  if [ $? -gt 0 ] ; then
596    echo "IGCM_sys_CountFileBuffer : erreur."
597  fi
598  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
599}
600
601#D-#==================================================
602#D-function IGCM_sys_Tree
603#D-* Purpose: Tree directories with files on ${ARCHIVE}
604#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
605#D-
606function IGCM_sys_Tree {
607  IGCM_debug_PushStack "IGCM_sys_Tree" $@
608  if ( $DEBUG_sys ) ; then
609    echo "IGCM_sys_Tree :" $@
610  fi
611
612  \ls -lR ${@}
613
614  IGCM_debug_PopStack "IGCM_sys_Tree"
615}
616
617#D-#==================================================
618#D-function IGCM_sys_Tar
619#D-* Purpose: master tar command
620#D-* Examples:
621#D-
622function IGCM_sys_Tar {
623  IGCM_debug_PushStack "IGCM_sys_Tar" $@
624  if ( $DEBUG_sys ) ; then
625    echo "IGCM_sys_Tar :" $@
626  fi
627  IGCM_sys_RshMaster "\tar $@"
628  if [ $? -gt 0 ] ; then
629    echo "IGCM_sys_Tar : erreur."
630    IGCM_debug_Exit "IGCM_sys_Tar"
631  fi
632  IGCM_debug_PopStack "IGCM_sys_Tar"
633}
634
635#D-#==================================================
636#D-function IGCM_sys_UnTar
637#D-* Purpose: master un-tar command
638#D-* Examples:
639#D-
640function IGCM_sys_UnTar {
641  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
642  if ( $DEBUG_sys ) ; then
643    echo "IGCM_sys_UnTar :" $@
644  fi
645  \tar xvf $1
646  if [ $? -gt 0 ] ; then
647    echo "IGCM_sys_UnTar : erreur."
648    IGCM_debug_Exit "IGCM_sys_UnTar"
649  fi
650  IGCM_debug_PopStack "IGCM_sys_UnTar"
651}
652
653#D-#==================================================
654#D-function IGCM_sys_Qsub
655#D-* Purpose: Qsub new job
656#D-* Examples:
657#D-
658function IGCM_sys_Qsub {
659  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
660  if ( $DEBUG_sys ) ; then
661    echo "IGCM_sys_Qsub :" $@
662  fi
663  /usr/bin/nqsII/qsub -o ${Script_Output} -N ${config_UserChoices_JobName}.${CumulPeriod} < $1
664  if [ $? -gt 0 ] ; then
665    echo "IGCM_sys_Qsub : erreur -o ${Script_Output} -N ${config_UserChoices_JobName}.${CumulPeriod} $@."
666    IGCM_debug_Exit "IGCM_sys_Qsub"
667  fi
668  IGCM_debug_PopStack "IGCM_sys_Qsub"
669}
670
671#D-#==================================================
672#D-function IGCM_sys_QsubPost
673#D-* Purpose: Qsub new job on scalaire
674#D-* Examples:
675#D-
676function IGCM_sys_QsubPost {
677  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
678  if ( $DEBUG_sys ) ; then
679    echo "IGCM_sys_QsubPost :" $@
680  fi
681
682# ============ FRONTEND START ============ #
683
684  /usr/bin/nqsII/qsub -q scalaire -o ${POST_DIR}/${Script_Post_Output}.out ${libIGCM}/$1.job -v ${listVarEnv}
685
686# ============ FRONTEND  END  ============ #
687
688# ============ CESIUM START ============ #
689#  typeset NB_ESSAI DELAI status i
690#  # number of tentative
691#  NB_ESSAI=10
692#  # time delay between tentative
693#  DELAI=10
694#  (( i = 0 ))
695#  while [ $i -lt $NB_ESSAI ] ; do
696#    /usr/local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}" ${libIGCM_POST}/$1.job
697#    status=$?
698#    if [ ${status} -gt 0 ]; then
699#      sleep $DELAI
700#    else
701#      break
702#    fi
703#    (( i = i + 1 ))
704#  done
705# ============ CESIUM  END  ============ #
706
707  if [ $? -gt 0 ] ; then
708    echo "IGCM_sys_QsubPost : erreur " $@
709    IGCM_debug_Exit "IGCM_sys_QsubPost"
710  fi
711  IGCM_debug_PopStack "IGCM_sys_QsubPost"
712}
713
714#D-*************************
715#D- File transfer functions
716#D-*************************
717#D-
718
719#D-#==================================================
720#D-function IGCM_sys_Rsync_out
721#D-* Purpose: treat return val of rsync
722#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
723#D-  Error values and explanations can depend on your system version.
724function IGCM_sys_Rsync_out {
725  status=$1
726  if [ ! $status ] ; then
727    echo "rsync error !"
728  fi
729
730  if [ $MYLANG = "fr" ]; then
731    case $status in
732    0)  return ;;
733    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
734      echo "Erreur de syntaxe ou d'utilisation."
735      return;;
736    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
737      echo "Incompatibilité de protocole."
738      return;;
739    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
740      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
741      echo "répertoires"
742      return;;
743    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
744      echo "Action demandée non supportée : une tentative de manipulation de"
745      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
746      echo "été faite ; ou une option qui est supportée par le  client  mais"
747      echo "pas par le serveur a été spécifiée."
748      return;;
749    10) echo "Erreur de rsync ; RERR_SOCKETIO"
750      echo "Erreur dans le socket d'entrée sortie"
751      return;;
752    11) echo "Erreur de rsync ; RERR_FILEIO"
753      echo "Erreur d'entrée sortie fichier"
754      return;;
755    12) echo "Erreur de rsync ; RERR_STREAMIO"
756      echo "Erreur dans flux de donnée du protocole rsync"
757      return;;
758    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
759      echo "Erreur avec les diagnostics du programme"
760      return;;
761    14) echo "Erreur de rsync ; RERR_IPC"
762      echo "Erreur dans le code IPC"
763      return;;
764    20) echo "Erreur de rsync ; RERR_SIGNAL"
765      echo "SIGUSR1 ou SIGINT reçu"
766      return;;
767    21) echo "Erreur de rsync ; RERR_WAITCHILD"
768      echo "Une erreur retournée par waitpid()"
769      return;;
770    22) echo "Erreur de rsync ; RERR_MALLOC"
771      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
772      return;;
773    23) echo ""
774      echo "Erreur fichier inexistant"
775      return;;
776    30) echo "Erreur de rsync ; RERR_TIMEOUT"
777      echo "Temps d'attente écoulé dans l'envoi/réception de données"
778      return;;
779    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $status
780      return;;
781    esac
782  elif [ $MYLANG = "en" ] ; then
783    case $status in
784    0)  return;;
785    1)  echo "rsync error : Syntax or usage error "
786      return;;
787    2)  echo "rsync error : Protocol incompatibility "
788      return;;
789    3)  echo "rsync error : Errors selecting input/output files, dirs"
790      return;;
791    4)  echo "rsync error : Requested action not supported: an attempt"
792      echo "was made to manipulate 64-bit files on a platform that cannot support"
793      echo "them; or an option was specified that is supported by the client and"
794      echo "not by the server."
795      return;;
796    5)  echo "rsync error : Error starting client-server protocol"
797      return;;
798    10) echo "rsync error : Error in socket I/O "
799      return;;
800    11) echo "rsync error : Error in file I/O "
801      return;;
802    12) echo "rsync error : Error in rsync protocol data stream "
803      return;;
804    13) echo "rsync error : Errors with program diagnostics "
805      return;;
806    14) echo "rsync error : Error in IPC code "
807      return;;
808    20) echo "rsync error : Received SIGUSR1 or SIGINT "
809      return;;
810    21) echo "rsync error : Some error returned by waitpid() "
811      return;;
812    22) echo "rsync error : Error allocating core memory buffers "
813      return;;
814    23) echo "rsync error : Partial transfer due to error"
815      return;;
816    24) echo "rsync error : Partial transfer due to vanished source files"
817      return;;
818    30) echo "rsync error : Timeout in data send/receive "
819      return;;
820    *)  echo "rsync error : return code of rsync unknown :" $status
821      return;;
822    esac
823  else
824    echo "unknown language $MYLANG."
825    return
826  fi
827}
828
829#D-#==================================================
830#D-function IGCM_sys_Miror_libIGCM
831#D-* Purpose: Mirror libIGCM PATH and lib to cesium
832#D-* Examples:
833#D-
834function IGCM_sys_Mirror_libIGCM {
835  IGCM_debug_PushStack "IGCM_sys_Mirror_libIGCM"
836  if ( $DEBUG_sys ) ; then
837    echo "IGCM_sys_Mirror_libIGCM"
838  fi
839
840  typeset status DEST
841
842  mkdir -p ${HOME}/MIRROR/${PATHlibIGCM}
843
844  echo ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} > out_rsync 2>&1
845  ${RSYNC} ${RSYNC_opt} ${libIGCM} ${HOME}/MIRROR/${PATHlibIGCM} >> out_rsync 2>&1
846  status=$?
847
848  if [ ${status} -gt 0 ] ; then
849    echo "IGCM_sys_Mirror_libIGCM Warning : no libIGCM on cesium."
850    cat out_rsync
851  fi
852  IGCM_debug_PopStack "IGCM_sys_Mirror_libIGCM"
853}
854
855#D-#==================================================
856#D-function IGCM_sys_Cp
857#D-* Purpose: generic cp
858#D-* Examples:
859#D-
860function IGCM_sys_Cp {
861  IGCM_debug_PushStack "IGCM_sys_Cp" $@
862  if ( $DEBUG_sys ) ; then
863    echo "IGCM_sys_Cp :" $@
864  fi
865
866  typeset status
867
868  echo cp $@ > out_rsync 2>&1
869  \cp $@ >> out_rsync 2>&1
870  status=$?
871
872  if [ ${status} -gt 0 ] ; then
873    echo "IGCM_sys_Cp : error code ${status}"
874    cat out_rsync
875    IGCM_debug_Exit "IGCM_sys_Cp"
876  else
877    \rm out_rsync
878  fi
879  IGCM_debug_PopStack "IGCM_sys_Cp"
880}
881
882#D-#==================================================
883#D-function IGCM_sys_Rm
884#D-* Purpose: generic rm
885#D-* Examples:
886#D-
887function IGCM_sys_Rm {
888  IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
889  if ( $DEBUG_sys ) ; then
890    echo "IGCM_sys_Rm :" $@
891  fi
892
893  typeset status
894
895  echo rm $@ > out_rsync 2>&1
896  \rm $@ >> out_rsync 2>&1
897  status=$?
898
899  if [ ${status} -gt 0 ] ; then
900    echo "IGCM_sys_Rm : error code ${status}"
901    cat out_rsync
902    IGCM_debug_Exit "IGCM_sys_Rm"
903  else
904    \rm out_rsync
905  fi
906  IGCM_debug_PopStack "IGCM_sys_Rm"
907}
908
909#D-#==================================================
910#D-function IGCM_sys_Mv
911#D-* Purpose: generic move
912#D-* Examples:
913#D-
914function IGCM_sys_Mv {
915  IGCM_debug_PushStack "IGCM_sys_Mv" $@
916  if ( $DEBUG_sys ) ; then
917    echo "IGCM_sys_Mv :" $@
918  fi
919
920  if [ $DRYRUN = 0 ]; then
921
922    typeset status
923
924    echo mv $@ > out_rsync 2>&1
925    \mv $@ >> out_rsync 2>&1
926    status=$?
927
928    if [ ${status} -gt 0 ] ; then
929      echo "IGCM_sys_Mv : error code ${status}"
930      cat out_rsync
931      IGCM_debug_Exit "IGCM_sys_Mv"
932    else
933      \rm out_rsync
934    fi
935  else
936    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
937  fi
938
939  IGCM_debug_PopStack "IGCM_sys_Mv"
940}
941
942#D-#==================================================
943#D-function IGCM_sys_Put_Dir
944#D-* Purpose: Copy a complete directory on $(ARCHIVE)
945#D-* Examples:
946#D-
947function IGCM_sys_Put_Dir {
948  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
949  if ( $DEBUG_sys ) ; then
950    echo "IGCM_sys_Put_Dir :" $@
951  fi
952  if [ $DRYRUN = 0 ]; then
953    if [ ! -d ${1} ] ; then
954      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
955      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
956      return
957    fi
958
959    typeset status
960
961    # Only if we use rsync
962    #IGCM_sys_TestDirArchive $( dirname $2 )
963    #
964    #USUAL WAY
965    \cp -r $1 $2 > out_rsync 2>&1
966    status=$?
967
968    if [ ${status} -gt 0 ] ; then
969      echo "IGCM_sys_Put_Dir : error code ${status}"
970      cat out_rsync
971      IGCM_debug_Exit "IGCM_sys_Put_Dir"
972    else
973      \rm out_rsync
974    fi
975  else
976    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
977  fi
978  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
979}
980
981#D-#==================================================
982#D-function IGCM_sys_Get_Dir
983#D-* Purpose: Copy a complete directory from ${ARCHIVE}
984#D-* Examples:
985#D-
986function IGCM_sys_Get_Dir {
987  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
988  if ( $DEBUG_sys ) ; then
989    echo "IGCM_sys_Get_Dir :" $@
990  fi
991  if [ $DRYRUN = 0 ]; then
992#    if [ ! -d ${1} ] ; then
993#      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
994#      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
995#      return
996#    fi
997
998    typeset NB_ESSAI DELAI status i
999    # number of tentative
1000    NB_ESSAI=3
1001    # time delay between tentative
1002    DELAI=2
1003
1004    # Only if we use rsync
1005    #IGCM_sys_TestDirArchive $( dirname $2 )
1006    #
1007    # USUAL WAY
1008    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
1009    ccc_hsm get -r $1
1010
1011    i=0
1012    while [ $i -lt $NB_ESSAI ] ; do
1013      \cp -ur $1 $2 >> out_rsync 2>&1
1014      status=$?
1015      if [ ${status} -gt 0 ]; then
1016        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
1017        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
1018        sleep $DELAI
1019      else
1020        break
1021      fi
1022      (( i = i + 1 ))
1023    done
1024
1025    if [ ${status} -gt 0 ] ; then
1026      echo "IGCM_sys_Get_Dir : error."
1027      cat out_rsync
1028      IGCM_debug_Exit "IGCM_sys_Get_Dir"
1029    else
1030      \rm out_rsync
1031    fi
1032  else
1033    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1034  fi
1035  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
1036}
1037
1038#====================================================
1039#- Call IGCM_sys_Mirror_libIGCM now !
1040if ( $MirrorlibIGCM ) ; then
1041  IGCM_sys_Mirror_libIGCM
1042fi
1043
1044#D-#==================================================
1045#D-function IGCM_sys_Put_Rest
1046#D-* Purpose: Put computied restarts on ${ARCHIVE}.
1047#D-           File and target directory must exist.
1048#D-* Examples:
1049#D-
1050function IGCM_sys_Put_Rest {
1051  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
1052  if ( $DEBUG_sys ) ; then
1053    echo "IGCM_sys_Put_Rest :" $@
1054  fi
1055  if [ $DRYRUN = 0 ]; then
1056    if [ ! -f ${1} ] ; then
1057      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
1058      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1059    fi
1060
1061    typeset status
1062    #
1063    if [ X${JobType} = XRUN ] ; then
1064      IGCM_sys_Chmod 444 ${1}
1065    fi
1066
1067    #
1068    # USUAL WAY
1069    \cp $1 $2 > out_rsync 2>&1
1070    status=$?
1071
1072#       #RSYNC WITH NETWORK SSH CALL
1073#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1074#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1075
1076#       #RSYNC WITH NFS USE
1077#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1078#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1079
1080#       status=$?
1081#       IGCM_sys_Rsync_out $status
1082
1083#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1084#       (( status=status+$? ))
1085
1086    if [ ${status} -gt 0 ] ; then
1087      echo "IGCM_sys_Put_Rest : cp failed error code ${status}"
1088      cat out_rsync
1089      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1090    else
1091      \rm out_rsync
1092    fi
1093  else
1094    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1095  fi
1096  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1097}
1098
1099#D-#==================================================
1100#D-function IGCM_sys_PutBuffer_Rest
1101#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
1102#D-           File and target directory must exist.
1103#D-* Examples:
1104#D-
1105function IGCM_sys_PutBuffer_Rest {
1106  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
1107  if ( $DEBUG_sys ) ; then
1108    echo "IGCM_sys_PutBuffer_Rest :" $@
1109  fi
1110  if [ $DRYRUN = 0 ]; then
1111    if [ ! -f ${1} ] ; then
1112      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
1113      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1114    fi
1115
1116    typeset status
1117    #
1118    if [ X${JobType} = XRUN ] ; then
1119      IGCM_sys_Chmod 444 ${1}
1120    fi
1121
1122    #
1123    # USUAL WAY
1124    \cp $1 $2 > out_rsync 2>&1
1125    status=$?
1126
1127    if [ ${status} -gt 0 ] ; then
1128      echo "IGCM_sys_PutBuffer_Rest : error code ${status}"
1129      cat out_rsync
1130      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
1131    else
1132      \rm out_rsync
1133    fi
1134  else
1135    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1136  fi
1137  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
1138}
1139
1140#D-#==================================================
1141#D-function IGCM_sys_PrepareTaredRestart
1142#D-* Purpose: Prepare tared restart to be access by computing job. Identity here.
1143#D-* Examples:
1144#D-
1145function IGCM_sys_PrepareTaredRestart {
1146  IGCM_debug_PushStack "IGCM_sys_PrepareTaredRestart" $@
1147  echo $1
1148  IGCM_debug_PopStack "IGCM_sys_PrepareTaredRestart"
1149}
1150
1151#D-#==================================================
1152#D-function IGCM_sys_Put_Out
1153#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1154#D-* Examples:
1155#D-
1156function IGCM_sys_Put_Out {
1157  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1158  if ( $DEBUG_sys ) ; then
1159    echo "IGCM_sys_Put_Out :" $@
1160  fi
1161
1162  typeset NB_ESSAI DELAI status i exist skip
1163
1164  # number of tentative
1165  NB_ESSAI=3
1166  # time delay between tentative
1167  DELAI=2
1168
1169  if [ $DRYRUN = 0 ]; then
1170    if [ ! -f ${1} ] ; then
1171      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1172      IGCM_debug_PopStack "IGCM_sys_Put_Out"
1173      return 1
1174    fi
1175    #
1176    IGCM_sys_MkdirArchive $( dirname $2 )
1177    #
1178
1179    #=====================================================
1180    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1181    #=====================================================
1182
1183    #echo ${2} | grep "${R_OUT}" > /dev/null 2>&1
1184    #if [ $? -eq 0 ] ; then
1185    #    typeset WORKPATH FILEPATH
1186    #    WORKPATH=$( dirname $2 | sed -e "s|${R_OUT}|${R_BUF}|" )
1187    #    IGCM_sys_MkdirWork ${WORKPATH}
1188    #    FILEPATH=${WORKPATH}/$( basename $2 )
1189    #    #
1190    #    IGCM_sys_Cp ${1} ${FILEPATH}
1191    #fi
1192
1193    if [ X${JobType} = XRUN ] ; then
1194      if [ X${3} = X ] ; then
1195        IGCM_sys_Chmod 444 ${1}
1196      fi
1197    fi
1198    #
1199
1200    exist=false
1201    skip=false
1202    if [ -f $2 ] ; then
1203      IGCM_debug_Print 1 "$2 already exist"
1204      #dmget $2
1205      ccc_hsm get $2
1206      exist=true
1207      if [ "X$( diff $1 $2 )" = X ] ; then
1208        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1209        skip=true
1210      else
1211        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1212        skip=false
1213      fi
1214    fi
1215    #
1216    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1217      IGCM_sys_Chmod u+w $2
1218    fi
1219
1220    if [ X${skip} = Xfalse ] ; then
1221      i=0
1222      while [ $i -lt $NB_ESSAI ] ; do
1223        # USUAL WAY
1224        \cp $1 $2 > out_rsync 2>&1
1225        status=$?
1226        if [ ${status} -gt 0 ]; then
1227          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1228          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
1229          sleep $DELAI
1230        else
1231          break
1232        fi
1233        (( i = i + 1 ))
1234      done
1235    fi
1236
1237#       #RSYNC WITH NETWORK SSH CALL
1238#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1239#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1240
1241#       #RSYNC WITH NFS USE
1242#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1243#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1244
1245#       status=$?
1246#       IGCM_sys_Rsync_out $status
1247
1248#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1249#       (( status=status+$? ))
1250
1251    if [ ${status} -gt 0 ] ; then
1252      echo "IGCM_sys_Put_Out : error."
1253      cat out_rsync
1254      IGCM_debug_Exit "IGCM_sys_Put_Out"
1255    else
1256      \rm out_rsync
1257    fi
1258  else
1259    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1260  fi
1261  IGCM_debug_PopStack "IGCM_sys_Put_Out"
1262  return 0
1263}
1264
1265#D-#==================================================
1266#D-function IGCM_sys_PutBuffer_Out
1267#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1268#D-* Examples:
1269#D-
1270function IGCM_sys_PutBuffer_Out {
1271  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1272  if ( $DEBUG_sys ) ; then
1273    echo "IGCM_sys_PutBuffer_Out :" $@
1274  fi
1275
1276  typeset NB_ESSAI DELAI status i exist skip
1277
1278  # number of tentative
1279  NB_ESSAI=3
1280  # time delay between tentative
1281  DELAI=2
1282
1283  if [ $DRYRUN = 0 ]; then
1284    if [ ! -f ${1} ] ; then
1285      echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1286      IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1287      return 1
1288    fi
1289    #
1290    IGCM_sys_Mkdir $( dirname $2 )
1291    #
1292
1293    if [ X${JobType} = XRUN ] ; then
1294      if [ X${3} = X ] ; then
1295        IGCM_sys_Chmod 444 ${1}
1296      fi
1297    fi
1298
1299    exist=false
1300    skip=false
1301    if [ -f $2 ] ; then
1302      IGCM_debug_Print 1 "$2 already exist"
1303      exist=true
1304      if [ "X$( diff $1 $2 )" = X ] ; then
1305        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1306        skip=true
1307      else
1308        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1309        skip=false
1310      fi
1311    fi
1312    #
1313    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1314      IGCM_sys_Chmod u+w $2
1315    fi
1316
1317    if [ X${skip} = Xfalse ] ; then
1318      i=0
1319      while [ $i -lt $NB_ESSAI ] ; do
1320        # USUAL WAY
1321        \cp $1 $2 > out_rsync 2>&1
1322        status=$?
1323        if [ ${status} -gt 0 ]; then
1324          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
1325          IGCM_debug_Print 2 "IGCM_sys_PutBuffer_Out : sleep ${DELAI} seconds and try again."
1326          sleep $DELAI
1327        else
1328          break
1329        fi
1330        (( i = i + 1 ))
1331      done
1332    fi
1333
1334    if [ ${status} -gt 0 ] ; then
1335      echo "IGCM_sys_PutBuffer_Out : error."
1336      cat out_rsync
1337      IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1338    else
1339      \rm out_rsync
1340    fi
1341  else
1342    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1343  fi
1344  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1345  return 0
1346}
1347
1348#D-#==================================================
1349#D-function IGCM_sys_Get
1350#D-* Purpose: Get a file from ${ARCHIVE}
1351#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1352#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
1353function IGCM_sys_Get {
1354  IGCM_debug_PushStack "IGCM_sys_Get" $@
1355
1356  typeset DEST dm_liste ifile target file_work
1357  typeset NB_ESSAI DELAI status i
1358
1359  if ( $DEBUG_sys ) ; then
1360    echo "IGCM_sys_Get :" $@
1361  fi
1362
1363  # number of tentative
1364  NB_ESSAI=3
1365  # time delay between tentative
1366  DELAI=2
1367
1368  if [ $DRYRUN -le 2 ]; then
1369    if [ X${1} = X'/l' ] ; then
1370      # test if the first file is present in the old computation :
1371      eval set +A dm_liste \${${2}}
1372    else
1373      eval set +A dm_liste ${1}
1374    fi
1375    eval DEST=\${${#}}
1376
1377    #=====================================================
1378    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1379    #=====================================================
1380
1381    # Is it an R_OUT file (not R_IN) ?
1382    #echo ${dm_liste[0]} | grep "${R_OUT}" > /dev/null 2>&1
1383    #if [ $? -eq 0 ] ; then
1384    #    # Yes  ? then we try to get it in SCRATCHDIR
1385    #    set +A file_work $( echo ${dm_liste[*]} | sed -e "s|${R_OUT}|${R_BUF}|g" )
1386    #    if [ -f ${file_work[0]} ] ; then
1387    #   IGCM_sys_Cp ${file_work[*]} ${DEST}
1388    #   IGCM_debug_PopStack "IGCM_sys_Get"
1389    #   return
1390    #    fi
1391    #fi
1392
1393    ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1394    status=$?
1395    if [ ${status} -gt 0 ] ; then
1396      echo "WARNING IGCM_sys_Get : error code ${status}"
1397      cat out_rsync
1398      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1399    fi
1400
1401    #if [ ${status} -gt 0 ] ; then
1402    #    if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
1403    #   cat out_rsync
1404    #   echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1405    #   sleep 30
1406    #   echo "We try another time"
1407    ##  dmget ${dm_liste[*]} > out_rsync 2>&1
1408    #   ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1409    #   status=$?
1410    #   if [ ${status} -gt 0 ] ; then
1411    #       echo "ERROR IGCM_sys_Get : again demigration error :"
1412    #       cat out_rsync
1413    #       IGCM_debug_Exit "IGCM_sys_Get"
1414    #   fi
1415    #    else
1416    #   echo "ERROR IGCM_sys_Get : demigration error :"
1417    #   cat out_rsync
1418    #   IGCM_debug_Exit "IGCM_sys_Get"
1419    #    fi
1420    #fi
1421
1422    #   #RSYNC WITH NETWORK SSH CALL
1423    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1424    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1425
1426    #   #RSYNC WITH NFS USE
1427    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1428    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1429
1430    #   status=$?
1431    #   IGCM_sys_Rsync_out $status
1432
1433    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1434    #   (( status=status+$? ))
1435
1436    #USUAL WAY
1437    if [ X${1} = X'/l' ] ; then
1438      for target in ${dm_liste[*]} ; do
1439        local_file=$( basename ${target} )
1440        # test if the target file is present before the loop
1441        IGCM_sys_TestFileArchive ${target}
1442        status=$?
1443        if [ ${status} -gt 0 ] ; then
1444          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
1445          IGCM_debug_Exit "IGCM_sys_Get"
1446        else
1447          i=0
1448          while [ $i -lt $NB_ESSAI ] ; do
1449            \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1450            status=$?
1451            if [ ${status} -gt 0 ]; then
1452              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1453              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1454              sleep $DELAI
1455            else
1456              break
1457            fi
1458            (( i = i + 1 ))
1459          done
1460          if [ ${status} -gt 0 ] ; then
1461            echo "IGCM_sys_Get : error"
1462            cat out_rsync
1463            \rm out_rsync
1464            IGCM_debug_Exit "IGCM_sys_Get"
1465          else
1466            \rm out_rsync
1467          fi
1468        fi
1469      done
1470    else
1471      i=0
1472      while [ $i -lt $NB_ESSAI ] ; do
1473        \cp ${dm_liste} ${DEST} >> out_rsync 2>&1
1474        status=$?
1475        if [ ${status} -gt 0 ]; then
1476          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
1477          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1478          sleep $DELAI
1479        else
1480          break
1481        fi
1482        (( i = i + 1 ))
1483      done
1484      if [ ${status} -gt 0 ] ; then
1485        echo "IGCM_sys_Get : error"
1486        cat out_rsync
1487        \rm out_rsync
1488        IGCM_debug_Exit "IGCM_sys_Get"
1489      else
1490        \rm out_rsync
1491      fi
1492    fi
1493  else
1494    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1495  fi
1496  IGCM_debug_PopStack "IGCM_sys_Get"
1497}
1498
1499#D-#==================================================
1500#D-function IGCM_sys_GetBuffer
1501#D-* Purpose: Get a file from ${SCRATCHDIR}
1502#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1503#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1504function IGCM_sys_GetBuffer {
1505  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1506
1507  typeset DEST buf_liste ifile target file_work
1508  typeset NB_ESSAI DELAI status i
1509
1510  if ( $DEBUG_sys ) ; then
1511    echo "IGCM_sys_GetBuffer :" $@
1512  fi
1513
1514  # number of tentative
1515  NB_ESSAI=3
1516  # time delay between tentative
1517  DELAI=2
1518
1519  if [ $DRYRUN -le 2 ]; then
1520    if [ X${1} = X'/l' ] ; then
1521      # test if the first file is present in the old computation :
1522      eval set +A buf_liste \${${2}}
1523    else
1524      eval set +A buf_liste ${1}
1525    fi
1526    eval DEST=\${${#}}
1527
1528    #USUAL WAY
1529    if [ X${1} = X'/l' ] ; then
1530      for target in ${buf_liste[*]} ; do
1531        local_file=$( basename ${target} )
1532        i=0
1533        while [ $i -lt $NB_ESSAI ] ; do
1534          \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1535          status=$?
1536          if [ ${status} -gt 0 ]; then
1537            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1538            IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1539            sleep $DELAI
1540          else
1541            break
1542          fi
1543          (( i = i + 1 ))
1544        done
1545        if [ ${status} -gt 0 ] ; then
1546          echo "IGCM_sys_Get : error"
1547          cat out_rsync
1548          \rm out_rsync
1549          IGCM_debug_Exit "IGCM_sys_GetBuffer"
1550        else
1551          \rm out_rsync
1552        fi
1553      done
1554    else
1555      i=0
1556      while [ $i -lt $NB_ESSAI ] ; do
1557        \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1558        status=$?
1559        if [ ${status} -gt 0 ]; then
1560          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : cp failed error code ${status} ${i}/${NB_ESSAI}"
1561          IGCM_debug_Print 2 "IGCM_sys_GetBuffer : sleep ${DELAI} seconds and try again."
1562          sleep $DELAI
1563        else
1564          break
1565        fi
1566        (( i = i + 1 ))
1567      done
1568      if [ ${status} -gt 0 ] ; then
1569        echo "IGCM_sys_Get : error"
1570        cat out_rsync
1571        \rm out_rsync
1572        IGCM_debug_Exit "IGCM_sys_GetBuffer"
1573      else
1574        \rm out_rsync
1575      fi
1576    fi
1577  else
1578    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1579  fi
1580  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1581}
1582
1583#D-#==================================================
1584#D-function IGCM_sys_GetDate_FichWork
1585#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1586#D-* Examples:
1587#D-
1588function IGCM_sys_GetDate_FichWork {
1589  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1590  if ( $DEBUG_sys ) ; then
1591    echo "IGCM_sys_GetDate_FichWork :" $@
1592  fi
1593  typeset dateF
1594  set +A dateF -- $( IGCM_sys_RshMaster "ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1}" )
1595  eval ${2}=${dateF[5]}
1596
1597    # donne la date filesys d'un fichier sur la machine work
1598  IGCM_debug_PopStack "IGCM_sys_FichWork"
1599}
1600
1601#D-#==================================================
1602#D-function IGCM_sys_GetDate_FichArchive
1603#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1604#D-* Examples:
1605#D-
1606function IGCM_sys_GetDate_FichArchive {
1607  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1608  if ( $DEBUG_sys ) ; then
1609    echo "IGCM_sys_GetDate_FichArchive :" $@
1610  fi
1611  typeset dateF
1612  set +A dateF -- $( IGCM_sys_RshMaster "ls -l --full-time --time-style='+%Y%m%d%H%M%S' ${1}" )
1613  eval ${2}=${dateF[5]}
1614
1615  IGCM_debug_PopStack "IGCM_sys_FichArchive"
1616}
1617
1618##############################################################
1619# REBUILD OPERATOR
1620
1621function IGCM_sys_rebuild {
1622  IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1623  if ( $DEBUG_sys ) ; then
1624    echo "IGCM_sys_rebuild :" $@
1625  fi
1626
1627  typeset NB_ESSAI DELAI status i firstArg
1628  # number of tentative
1629  NB_ESSAI=3
1630  # time delay between tentative
1631  DELAI=2
1632
1633  i=0
1634  while [ $i -lt $NB_ESSAI ] ; do
1635      /home/cont003/p86ipsl/SX8/bin/rebuild -f -o $@ > out_rsync 2>&1
1636    status=$?
1637    if [ ${status} -gt 0 ] ; then
1638      IGCM_debug_Print 2 "IGCM_sys_rebuild : error code ${status}"
1639      cat out_rsync
1640      \rm out_rsync
1641      IGCM_debug_Print 2 "IGCM_sys_rebuild : ${i}/${NB_ESSAI} sleep ${DELAI} seconds and try again."
1642      firstArg=${1}
1643      \rm ${firstArg}
1644      sleep $DELAI
1645    else
1646      \rm out_rsync
1647      break
1648    fi
1649    (( i = i + 1 ))
1650  done
1651
1652  if [ ${status} -gt 0 ] ; then
1653    echo "IGCM_sys_rebuild : rebuild error"
1654    IGCM_debug_Exit "rebuild"
1655  fi
1656
1657  IGCM_debug_PopStack "IGCM_sys_rebuild"
1658}
1659
1660function IGCM_sys_rebuild_station {
1661  IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
1662  typeset i list_opt file_in file_out prefix_invert list_invert
1663  if ( $DEBUG_sys ) ; then
1664    echo "IGCM_sys_rebuild_station :" $@
1665  fi
1666  list_opt=$@
1667
1668  # Invert Axis : t,x -> x,t
1669  #               t,pres,x -> x,t,pres
1670  # So that we can concatenate along x
1671  i=0
1672  for file_in in ${list_opt} ; do
1673    (( i = i + 1))
1674    [ ${i} = 1 ] && file_out=${file_in} && continue
1675    prefix_invert=$( basename ${file_in} .nc )
1676    IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1677    list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1678  done
1679
1680  # Concatenate
1681  IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1682
1683  # Re-ivert file
1684  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1685
1686  # Station re-ordering is too expansive to be run within libICGM
1687  # This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1688  # This re-ordering must be done "in memory" by the cmorization process
1689  # Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1690  # BEGIN reordering
1691
1692  # Only LMDZ text output contains the exact ordering of the station.
1693  # We isolate this in the code below:
1694  #  0  38  -157.5000000000000  70.98591549295774
1695  #  0  54  27.49999999999999   67.18309859154928
1696  #  0  56  -62.50000000000001  82.39436619718309
1697  #  0  79  12.49999999999999   78.59154929577466
1698  #  0  116 -165.0000000000000  76.05633802816901
1699  #  0  117 130.0000000000000   70.98591549295774
1700  #  0  118 110.0000000000000   87.46478873239437
1701  #  1  40  4.999999999999995   51.97183098591550
1702#  typeset iStation iProc list_opt file_in file_out prefix_invert
1703#  typeset -Z4 j4
1704#  typeset -Z3 j3
1705
1706#  unset list_opt
1707#  set +A list_opt $@
1708
1709  # Filename after rebuild
1710#  file_out=${list_opt[0]}
1711  # Prefix of output files
1712#  prefix_invert=$( basename ${file_out} .nc )
1713  # Number of procs
1714#  num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} | wc -l )
1715
1716#  iProc=0
1717#  while [ ${iProc} -lt ${num_proc} ] ; do
1718    # Array containing Station as a number
1719#    unset proc_stn
1720#    set +A proc_stn $( grep "iophy_mpi rank ip lon lat  $iProc" ${PREFIX}_${Exe_Output} | sed -e "s/iophy_mpi rank ip lon lat //g" | awk ' {print $2}' )
1721    # Number of stations produced by processor proc
1722#    stationLast=${#proc_stn[*]}
1723    # Proc number on 4 digits
1724#    j4=${iProc}
1725    # Init
1726#    iStation=0
1727#    while [ ${iStation} -lt ${stationLast} ] ; do
1728      # Station number on 3 digits
1729#      j3=${proc_stn[${iStation}]}
1730      # Extract station
1731      # Invert Axis : t,x -> x,t
1732      #               t,pres,x -> x,t,pres
1733      # So that we can concatenate along x
1734#      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1735#      (( iStation = iStation + 1 ))
1736#    done
1737#    (( iProc = iProc + 1 ))
1738#  done
1739
1740  # Concatenate all station along x
1741#  IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1742
1743  # Re-invert file
1744#  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1745
1746  # END reordering
1747
1748  IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1749}
1750
1751############################################################
1752# Activate Running Environnment Variables
1753
1754function IGCM_sys_activ_variables {
1755  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1756  if ( $DEBUG_sys ) ; then
1757    echo "IGCM_sys_activ_variables"
1758  fi
1759
1760# --------------------------------------------------------------------
1761#D- MPI specifications
1762# --------------------------------------------------------------------
1763
1764#D-- MPISUSPEND
1765  export MPISUSPEND=${MPISUSPEND:=OFF}
1766
1767#D-- MPIPROGINF #other choices : ALL_DETAIL2
1768  export MPIPROGINF=ALL
1769#D- activate ftrace (with -ftrace)
1770  export F_FTRACE=YES
1771#D- communication information (with -ftrace)
1772  export MPICOMMINF=DETAIL
1773
1774# --------------------------------------------------------------------
1775#D- Other specifications
1776# --------------------------------------------------------------------
1777
1778#D- max number of character/line in output job
1779  export F_SYSLEN=5000
1780#D- number of error that can be admitted on the NEC
1781  export F_ERRCNT=0
1782#D- global performance
1783  export F_PROGINF=DETAIL
1784
1785#D- I/O performance (FORTRAN I/O only not netCDF)
1786  export F_FILEINF=${F_FILEINF:=NO}
1787#D- netCDF I/O performance
1788  export NC_FILEINF=${NC_FILEINF:=NO}
1789
1790  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1791}
1792
1793############################################################
1794# Desactivate Running Environnment Variables
1795
1796function IGCM_sys_desactiv_variables {
1797  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1798  if ( $DEBUG_sys ) ; then
1799    echo "IGCM_sys_desactiv_variables"
1800  fi
1801# --------------------------------------------------------------------
1802#D- MPI specifications
1803# --------------------------------------------------------------------
1804
1805#D-- MPIPROGINF
1806  export MPIPROGINF=NO
1807
1808# --------------------------------------------------------------------
1809#D- Other specifications
1810# --------------------------------------------------------------------
1811
1812#D- global performance
1813  export F_PROGINF=NO 
1814
1815  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1816}
1817
1818############################################################
1819# Build MPI/OMP scripts
1820function IGCM_sys_build_execution_scripts
1821{
1822  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1823  if ( $DEBUG_sys ) ; then
1824    echo "IGCM_sys_build_execution_scripts " $@
1825  fi
1826
1827  typeset NbNodes_Job NbProc_Job comp_proc_mpi_loc comp_proc_omp_loc mpi_count
1828 
1829  if [ ! -f ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} ]  ; then
1830    IGCM_debug_Exit "IGCM_sys_mercurex9 build_execution_scripts : Job_${config_UserChoices_JobName} don't exist in SUBMIT_DIR : ${SUBMIT_DIR} "
1831  fi
1832
1833  if ( ${OK_PARA_MPMD} ) ; then
1834
1835    if [ -f run_file ] ; then
1836      IGCM_sys_Rm -f run_file
1837    fi
1838    touch run_file
1839
1840    if ( ${OK_PARA_OMP} ) ; then
1841      IGCM_debug_Print 2 "Error in config.card on SX9 : OpenMP not available"
1842      exit 1
1843    else
1844# OLD :
1845#PBS -v BATCH_NUM_PROC_TOT=4
1846#PBS -l cpunum_job=${BATCH_NUM_PROC_TOT}
1847      echo "Job_${config_UserChoices_JobName} includes BATCH_NUM_PROC_TOT = \c"
1848      cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep -- "-v *BATCH_NUM_PROC_TOT" | sed -e "s/.*BATCH_NUM_PROC_TOT *= *//" 
1849      NbProc_Job=$( cat ${SUBMIT_DIR}/Job_${config_UserChoices_JobName} | grep -- "-v *BATCH_NUM_PROC_TOT" | sed -e "s/.*BATCH_NUM_PROC_TOT *= *//" )
1850      NbProc_Job=${NbProc_Job:=0}
1851      if [ ${NbProc_Job} -eq 0 ] ; then
1852        IGCM_debug_Print 2 "Error in Job_${config_UserChoices_JobName} ressources : no BATCH_NUM_PROC_TOT defined with MPI only run."
1853        exit 1
1854      fi
1855    fi
1856
1857# run_file construction
1858
1859# Then first loop on the components for the coupler ie oasis
1860
1861### the coupler ie oasis must be the first one
1862    for comp in ${config_ListOfComponents[*]} ; do
1863     
1864      eval ExeNameIn=\${config_Executable_${comp}[0]}
1865      eval ExeNameOut=\${config_Executable_${comp}[1]}
1866     
1867      # for CPL component only
1868      if [ "X${comp}" = "XCPL" ] ; then
1869
1870        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1871        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1872       
1873        if ( ${OK_PARA_MPI} ) ; then
1874          echo "-p ${comp_proc_mpi_loc} -e ./${ExeNameOut}" >> run_file
1875        fi
1876      fi
1877    done
1878
1879# Then second loop on the components
1880
1881    for comp in ${config_ListOfComponents[*]} ; do
1882     
1883      eval ExeNameIn=\${config_Executable_${comp}[0]}
1884      eval ExeNameOut=\${config_Executable_${comp}[1]}
1885     
1886      # Only if we really have an executable for the component and not the coupler ie oasis:
1887      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1888
1889        eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1890        eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1891       
1892        if ( ${OK_PARA_MPI} ) ; then
1893          echo "-p ${comp_proc_mpi_loc} -e ./${ExeNameOut}" >> run_file
1894        fi
1895      fi
1896    done
1897
1898    EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1899
1900    IGCM_sys_Chmod u+x run_file
1901    if ( $DEBUG_sys ) ; then
1902      echo "run_file contains : "
1903      cat run_file
1904    fi
1905
1906  else # Only one executable. launch it.
1907
1908    for comp in ${config_ListOfComponents[*]} ; do
1909
1910      eval ExeNameIn=\${config_Executable_${comp}[0]}
1911      eval ExeNameOut=\${config_Executable_${comp}[1]}
1912
1913      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1914        EXECUTION="time mpirun -np ${BATCH_NUM_PROC_TOT} ./${ExeNameOut}"
1915      fi
1916    done
1917
1918  fi
1919
1920  IGCM_debug_Print 1 "sys mercurex9 : execution command is "
1921  IGCM_debug_Print 1 "$EXECUTION"
1922
1923  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1924
1925}
1926
1927function IGCM_sys_build_run_file {
1928
1929IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1930
1931}
1932
1933############################################################
1934# Check of space available on temporary filesytems
1935function IGCM_sys_check_quota {
1936  IGCM_debug_PushStack "IGCM_sys_check_quota" 
1937
1938  if ( $DEBUG_sys ) ; then
1939    echo "IGCM_sys_check_quota "
1940  fi
1941  # Limit of quota (in %)
1942  limit_quota=95
1943
1944  # Check of the volume (use of RshMaster waiting for ccc_quota command available on SX9 nodes)
1945#  volume_quota=$( IGCM_sys_RshMaster "/applications/ccc-tools/ccc_quota | grep ' scratch' | awk '{print \$2}'" )
1946#  volume_avail=$( IGCM_sys_RshMaster "/applications/ccc-tools/ccc_quota | grep ' scratch' | awk '{print \$3}'" )
1947
1948  volume_quota=$(quota -h | grep 'scratch' | awk '{print $2}')
1949  volume_avail=$(quota -h | grep 'scratch' | awk '{print $3}')
1950
1951  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1952
1953    unit_quota=$( echo $volume_quota | awk '{print(substr($0, length($0), length($0)))}' )
1954    unit_avail=$( echo $volume_avail | awk '{print(substr($0, length($0), length($0)))}' )
1955
1956    if [ "${unit_quota}" = "*" ] ; then
1957        IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1958        IGCM_debug_Print 1 "More than 100% of your quota is used"
1959        IGCM_debug_Print 1 "Use the ccc_quota command to check"
1960        IGCM_debug_Print 1 "You must have more than 10% available to run"
1961        IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1962        IGCM_debug_Verif_Exit
1963    fi
1964
1965    temp_avail=${volume_avail%%${unit_avail}*}
1966    temp_quota=${volume_quota%%${unit_quota}*}
1967
1968    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1969
1970    # Convertion
1971      if [ ${unit_avail} = "T" ] ; then
1972          (( temp_avail = temp_avail * 1000000000000 ))
1973      elif [ ${unit_avail} = "G" ] ; then
1974          (( temp_avail = temp_avail * 1000000000 ))
1975      elif [ ${unit_avail} = "M" ] ; then
1976          (( temp_avail = temp_avail * 1000000 ))
1977      elif [ ${unit_avail} = "k" ] ; then
1978          (( temp_avail = temp_avail * 1000 ))
1979      else
1980          (( temp_avail = volume_avail ))
1981      fi
1982      if [ ${unit_quota} = "T" ] ; then
1983          (( temp_quota = temp_quota * 1000000000000 ))
1984      elif [ ${unit_quota} = "G" ] ; then
1985          (( temp_quota = temp_quota * 1000000000 ))
1986      elif [ ${unit_quota} = "M" ] ; then
1987          (( temp_quota = temp_quota * 1000000 ))
1988      elif [ ${unit_quota} = "k" ] ; then
1989          (( temp_quota = temp_quota * 1000 ))
1990      else
1991          (( temp_quota = volume_quota ))
1992      fi
1993    fi
1994
1995    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1996#    echo "volume ratio is " $quota_volume
1997
1998    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1999      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
2000      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
2001      IGCM_debug_Print 1 "Use the ccc_quota command to check"
2002      IGCM_debug_Print 1 "You must have more than 10% available to run"
2003      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
2004      IGCM_debug_Verif_Exit
2005    fi
2006
2007  fi
2008  IGCM_debug_PopStack "IGCM_sys_check_quota"
2009}
Note: See TracBrowser for help on using the repository browser.