source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_mercure.ksh @ 695

Last change on this file since 695 was 695, checked in by sdipsl, 12 years ago
  • mistake
  • Property svn:keywords set to Revision Author Date
File size: 50.0 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Mercure X64
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe param files |            |  Chmod  |                           |
46# |          |      Qsub           |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset  HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset  LOGIN=${LOGIN:=$( whoami )}
69
70#D-
71#D-#==================================================
72#D-Program used in libIGCM
73#D-#==================================================
74
75# rsync with path
76typeset -r RSYNC=/usr/bin/rsync
77# RSYNC_opt args to rsync
78typeset -r RSYNC_opt="-va"
79# ie storage filesystem
80typeset -r RHOST=mercure
81
82#====================================================
83# Set environment tools (ferret, nco, cdo)
84#====================================================
85. /home/cont003/p86ipsl/.atlas_env_mercure01_ksh
86
87#====================================================
88# Host specific DIRECTORIES
89#====================================================
90
91#====================================================
92#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
93typeset -r R_EXE="${MODIPSL}/bin"
94
95#====================================================
96# For interactive jobs on mercure
97PBS_O_WORKDIR=${PBS_O_WORKDIR:=$(pwd)}
98
99#====================================================
100#- SUBMIT_DIR : submission dir
101typeset SUBMIT_DIR=${SUBMIT_DIR:=${PBS_O_WORKDIR}}
102
103#====================================================
104#- ARCHIVE (dedicated to large files)
105typeset -r ARCHIVE=${CCCSTOREDIR}
106
107#- ARCHIVE (dedicated to small/medium files)
108typeset -r STORAGE=${CCCWORKDIR}
109
110#====================================================
111#- IN
112typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
113typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/dmnfs/cont003/p24data}
114
115#====================================================
116#- R_OUT
117typeset -r R_OUT=${ARCHIVE}/IGCM_OUT
118
119#====================================================
120#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
121typeset -r R_FIG=${STORAGE}/IGCM_OUT
122
123#====================================================
124#- R_BUF (ONLY FOR double copy an scratch)
125typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
126
127#====================================================
128#- BIG_DIR : BIG_DIR to store files waiting for rebuild
129typeset -r BIG_DIR=${BIG_DIR:=${SCRATCHDIR}/REBUILD}
130
131#====================================================
132#- OUT_POST
133typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT
134
135#====================================================
136#- RUN_DIR_PATH : Temporary working directory (=> TMP)
137typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${TMPDIR}}
138
139#====================================================
140#- Max number of arguments passed to nco operator or demigration command
141UNIX_MAX_LIMIT=360
142
143#- set PackDefault true on NEC front-end
144PackDefault=true
145
146#D-#==================================================
147#D-function IGCM_sys_RshMaster
148#D-* Purpose: Just a fake command to wrapp
149#D-           IGCM_card call in post-treatment
150#D-           Ulam do not see brodie filesystem
151#D-           Cesium do not see all mercure filesystem
152#D-           That's why we need this hack.
153#D-* Examples:
154#D-
155function IGCM_sys_RshMaster {
156  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
157  /bin/ksh <<-EOF
158    export libIGCM=${libIGCM}
159    export DEBUG_debug=${DEBUG_debug}
160    . ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
161    . ${libIGCM}/libIGCM_card/libIGCM_card.ksh
162    ${@}
163EOF
164  if [ $? -gt 0 ] ; then
165    echo "IGCM_sys_RshMaster : erreur."
166    IGCM_debug_Exit "IGCM_sys_RshMaster"
167  fi
168  IGCM_debug_PopStack "IGCM_sys_RshMaster"
169}
170
171#D-#==================================================
172#D-function IGCM_sys_RshArchive
173#D-* Purpose: Archive rsh command
174#D-* Examples:
175#D-
176function IGCM_sys_RshArchive {
177  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
178  /bin/ksh <<-EOF
179    ${@}
180EOF
181  if [ $? -gt 0 ] ; then
182    echo "IGCM_sys_RshArchive : erreur."
183    IGCM_debug_Exit "IGCM_sys_RshArchive"
184  fi
185  IGCM_debug_PopStack "IGCM_sys_RshArchive"
186}
187
188#D-#==================================================
189#D-function IGCM_sys_RshPost
190#D-* Purpose: Post-process rsh command
191#D-* Examples:
192#D-
193function IGCM_sys_RshPost {
194  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
195  if ( $DEBUG_sys ) ; then
196    echo "IGCM_sys_RshPost :" $@
197  fi
198  /bin/ksh ${@}
199  if [ $? -gt 0 ] ; then
200    echo "IGCM_sys_RshPost : erreur."
201    IGCM_debug_Exit "IGCM_sys_RshPost"
202  fi
203  IGCM_debug_PopStack "IGCM_sys_RshPost"
204}
205
206#D-#==================================================
207#D-function IGCM_sys_SendMail
208#D-* Purpose: Send mail when simulation is over
209#D-* Examples:
210#D-
211function IGCM_sys_SendMail {
212  IGCM_debug_PushStack "IGCM_sys_SendMail" $@
213  if ( $DEBUG_sys ) ; then
214    echo "IGCM_sys_SendMail :" $@
215  fi
216
217  if ( ${ExitFlag} ) ; then
218    status=failed
219  else
220    status=completed
221  fi
222
223  cat  << END_MAIL > job_end.mail
224Dear ${LOGIN},
225
226  Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
227  Job started : ${DateBegin}
228  Job ended   : ${DateEnd}
229  Output files are available in ${R_SAVE}
230  Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
231END_MAIL
232
233  if  [ X"${config_UserChoices_MailName}" != X ] ; then
234    mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} <  job_end.mail
235  elif [ -f ~/.forward ] ; then
236    mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
237  else
238    mailx -s "${config_UserChoices_JobName} ${status}" ${LOGIN} < job_end.mail
239  fi
240
241  if [ $? -gt 0 ] ; then
242    echo "IGCM_sys_SendMail : erreur."
243    IGCM_debug_Exit "IGCM_sys_SendMail"
244  fi
245  IGCM_debug_PopStack "IGCM_sys_SendMail"
246}
247
248#D-#==================================================
249#D-function IGCM_sys_Mkdir
250#D-* Purpose: Master locale mkdir command
251#D-* Examples:
252#D-
253function IGCM_sys_Mkdir {
254  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
255  if ( $DEBUG_sys ) ; then
256    echo "IGCM_sys_Mkdir :" $@
257  fi
258  if [ ! -d ${1} ]; then
259    \mkdir -p $1
260    if [ $? -gt 0 ] ; then
261      echo "IGCM_sys_Mkdir : erreur."
262      IGCM_debug_Exit "IGCM_sys_Mkdir"
263    fi
264  fi
265  # vérification :
266  if [ ! -d ${1} ] ; then
267    echo "IGCM_sys_Mkdir : erreur."
268    IGCM_debug_Exit "IGCM_sys_Mkdir"
269  fi
270  IGCM_debug_PopStack "IGCM_sys_Mkdir"
271}
272
273#D-#==================================================
274#D-function IGCM_sys_MkdirArchive
275#D-* Purpose: Mkdir on Archive
276#D-* Examples:
277#D-
278function IGCM_sys_MkdirArchive {
279  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
280  if ( $DEBUG_sys ) ; then
281    echo "IGCM_sys_MkdirArchive :" $@
282  fi
283  #- creation de repertoire sur le serveur fichier
284  if [ ! -d ${1} ]; then
285    \mkdir -p $1
286    if [ $? -gt 0 ] ; then
287      echo "IGCM_sys_MkdirArchive : erreur."
288      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
289    fi
290  fi
291  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
292}
293
294#D-#==================================================
295#D-function IGCM_sys_MkdirWork
296#D-* Purpose: Mkdir on Work
297#D-* Examples:
298#D-
299function IGCM_sys_MkdirWork {
300  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
301  if ( $DEBUG_sys ) ; then
302    echo "IGCM_sys_MkdirWork :" $@
303  fi
304  #- creation de repertoire sur le serveur fichier
305  if [ ! -d ${1} ]; then
306    \mkdir -p $1
307    if [ $? -gt 0 ] ; then
308      echo "IGCM_sys_MkdirWork : erreur."
309      IGCM_debug_Exit "IGCM_sys_MkdirWork"
310    fi
311  fi
312  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
313}
314
315#D-#==================================================
316#D-function IGCM_sys_Cd
317#D-* Purpose: master cd command
318#D-* Examples:
319#D-
320function IGCM_sys_Cd {
321  IGCM_debug_PushStack "IGCM_sys_Cd" $@
322  if ( $DEBUG_sys ) ; then
323    echo "IGCM_sys_Cd :" $@
324  fi
325  \cd $1
326  if [ $? -gt 0 ] ; then
327    echo "IGCM_sys_Cd : erreur."
328    IGCM_debug_Exit "IGCM_sys_Cd"
329  fi
330  IGCM_debug_PopStack "IGCM_sys_Cd"
331}
332
333#D-#==================================================
334#D-function IGCM_sys_Chmod
335#D-* Purpose: Chmod
336#D-* Examples:
337#D-
338function IGCM_sys_Chmod {
339  IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
340  if ( $DEBUG_sys ) ; then
341    echo "IGCM_sys_Chmod :" $@
342  fi
343  if [ $DRYRUN -le 1 ]; then
344    \chmod $@
345    if [ $? -gt 0 ] ; then
346      echo "IGCM_sys_Chmod : erreur."
347      IGCM_debug_Exit "IGCM_sys_Chmod"
348    fi
349  else
350    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
351  fi
352  IGCM_debug_PopStack "IGCM_sys_Chmod"
353}
354
355#D-#==================================================
356#D-function IGCM_sys_FileSize
357#D-* Purpose: Filesize
358#D-* Examples:
359#D-
360function IGCM_sys_FileSize {
361  IGCM_debug_PushStack "IGCM_sys_FileSize" $@
362
363  typeset sizeF
364  set +A sizeF -- $( ls -la ${1} )
365  if [ $? -gt 0 ] ; then
366    IGCM_debug_Exit "IGCM_sys_FileSize"
367  fi
368  eval ${2}=${sizeF[4]}
369
370  IGCM_debug_PopStack "IGCM_sys_FileSize"
371}
372
373#D-#==================================================
374#D-function IGCM_sys_TestDir
375#D-* Purpose: Test Directory that must exists
376#D-* Examples:
377#D-
378function IGCM_sys_TestDir {
379  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
380  if ( $DEBUG_sys ) ; then
381    echo "IGCM_sys_TestDir :" $@
382  fi
383  typeset ExistFlag
384  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
385  IGCM_debug_PopStack "IGCM_sys_TestDir"
386
387  return ${ExistFlag}
388}
389
390#D-#==================================================
391#D-function IGCM_sys_TestDirArchive
392#D-* Purpose: Test Directory that must exists on Archive
393#D-* Examples:
394#D-
395function IGCM_sys_TestDirArchive {
396  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
397  if ( $DEBUG_sys ) ; then
398    echo "IGCM_sys_TestDirArchive :" $@
399  fi
400  typeset ExistFlag
401  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
402  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
403
404  return ${ExistFlag}
405}
406
407#D-#==================================================
408#D-function IGCM_sys_IsFileArchived
409#D-* Purpose: Test file that must NOT EXISTS on Archive
410#D-* Examples:
411#D-
412function IGCM_sys_IsFileArchived {
413  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
414  if ( $DEBUG_sys ) ; then
415    echo "IGCM_sys_IsFileArchived :" $@
416  fi
417  typeset IsArchivedFlag
418  IsArchivedFlag=$( [ X$( echo $1 | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
419  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
420
421  return ${IsArchivedFlag}
422}
423
424#D-#==================================================
425#D-function IGCM_sys_TestFileArchive
426#D-* Purpose: Test file that must NOT EXISTS on Archive
427#D-* Examples:
428#D-
429function IGCM_sys_TestFileArchive {
430  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
431  typeset ExistFlag
432  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
433  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
434
435  return ${ExistFlag}
436}
437
438#D-#==================================================
439#D-function IGCM_sys_TestFileBuffer
440#D-* Purpose: Test file that must NOT EXISTS on Buffer
441#D-* Examples:
442#D-
443function IGCM_sys_TestFileBuffer {
444  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
445  typeset ExistFlag
446  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
447  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
448
449  return ${ExistFlag}
450}
451
452#D-#==================================================
453#D-function IGCM_sys_CountFileArchive
454#D-* Purpose: Count files on Archive filesystem
455#D-* Examples:
456#D-
457function IGCM_sys_CountFileArchive {
458  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
459  ls ${@} 2>/dev/null | wc -l
460  if [ $? -gt 0 ] ; then
461    echo "IGCM_sys_CountFileArchive : erreur."
462  fi
463  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
464}
465
466#D-#==================================================
467#D-function IGCM_sys_CountFileBuffer
468#D-* Purpose: Count files on Scratch filesystem
469#D-* Examples:
470#D-
471function IGCM_sys_CountFileBuffer {
472  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
473  ls ${@} 2>/dev/null | wc -l
474  if [ $? -gt 0 ] ; then
475    echo "IGCM_sys_CountFileBuffer : erreur."
476  fi
477  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
478}
479
480#D-#==================================================
481#D-function IGCM_sys_Tree
482#D-* Purpose: Tree directories with files on ${ARCHIVE}
483#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
484#D-
485function IGCM_sys_Tree {
486  IGCM_debug_PushStack "IGCM_sys_Tree" $@
487  if ( $DEBUG_sys ) ; then
488    echo "IGCM_sys_Tree :" $@
489  fi
490
491  \tree -f $@
492
493  IGCM_debug_PopStack "IGCM_sys_Tree"
494}
495
496#D-#==================================================
497#D-function IGCM_sys_Tar
498#D-* Purpose: master tar command
499#D-* Examples:
500#D-
501function IGCM_sys_Tar {
502  IGCM_debug_PushStack "IGCM_sys_Tar" $@
503  if ( $DEBUG_sys ) ; then
504    echo "IGCM_sys_Tar :" $@
505  fi
506  \tar cf $@
507  if [ $? -gt 0 ] ; then
508    echo "IGCM_sys_Tar : erreur."
509    IGCM_debug_Exit "IGCM_sys_Tar"
510  fi
511  IGCM_debug_PopStack "IGCM_sys_Tar"
512}
513
514#D-#==================================================
515#D-function IGCM_sys_UnTar
516#D-* Purpose: master un-tar command
517#D-* Examples:
518#D-
519function IGCM_sys_UnTar {
520  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
521  if ( $DEBUG_sys ) ; then
522    echo "IGCM_sys_UnTar :" $@
523  fi
524  \tar xvf $1
525  if [ $? -gt 0 ] ; then
526    echo "IGCM_sys_UnTar : erreur."
527    IGCM_debug_Exit "IGCM_sys_UnTar"
528  fi
529  IGCM_debug_PopStack "IGCM_sys_UnTar"
530}
531
532#D-#==================================================
533#D-function IGCM_sys_QsubPost
534#D-* Purpose: Qsub new job on scalaire
535#D-* Examples:
536#D-
537function IGCM_sys_QsubPost {
538  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
539  if ( $DEBUG_sys ) ; then
540    echo "IGCM_sys_QsubPost :" $@
541  fi
542  /usr/bin/nqsII/qsub -q scalaire -o ${POST_DIR}/${Script_Post_Output}.out ${libIGCM}/$1.job -v ${listVarEnv}
543  if [ $? -gt 0 ] ; then
544    echo "IGCM_sys_QsubPost : erreur " $@
545    IGCM_debug_Exit "IGCM_sys_QsubPost"
546  fi
547  IGCM_debug_PopStack "IGCM_sys_QsubPost"
548}
549
550#D-*************************
551#D- File transfer functions
552#D-*************************
553#D-
554
555#D-#==================================================
556#D-function IGCM_sys_Rsync_out
557#D-* Purpose: treat return val of rsync
558#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
559#D-  Error values and explanations can depend on your system version.
560function IGCM_sys_Rsync_out {
561  RET=$1
562  if [ ! $RET ] ; then
563    echo "rsync error !"
564  fi
565
566  if [ $MYLANG = "fr" ]; then
567    case $RET in
568    0)  return ;;
569    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
570      echo "Erreur de syntaxe ou d'utilisation."
571      return;;
572    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
573      echo "Incompatibilité de protocole."
574      return;;
575    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
576      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
577      echo "répertoires"
578      return;;
579    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
580      echo "Action demandée non supportée : une tentative de manipulation de"
581      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
582      echo "été faite ; ou une option qui est supportée par le  client  mais"
583      echo "pas par le serveur a été spécifiée."
584      return;;
585    10) echo "Erreur de rsync ; RERR_SOCKETIO"
586      echo "Erreur dans le socket d'entrée sortie"
587      return;;
588    11) echo "Erreur de rsync ; RERR_FILEIO"
589      echo "Erreur d'entrée sortie fichier"
590      return;;
591    12) echo "Erreur de rsync ; RERR_STREAMIO"
592      echo "Erreur dans flux de donnée du protocole rsync"
593      return;;
594    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
595      echo "Erreur avec les diagnostics du programme"
596      return;;
597    14) echo "Erreur de rsync ; RERR_IPC"
598      echo "Erreur dans le code IPC"
599      return;;
600    20) echo "Erreur de rsync ; RERR_SIGNAL"
601      echo "SIGUSR1 ou SIGINT reçu"
602      return;;
603    21) echo "Erreur de rsync ; RERR_WAITCHILD"
604      echo "Une erreur retournée par waitpid()"
605      return;;
606    22) echo "Erreur de rsync ; RERR_MALLOC"
607      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
608      return;;
609    23) echo ""
610      echo "Erreur fichier inexistant"
611      return;;
612    30) echo "Erreur de rsync ; RERR_TIMEOUT"
613      echo "Temps d'attente écoulé dans l'envoi/réception de données"
614      return;;
615    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $RET
616      return;;
617    esac
618  elif [ $MYLANG = "en" ] ; then
619    case $RET in
620    0)  return;;
621    1)  echo "rsync error : Syntax or usage error "
622      return;;
623    2)  echo "rsync error : Protocol incompatibility "
624      return;;
625    3)  echo "rsync error : Errors selecting input/output files, dirs"
626      return;;
627    4)  echo "rsync error : Requested action not supported: an attempt"
628      echo "was made to manipulate 64-bit files on a platform that cannot support"
629      echo "them; or an option was specified that is supported by the client and"
630      echo "not by the server."
631      return;;
632    5)  echo "rsync error : Error starting client-server protocol"
633      return;;
634    10) echo "rsync error : Error in socket I/O "
635      return;;
636    11) echo "rsync error : Error in file I/O "
637      return;;
638    12) echo "rsync error : Error in rsync protocol data stream "
639      return;;
640    13) echo "rsync error : Errors with program diagnostics "
641      return;;
642    14) echo "rsync error : Error in IPC code "
643      return;;
644    20) echo "rsync error : Received SIGUSR1 or SIGINT "
645      return;;
646    21) echo "rsync error : Some error returned by waitpid() "
647      return;;
648    22) echo "rsync error : Error allocating core memory buffers "
649      return;;
650    23) echo "rsync error : Partial transfer due to error"
651      return;;
652    24) echo "rsync error : Partial transfer due to vanished source files"
653      return;;
654    30) echo "rsync error : Timeout in data send/receive "
655      return;;
656    *)  echo "rsync error : return code of rsync unknown :" $RET
657      return;;
658    esac
659  else
660    echo "unknown language $MYLANG."
661    return
662  fi
663}
664
665#D-#==================================================
666#D-function IGCM_sys_Cp
667#D-* Purpose: generic cp
668#D-* Examples:
669#D-
670function IGCM_sys_Cp {
671  IGCM_debug_PushStack "IGCM_sys_Cp" $@
672  if ( $DEBUG_sys ) ; then
673    echo "IGCM_sys_Cp :" $@
674  fi
675
676  typeset RET
677
678  echo cp $@ > out_rsync 2>&1
679  \cp $@ >> out_rsync 2>&1
680  RET=$?
681
682  if [ ${RET} -gt 0 ] ; then
683    echo "IGCM_sys_Cp : error."
684    cat out_rsync
685    IGCM_debug_Exit "IGCM_sys_Cp"
686  else
687    \rm out_rsync
688  fi
689  IGCM_debug_PopStack "IGCM_sys_Cp"
690}
691
692#D-#==================================================
693#D-function IGCM_sys_Rm
694#D-* Purpose: generic rm
695#D-* Examples:
696#D-
697function IGCM_sys_Rm {
698  IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
699  if ( $DEBUG_sys ) ; then
700    echo "IGCM_sys_Rm :" $@
701  fi
702
703  typeset RET
704
705  echo rm $@ > out_rsync 2>&1
706  \rm $@ >> out_rsync 2>&1
707  RET=$?
708
709  if [ ${RET} -gt 0 ] ; then
710    echo "IGCM_sys_Rm : error."
711    cat out_rsync
712    IGCM_debug_Exit "IGCM_sys_Rm"
713  else
714    \rm out_rsync
715  fi
716  IGCM_debug_PopStack "IGCM_sys_Rm"
717}
718
719#D-#==================================================
720#D-function IGCM_sys_RmRunDir
721#D-* Purpose: rm tmpdir (dummy function most of the time batch
722#D-                      scheduler will do the job)
723#D-           Dummy function on mercure front-end
724#D-* Examples:
725#D-
726function IGCM_sys_RmRunDir {
727    IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
728    if ( $DEBUG_sys ) ; then
729        echo "IGCM_sys_RmRunDir :" $@
730        echo "Dummy call, let the scheduler do that."
731    fi
732    IGCM_debug_PopStack "IGCM_sys_RmRunDir"
733}
734
735#D-#==================================================
736#D-function IGCM_sys_Mv
737#D-* Purpose: generic move
738#D-* Examples:
739#D-
740function IGCM_sys_Mv {
741  IGCM_debug_PushStack "IGCM_sys_Mv" $@
742  if ( $DEBUG_sys ) ; then
743    echo "IGCM_sys_Mv :" $@
744  fi
745
746  if [ $DRYRUN = 0 ]; then
747
748    typeset RET
749
750    echo mv $@ > out_rsync 2>&1
751    \mv $@ >> out_rsync 2>&1
752    RET=$?
753
754    if [ ${RET} -gt 0 ] ; then
755      echo "IGCM_sys_Mv : error in mv."
756      cat out_rsync
757      IGCM_debug_Exit "IGCM_sys_Mv"
758    else
759      \rm out_rsync
760    fi
761  else
762    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
763  fi
764
765  IGCM_debug_PopStack "IGCM_sys_Mv"
766}
767
768#D-#==================================================
769#D-function IGCM_sys_Put_Dir
770#D-* Purpose: Copy a complete directory on $(ARCHIVE)
771#D-* Examples:
772#D-
773function IGCM_sys_Put_Dir {
774  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
775  if ( $DEBUG_sys ) ; then
776    echo "IGCM_sys_Put_Dir :" $@
777  fi
778  if [ $DRYRUN = 0 ]; then
779    if [ ! -d ${1} ] ; then
780      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
781      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
782      return
783    fi
784
785    typeset RET
786
787    # Only if we use rsync
788    #IGCM_sys_TestDirArchive $( dirname $2 )
789    #
790    #USUAL WAY
791    \cp -r $1 $2 > out_rsync 2>&1
792    RET=$?
793
794    if [ ${RET} -gt 0 ] ; then
795      echo "IGCM_sys_Put_Dir : error."
796      cat out_rsync
797      IGCM_debug_Exit "IGCM_sys_Put_Dir"
798    else
799      \rm out_rsync
800    fi
801  else
802    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
803  fi
804  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
805}
806
807#D-#==================================================
808#D-function IGCM_sys_Get_Dir
809#D-* Purpose: Copy a complete directory from ${ARCHIVE}
810#D-* Examples:
811#D-
812function IGCM_sys_Get_Dir {
813  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
814  if ( $DEBUG_sys ) ; then
815    echo "IGCM_sys_Get_Dir :" $@
816  fi
817  if [ $DRYRUN = 0 ]; then
818#    if [ ! -d ${1} ] ; then
819#      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
820#      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
821#      return
822#    fi
823
824    typeset RET
825
826    # Only if we use rsync
827    #IGCM_sys_TestDirArchive $( dirname $2 )
828    #
829    #USUAL WAY
830    # add dmget (to demigrate all offline files) to reduce time of this command :
831    #dmget $1/*
832    ccc_hsm get $1/*
833    \cp -r $1 $2 > out_rsync 2>&1
834    RET=$?
835
836    if [ ${RET} -gt 0 ] ; then
837      echo "IGCM_sys_Get_Dir : error."
838      cat out_rsync
839      IGCM_debug_Exit "IGCM_sys_Get_Dir"
840    else
841      \rm out_rsync
842    fi
843  else
844    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
845  fi
846  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
847}
848
849#D-#==================================================
850#D-function IGCM_sys_Get_Master
851#D-* Purpose: Copy a complete directory from MASTER filesystem
852#D-* Examples:
853#D-
854function IGCM_sys_Get_Master {
855  IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
856  if ( $DEBUG_sys ) ; then
857    echo "IGCM_sys_Get_Master :" $@
858  fi
859  if [ $DRYRUN = 0 ]; then
860    if ( [ ! -d ${1} ] && [ ! -f ${1} ] ) ; then
861      echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ."
862      IGCM_debug_PopStack "IGCM_sys_Get_Master"
863      return
864    fi
865
866    typeset RET
867
868    #USUAL WAY
869    \cp -rL $1 $2 > out_rsync 2>&1
870    RET=$?
871
872    if [ ${RET} -gt 0 ] ; then
873      echo "IGCM_sys_Get_Master : error."
874      cat out_rsync
875      IGCM_debug_Exit "IGCM_sys_Get_Master"
876    else
877      \rm out_rsync
878    fi
879  else
880    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
881  fi
882  IGCM_debug_PopStack "IGCM_sys_Get_Master"
883}
884
885#D-#==================================================
886#D-function IGCM_sys_Put_Rest
887#D-* Purpose: Put computied restarts on ${ARCHIVE}.
888#D-           File and target directory must exist.
889#D-* Examples:
890#D-
891function IGCM_sys_Put_Rest {
892  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
893  if ( $DEBUG_sys ) ; then
894    echo "IGCM_sys_Put_Rest :" $@
895  fi
896  if [ $DRYRUN = 0 ]; then
897    if [ ! -f ${1} ] ; then
898      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
899      IGCM_debug_Exit "IGCM_sys_Put_Rest"
900    fi
901
902    typeset RET
903    #
904    if [ X${JobType} = XRUN ] ; then
905      IGCM_sys_Chmod 444 ${1}
906    fi
907
908    #
909    # USUAL WAY
910    \cp $1 $2 > out_rsync 2>&1
911    RET=$?
912
913#       #RSYNC WITH NETWORK SSH CALL
914#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
915#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
916
917#       #RSYNC WITH NFS USE
918#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
919#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
920
921#       RET=$?
922#       IGCM_sys_Rsync_out $RET
923
924#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
925#       (( RET=RET+$? ))
926
927    if [ ${RET} -gt 0 ] ; then
928      echo "IGCM_sys_Put_Rest : error."
929      cat out_rsync
930      IGCM_debug_Exit "IGCM_sys_Put_Rest"
931    else
932      \rm out_rsync
933    fi
934  else
935    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
936  fi
937  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
938}
939
940#D-#==================================================
941#D-function IGCM_sys_PutBuffer_Rest
942#D-* Purpose: Put computied restarts on ${SCRATCHDIR}.
943#D-           File and target directory must exist.
944#D-* Examples:
945#D-
946function IGCM_sys_PutBuffer_Rest {
947  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Rest" $@
948  if ( $DEBUG_sys ) ; then
949    echo "IGCM_sys_PutBuffer_Rest :" $@
950  fi
951  if [ $DRYRUN = 0 ]; then
952    if [ ! -f ${1} ] ; then
953      echo "ERROR : IGCM_sys_PutBuffer_Rest ${1} DOES NOT EXIST ."
954      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
955    fi
956
957    typeset RET
958    #
959    if [ X${JobType} = XRUN ] ; then
960      IGCM_sys_Chmod 444 ${1}
961    fi
962
963    #
964    # USUAL WAY
965    \cp $1 $2 > out_rsync 2>&1
966    RET=$?
967
968    if [ ${RET} -gt 0 ] ; then
969      echo "IGCM_sys_PutBuffer_Rest : error."
970      cat out_rsync
971      IGCM_debug_Exit "IGCM_sys_PutBuffer_Rest"
972    else
973      \rm out_rsync
974    fi
975  else
976    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
977  fi
978  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Rest"
979}
980
981#D-#==================================================
982#D-function IGCM_sys_Put_Out
983#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
984#D-* Examples:
985#D-
986function IGCM_sys_Put_Out {
987  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
988  if ( $DEBUG_sys ) ; then
989    echo "IGCM_sys_Put_Out :" $@
990  fi
991
992  typeset NB_ESSAI DELAI RET i exist skip
993
994  # number of tentative
995  NB_ESSAI=3
996  # time delay between tentative
997  DELAI=2
998
999  if [ $DRYRUN = 0 ]; then
1000    if [ ! -f ${1} ] ; then
1001      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1002      IGCM_debug_PopStack "IGCM_sys_Put_Out"
1003      return 1
1004    fi
1005    #
1006    IGCM_sys_MkdirArchive $( dirname $2 )
1007    #
1008
1009    #=====================================================
1010    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1011    #=====================================================
1012
1013    #echo ${2} | grep "${R_OUT}" > /dev/null 2>&1
1014    #if [ $? -eq 0 ] ; then
1015    #    typeset WORKPATH FILEPATH
1016    #    WORKPATH=$( dirname $2 | sed -e "s|${R_OUT}|${R_BUF}|" )
1017    #    IGCM_sys_MkdirWork ${WORKPATH}
1018    #    FILEPATH=${WORKPATH}/$( basename $2 )
1019    #    #
1020    #    IGCM_sys_Cp ${1} ${FILEPATH}
1021    #fi
1022
1023    if [ X${JobType} = XRUN ] ; then
1024      if [ X${3} = X ] ; then
1025        IGCM_sys_Chmod 444 ${1}
1026      fi
1027    fi
1028    #
1029
1030    exist=false
1031    skip=false
1032    if [ -f $2 ] ; then
1033      IGCM_debug_Print 1 "$2 already exist"
1034      #dmget $2
1035      ccc_hsm get $2
1036      exist=true
1037      if [ "X$( diff $1 $2 )" = X ] ; then
1038        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1039        skip=true
1040      else
1041        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1042        skip=false
1043      fi
1044    fi
1045    #
1046    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1047      IGCM_sys_Chmod u+w $2
1048    fi
1049
1050    if [ X${skip} = Xfalse ] ; then
1051      (( i = 0 ))
1052      while [ $i -lt $NB_ESSAI ] ; do
1053        # USUAL WAY
1054        \cp $1 $2 > out_rsync 2>&1
1055        RET=$?
1056        if [ ${RET} -gt 0 ]; then
1057          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed ${i}/${NB_ESSAI}"
1058          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
1059          sleep $DELAI
1060        else
1061          break
1062        fi
1063        (( i = i + 1 ))
1064      done
1065    fi
1066
1067#       #RSYNC WITH NETWORK SSH CALL
1068#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1069#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1070
1071#       #RSYNC WITH NFS USE
1072#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1073#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1074
1075#       RET=$?
1076#       IGCM_sys_Rsync_out $RET
1077
1078#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1079#       (( RET=RET+$? ))
1080
1081    if [ ${RET} -gt 0 ] ; then
1082      echo "IGCM_sys_Put_Out : error."
1083      cat out_rsync
1084      IGCM_debug_Exit "IGCM_sys_Put_Out"
1085    else
1086      \rm out_rsync
1087    fi
1088  else
1089    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1090  fi
1091  IGCM_debug_PopStack "IGCM_sys_Put_Out"
1092  return 0
1093}
1094
1095#D-#==================================================
1096#D-function IGCM_sys_PutBuffer_Out
1097#D-* Purpose: Copy a file on ${SCRATCHDIR} after having chmod it in readonly
1098#D-* Examples:
1099#D-
1100function IGCM_sys_PutBuffer_Out {
1101  IGCM_debug_PushStack "IGCM_sys_PutBuffer_Out" $@
1102  if ( $DEBUG_sys ) ; then
1103    echo "IGCM_sys_PutBuffer_Out :" $@
1104  fi
1105  if [ $DRYRUN = 0 ]; then
1106    if [ ! -f ${1} ] ; then
1107      echo "WARNING : IGCM_sys_PutBuffer_Out ${1} DOES NOT EXIST ."
1108      IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1109      return 1
1110    fi
1111    #
1112    IGCM_sys_Mkdir $( dirname $2 )
1113    #
1114    typeset RET
1115
1116    if [ X${JobType} = XRUN ] ; then
1117      if [ X${3} = X ] ; then
1118        IGCM_sys_Chmod 444 ${1}
1119      fi
1120    fi
1121    #
1122    # USUAL WAY
1123    \cp $1 $2 > out_rsync 2>&1
1124    RET=$?
1125
1126    if [ ${RET} -gt 0 ] ; then
1127      echo "IGCM_sys_PutBuffer_Out : error."
1128      cat out_rsync
1129      IGCM_debug_Exit "IGCM_sys_PutBuffer_Out"
1130    else
1131      \rm out_rsync
1132    fi
1133  else
1134    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1135  fi
1136  IGCM_debug_PopStack "IGCM_sys_PutBuffer_Out"
1137  return 0
1138}
1139
1140#D-#==================================================
1141#D-function IGCM_sys_Get
1142#D-* Purpose: Get a file from ${ARCHIVE}
1143#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1144#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
1145function IGCM_sys_Get {
1146  IGCM_debug_PushStack "IGCM_sys_Get" $@
1147
1148  typeset DEST RET dm_liste ifile target file_work
1149  typeset NB_ESSAI DELAI status i
1150
1151  if ( $DEBUG_sys ) ; then
1152    echo "IGCM_sys_Get :" $@
1153  fi
1154
1155  # number of tentative
1156  NB_ESSAI=3
1157  # time delay between tentative
1158  DELAI=2
1159
1160  if [ $DRYRUN -le 2 ]; then
1161    if [ X${1} = X'/l' ] ; then
1162      # test if the first file is present in the old computation :
1163      eval set +A dm_liste \${${2}}
1164    else
1165      eval set +A dm_liste ${1}
1166    fi
1167    eval DEST=\${${#}}
1168
1169    #=====================================================
1170    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1171    #=====================================================
1172
1173    # Is it an R_OUT file (not R_IN) ?
1174    #echo ${dm_liste[0]} | grep "${R_OUT}" > /dev/null 2>&1
1175    #if [ $? -eq 0 ] ; then
1176    #    # Yes  ? then we try to get it in SCRATCHDIR
1177    #    set +A file_work $( echo ${dm_liste[*]} | sed -e "s|${R_OUT}|${R_BUF}|g" )
1178    #    if [ -f ${file_work[0]} ] ; then
1179    #   IGCM_sys_Cp ${file_work[*]} ${DEST}
1180    #   IGCM_debug_PopStack "IGCM_sys_Get"
1181    #   return
1182    #    fi
1183    #fi
1184
1185    ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1186    RET=$?
1187    if [ ${RET} -gt 0 ] ; then
1188      echo "WARNING IGCM_sys_Get : demigration error."
1189      cat out_rsync
1190      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1191    fi
1192
1193    #if [ ${RET} -gt 0 ] ; then
1194    #    if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
1195    #   cat out_rsync
1196    #   echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1197    #   sleep 30
1198    #   echo "We try another time"
1199    ##  dmget ${dm_liste[*]} > out_rsync 2>&1
1200    #   ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1201    #   RET=$?
1202    #   if [ ${RET} -gt 0 ] ; then
1203    #       echo "ERROR IGCM_sys_Get : again demigration error :"
1204    #       cat out_rsync
1205    #       IGCM_debug_Exit "IGCM_sys_Get"
1206    #   fi
1207    #    else
1208    #   echo "ERROR IGCM_sys_Get : demigration error :"
1209    #   cat out_rsync
1210    #   IGCM_debug_Exit "IGCM_sys_Get"
1211    #    fi
1212    #fi
1213
1214    #USUAL WAY
1215    if [ X${1} = X'/l' ] ; then
1216      (( RET=0 ))
1217      for target in ${dm_liste[*]} ; do
1218        local_file=$( basename ${target} )
1219        (( i = 0 ))
1220        # test if the target file is present before the loop
1221        IGCM_sys_TestFileArchive ${target}
1222        RET=$?
1223        if [ ${RET} -gt 0 ] ; then
1224          echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
1225          IGCM_debug_Exit "IGCM_sys_Get"
1226        else
1227          while [ $i -lt $NB_ESSAI ] ; do
1228            \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1229            status=$?
1230            if [ ${status} -gt 0 ]; then
1231              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed ${i}/${NB_ESSAI}"
1232              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1233              sleep $DELAI
1234            else
1235              break
1236            fi
1237            (( i = i + 1 ))
1238          done
1239          (( RET = RET + status ))
1240        fi
1241      done
1242    else
1243      \cp ${dm_liste} ${DEST} >> out_rsync 2>&1
1244      RET=$?
1245    fi
1246
1247#       #RSYNC WITH NETWORK SSH CALL
1248#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1249#       ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1250
1251#       #RSYNC WITH NFS USE
1252#       echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1253#       ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1254
1255#       RET=$?
1256#       IGCM_sys_Rsync_out $RET
1257
1258#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1259#       (( RET=RET+$? ))
1260
1261    if [ ${RET} -gt 0 ] ; then
1262      echo "IGCM_sys_Get : copy error."
1263      cat out_rsync
1264#      IGCM_debug_Exit "IGCM_sys_Get"
1265    else
1266      \rm out_rsync
1267    fi
1268  else
1269    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1270  fi
1271  IGCM_debug_PopStack "IGCM_sys_Get"
1272}
1273
1274#D-#==================================================
1275#D-function IGCM_sys_GetBuffer
1276#D-* Purpose: Get a file from ${SCRATCHDIR}
1277#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1278#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1279function IGCM_sys_GetBuffer {
1280  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1281
1282  typeset DEST RET buf_liste ifile target file_work
1283
1284  if ( $DEBUG_sys ) ; then
1285    echo "IGCM_sys_GetBuffer :" $@
1286  fi
1287  if [ $DRYRUN -le 2 ]; then
1288    if [ X${1} = X'/l' ] ; then
1289      # test if the first file is present in the old computation :
1290      eval set +A buf_liste \${${2}}
1291    else
1292      eval set +A buf_liste ${1}
1293    fi
1294    eval DEST=\${${#}}
1295
1296    #USUAL WAY
1297    if [ X${1} = X'/l' ] ; then
1298      (( RET=0 ))
1299      for target in ${buf_liste[*]} ; do
1300        local_file=$( basename ${target} )
1301        \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1302        (( RET = RET + $? ))
1303      done
1304    else
1305      \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1306      RET=$?
1307    fi
1308
1309    if [ ${RET} -gt 0 ] ; then
1310      echo "IGCM_sys_GetBuffer : copy error."
1311      cat out_rsync
1312      IGCM_debug_Exit "IGCM_sys_GetBuffer"
1313    else
1314      \rm out_rsync
1315    fi
1316  else
1317    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1318  fi
1319  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1320}
1321
1322#D-#==================================================
1323#D-function IGCM_sys_GetDate_FichWork
1324#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1325#D-* Examples:
1326#D-
1327function IGCM_sys_GetDate_FichWork {
1328  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1329  if ( $DEBUG_sys ) ; then
1330    echo "IGCM_sys_GetDate_FichWork :" $@
1331  fi
1332  # donne la date filesys d'un fichier sur la machine work
1333  IGCM_debug_PopStack "IGCM_sys_FichWork"
1334}
1335
1336#D-#==================================================
1337#D-function IGCM_sys_GetDate_FichArchive
1338#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1339#D-* Examples:
1340#D-
1341function IGCM_sys_GetDate_FichArchive {
1342  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1343  if ( $DEBUG_sys ) ; then
1344    echo "IGCM_sys_GetDate_FichArchive :" $@
1345  fi
1346  IGCM_debug_PopStack "IGCM_sys_FichArchive"
1347}
1348
1349#D-#==================================================
1350#D-function IGCM_sys_Dods_Rm
1351#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1352#D-* Examples:
1353#D-
1354function IGCM_sys_Dods_Rm {
1355  if ( $DEBUG_sys ) ; then
1356    echo "IGCM_sys_Dods_Rm :" $@
1357  fi
1358  typeset RET
1359  RET=0
1360  if [ $DRYRUN = 0 ]; then
1361
1362#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
1363#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
1364#      echo "Nothing has been done."
1365#      return
1366#    fi
1367
1368    /ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
1369    RET=$?
1370
1371#       if [ ${RET} -gt 0 ] ; then
1372#           echo "IGCM_sys_Dods_Rm : error."
1373#           cat out_dods_rm
1374#           IGCM_debug_Exit "IGCM_sys_Dods_Rm"
1375#       else
1376#           rm out_dods_rm
1377#       fi
1378
1379  else
1380    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1381  fi
1382  return $RET
1383}
1384
1385#D-#==================================================
1386#D-function IGCM_sys_Dods_Cp
1387#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1388#D-* Examples:
1389#D-
1390function IGCM_sys_Dods_Cp {
1391  if ( $DEBUG_sys ) ; then
1392    echo "IGCM_sys_Dods_Cp :" $@
1393  fi
1394  typeset RET
1395  RET=0
1396  if [ $DRYRUN = 0 ]; then
1397
1398#    if [ ! -d ${R_SAVE}/${1} ] ; then
1399#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1400#      echo "Nothing has been done."
1401#      return
1402#    fi
1403
1404    /ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1405    RET=$?
1406
1407#       if [ ${RET} -gt 0 ] ; then
1408#           echo "IGCM_sys_Dods_Cp : error."
1409#           cat out_dods_cp
1410#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1411#       else
1412#           rm out_dods_cp
1413#       fi
1414
1415  else
1416    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1417  fi
1418  return $RET
1419}
1420
1421#D-#==================================================
1422#D-function IGCM_sys_Put_Dods
1423#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
1424#D-* Examples:
1425#D-
1426function IGCM_sys_Put_Dods {
1427  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1428  if ( $DEBUG_sys ) ; then
1429    echo "IGCM_sys_Put_Dods :" $@
1430  fi
1431  typeset RET
1432  if [ $DRYRUN = 0 ]; then
1433    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1434      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1435      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1436      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1437      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1438      return
1439    fi
1440
1441    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1442      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1443      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1444      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1445      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1446      return
1447    fi
1448    #
1449    if [ -d ${R_SAVE}/${1} ] ; then
1450      cd ${R_SAVE}
1451    elif [ -d ${R_FIGR}/${1} ] ; then
1452      cd ${R_FIGR}
1453    fi
1454
1455    IGCM_sys_Dods_Rm ${1}
1456    IGCM_sys_Dods_Cp ${1}
1457    RET=0
1458
1459    if [ ${RET} -gt 0 ] ; then
1460      echo "IGCM_sys_Put_Dods : error."
1461      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1462    fi
1463  else
1464    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1465  fi
1466  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1467}
1468
1469##############################################################
1470# REBUILD OPERATOR
1471
1472function IGCM_sys_rebuild {
1473  IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1474  if ( $DEBUG_sys ) ; then
1475    echo "IGCM_sys_rebuild :" $@
1476  fi
1477  /home/cont003/p86ipsl/X86_64/bin/rebuild -f -o $@
1478  if [ $? -gt 0 ] ; then
1479    echo "IGCM_sys_rebuild : erreur ${@}."
1480    IGCM_debug_Exit "rebuild"
1481  fi
1482
1483  IGCM_debug_PopStack "IGCM_sys_rebuild"
1484}
1485
1486function IGCM_sys_rebuild_station {
1487  IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
1488  typeset i list_opt file_in file_out prefix_invert list_invert
1489  if ( $DEBUG_sys ) ; then
1490    echo "IGCM_sys_rebuild_station :" $@
1491  fi
1492  list_opt=$@
1493
1494  # Invert Axis : t,x -> x,t
1495  #               t,pres,x -> x,t,pres
1496  # So that we can concatenate along x
1497  i=0
1498  for file_in in ${list_opt} ; do
1499    (( i = i + 1))
1500    [ ${i} = 1 ] && file_out=${file_in} && continue
1501    prefix_invert=$( basename ${file_in} .nc )
1502    IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1503    list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1504  done
1505
1506  # Concatenate
1507  IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1508
1509  # Re-ivert file
1510  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1511
1512  # Station re-ordering is too expansive to be run within libICGM
1513  # This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1514  # This re-ordering must be done "in memory" by the cmorization process
1515  # Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1516  # BEGIN reordering
1517
1518  # Only LMDZ text output contains the exact ordering of the station.
1519  # We isolate this in the code below:
1520  #  0  38  -157.5000000000000  70.98591549295774
1521  #  0  54  27.49999999999999   67.18309859154928
1522  #  0  56  -62.50000000000001  82.39436619718309
1523  #  0  79  12.49999999999999   78.59154929577466
1524  #  0  116 -165.0000000000000  76.05633802816901
1525  #  0  117 130.0000000000000   70.98591549295774
1526  #  0  118 110.0000000000000   87.46478873239437
1527  #  1  40  4.999999999999995   51.97183098591550
1528#  typeset iStation iProc list_opt file_in file_out prefix_invert
1529#  typeset -Z4 j4
1530#  typeset -Z3 j3
1531
1532#  unset list_opt
1533#  set +A list_opt $@
1534
1535  # Filename after rebuild
1536#  file_out=${list_opt[0]}
1537  # Prefix of output files
1538#  prefix_invert=$( basename ${file_out} .nc )
1539  # Number of procs
1540#  num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} | wc -l )
1541
1542#  iProc=0
1543#  while [ ${iProc} -lt ${num_proc} ] ; do
1544    # Array containing Station as a number
1545#    unset proc_stn
1546#    set +A proc_stn $( grep "iophy_mpi rank ip lon lat  $iProc" ${PREFIX}_${Exe_Output} | sed -e "s/iophy_mpi rank ip lon lat //g" | awk ' {print $2}' )
1547    # Number of stations produced by processor proc
1548#    stationLast=${#proc_stn[*]}
1549    # Proc number on 4 digits
1550#    j4=${iProc}
1551    # Init
1552#    iStation=0
1553#    while [ ${iStation} -lt ${stationLast} ] ; do
1554      # Station number on 3 digits
1555#      j3=${proc_stn[${iStation}]}
1556      # Extract station
1557      # Invert Axis : t,x -> x,t
1558      #               t,pres,x -> x,t,pres
1559      # So that we can concatenate along x
1560#      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1561#      (( iStation = iStation + 1 ))
1562#    done
1563#    (( iProc = iProc + 1 ))
1564#  done
1565
1566  # Concatenate all station along x
1567#  IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1568
1569  # Re-invert file
1570#  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1571
1572  # END reordering
1573
1574  IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1575}
1576
1577############################################################
1578# Check of space available on temporary filesytems
1579function IGCM_sys_check_quota {
1580    IGCM_debug_PushStack "IGCM_sys_check_quota"
1581    if ( $DEBUG_sys ) ; then
1582        echo "IGCM_sys_check_quota"
1583    fi
1584    IGCM_debug_PopStack "IGCM_sys_check_quota"
1585}
1586
1587##############################################################
1588# NCO OPERATOR
1589
1590function IGCM_sys_ncap2 {
1591  IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
1592  if ( $DEBUG_sys ) ; then
1593    echo "IGCM_sys_ncap2 :" $@
1594  fi
1595  ncap2 "$@"
1596  if [ $? -gt 0 ] ; then
1597    echo "IGCM_sys_ncap2 : erreur ${@}."
1598    IGCM_debug_Exit "ncap2"
1599  fi
1600
1601  IGCM_debug_PopStack "IGCM_sys_ncap2"
1602}
1603
1604function IGCM_sys_ncatted {
1605  IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
1606  if ( $DEBUG_sys ) ; then
1607    echo "IGCM_sys_ncatted :" $@
1608  fi
1609  ncatted "$@"
1610  if [ $? -gt 0 ] ; then
1611    echo "IGCM_sys_ncatted : erreur ${@}."
1612    IGCM_debug_Exit "ncatted"
1613  fi
1614
1615  IGCM_debug_PopStack "IGCM_sys_ncatted"
1616}
1617
1618function IGCM_sys_ncbo {
1619  IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
1620  if ( $DEBUG_sys ) ; then
1621    echo "IGCM_sys_ncbo :" $@
1622  fi
1623  ncbo $@
1624  if [ $? -gt 0 ] ; then
1625    echo "IGCM_sys_ncbo : erreur ${@}."
1626    IGCM_debug_Exit "ncbo"
1627  fi
1628
1629  IGCM_debug_PopStack "IGCM_sys_ncbo"
1630}
1631
1632function IGCM_sys_ncdiff {
1633  IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
1634  if ( $DEBUG_sys ) ; then
1635    echo "IGCM_sys_ncdiff :" $@
1636  fi
1637  ncdiff $@
1638  if [ $? -gt 0 ] ; then
1639    echo "IGCM_sys_ncdiff : erreur ${@}."
1640    IGCM_debug_Exit "ncdiff"
1641  fi
1642
1643  IGCM_debug_PopStack "IGCM_sys_ncdiff"
1644}
1645
1646function IGCM_sys_ncea {
1647  IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
1648  if ( $DEBUG_sys ) ; then
1649    echo "IGCM_sys_ncea :" $@
1650  fi
1651  ncea $@
1652  if [ $? -gt 0 ] ; then
1653    echo "IGCM_sys_ncea : erreur ${@}."
1654    IGCM_debug_Exit "ncea"
1655  fi
1656
1657  IGCM_debug_PopStack "IGCM_sys_ncea"
1658}
1659
1660function IGCM_sys_ncecat {
1661  IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
1662  if ( $DEBUG_sys ) ; then
1663    echo "IGCM_sys_ncecat :" $@
1664  fi
1665  ncecat $@
1666  if [ $? -gt 0 ] ; then
1667    echo "IGCM_sys_ncecat : erreur ${@}."
1668    IGCM_debug_Exit "ncecat"
1669  fi
1670
1671  IGCM_debug_PopStack "IGCM_sys_ncecat"
1672}
1673
1674function IGCM_sys_ncflint {
1675  IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
1676  if ( $DEBUG_sys ) ; then
1677    echo "IGCM_sys_ncflint :" $@
1678  fi
1679  ncflint $@
1680  if [ $? -gt 0 ] ; then
1681    echo "IGCM_sys_ncflint : erreur ${@}."
1682    IGCM_debug_Exit "ncflint"
1683  fi
1684
1685  IGCM_debug_PopStack "IGCM_sys_ncflint"
1686}
1687
1688function IGCM_sys_ncks {
1689  IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
1690  if ( $DEBUG_sys ) ; then
1691    echo "IGCM_sys_ncks :" $@
1692  fi
1693  ncks $@
1694  if [ $? -gt 0 ] ; then
1695    echo "IGCM_sys_ncks : erreur ${@}."
1696    IGCM_debug_Exit "ncks"
1697  fi
1698
1699  IGCM_debug_PopStack "IGCM_sys_ncks"
1700}
1701
1702function IGCM_sys_ncpdq {
1703  IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
1704  if ( $DEBUG_sys ) ; then
1705    echo "IGCM_sys_ncpdq :" $@
1706  fi
1707  ncpdq $@
1708  if [ $? -gt 0 ] ; then
1709    echo "IGCM_sys_ncpdq : erreur ${@}."
1710    IGCM_debug_Exit "ncpdq"
1711  fi
1712
1713  IGCM_debug_PopStack "IGCM_sys_ncpdq"
1714}
1715
1716function IGCM_sys_ncra {
1717  IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
1718  if ( $DEBUG_sys ) ; then
1719    echo "IGCM_sys_ncra :" $@
1720  fi
1721  ncra $@
1722  if [ $? -gt 0 ] ; then
1723    echo "IGCM_sys_ncra : erreur ${@}."
1724    IGCM_debug_Exit "ncra"
1725  fi
1726
1727  IGCM_debug_PopStack "IGCM_sys_ncra"
1728}
1729
1730function IGCM_sys_ncrcat {
1731  IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
1732  if ( $DEBUG_sys ) ; then
1733    echo "IGCM_sys_ncrcat :" $@
1734  fi
1735  ncrcat $@
1736  if [ $? -gt 0 ] ; then
1737    echo "IGCM_sys_ncrcat : erreur ${@}."
1738#       IGCM_debug_Exit "ncrcat"
1739  fi
1740
1741  IGCM_debug_PopStack "IGCM_sys_ncrcat"
1742}
1743
1744function IGCM_sys_ncrename {
1745  IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
1746  if ( $DEBUG_sys ) ; then
1747    echo "IGCM_sys_ncrename :" $@
1748  fi
1749  ncrename $@
1750  if [ $? -gt 0 ] ; then
1751    echo "IGCM_sys_ncrename : erreur ${@}."
1752    IGCM_debug_Exit "ncrename"
1753  fi
1754
1755  IGCM_debug_PopStack "IGCM_sys_ncrename"
1756}
1757
1758function IGCM_sys_ncwa {
1759  IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
1760  if ( $DEBUG_sys ) ; then
1761    echo "IGCM_sys_ncwa :" $@
1762  fi
1763  ncwa $@
1764  if [ $? -gt 0 ] ; then
1765    echo "IGCM_sys_ncwa : erreur ${@}."
1766    IGCM_debug_Exit "ncwa"
1767  fi
1768
1769  IGCM_debug_PopStack "IGCM_sys_ncwa"
1770}
1771
1772##############################################################
1773# CDO OPERATOR
1774
1775function IGCM_sys_cdo {
1776  IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
1777  if ( $DEBUG_sys ) ; then
1778    echo "IGCM_sys_cdo :" $@
1779  fi
1780  \cdo $@
1781  if [ $? -gt 0 ] ; then
1782    echo "IGCM_sys_cdo : erreur ${@}."
1783    IGCM_debug_PopStack "IGCM_sys_cdo"
1784    return 1
1785  else
1786    IGCM_debug_PopStack "IGCM_sys_cdo"
1787    return 0
1788  fi
1789
1790  IGCM_debug_PopStack "IGCM_sys_cdo"
1791}
1792
1793############################################################
1794# Activate Running Environnment Variables
1795
1796function IGCM_sys_activ_variables {
1797  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1798  if ( $DEBUG_sys ) ; then
1799    echo "IGCM_sys_activ_variables"
1800  fi
1801  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1802}
1803
1804############################################################
1805# Desactivate Running Environnment Variables
1806
1807function IGCM_sys_desactiv_variables {
1808  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1809  if ( $DEBUG_sys ) ; then
1810    echo "IGCM_sys_desactiv_variables"
1811  fi
1812  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1813}
1814
1815############################################################
1816# Build run file
1817
1818function IGCM_sys_build_run_file {
1819  IGCM_debug_PushStack "IGCM_sys_build_run_file"
1820  if ( $DEBUG_sys ) ; then
1821    echo "IGCM_sys_build_run_file"
1822  fi
1823  IGCM_debug_PopStack "IGCM_sys_build_run_file"
1824}
Note: See TracBrowser for help on using the repository browser.