source: tags/libIGCM_v2.0_beta4/libIGCM_sys/libIGCM_sys_cesium.ksh @ 1636

Last change on this file since 1636 was 697, checked in by sdipsl, 12 years ago
  • Add IGCM_sys_rebuild_station to all machines. Function dedicated to rebuild CFMIP file station. Regular rebuild can't do it.
  • Property svn:keywords set to Revision Author Date
File size: 50.4 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Cesium
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe param files |            |  Chmod  |                           |
46# |          |      Qsub           |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset  HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset  LOGIN=${LOGIN:=$( whoami )}
69
70#D-
71#D-#==================================================
72#D-Program used in libIGCM
73#D-#==================================================
74
75# rsync with path
76typeset -r RSYNC=/usr/bin/rsync
77# RSYNC_opt args to rsync
78typeset -r RSYNC_opt="-va"
79# ie storage filesystem
80typeset -r RHOST=cesium
81
82#====================================================
83# Source default environment
84#====================================================
85. /etc/profile
86
87#====================================================
88# Set environment tools (ferret, nco, cdo)
89#====================================================
90. /home/cont003/p86ipsl/.atlas_env_cesium_ksh
91
92#====================================================
93# Specific for ocean additionnal diagnostic
94export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
95export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
96
97#====================================================
98# Host specific DIRECTORIES
99#====================================================
100
101#====================================================
102#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
103typeset -r R_EXE="${MODIPSL}/bin"
104
105#====================================================
106# For interactive jobs on cesium
107PBS_O_WORKDIR=${PBS_O_WORKDIR:=$(pwd)}
108
109#====================================================
110#- SUBMIT_DIR : submission dir
111typeset SUBMIT_DIR=${SUBMIT_DIR:=${PBS_O_WORKDIR}}
112
113#====================================================
114#- ARCHIVE (dedicated to large files)
115typeset -r ARCHIVE=${CCCSTOREDIR}
116
117#- ARCHIVE (dedicated to small/medium files)
118typeset -r STORAGE=${CCCWORKDIR}
119
120#====================================================
121#- IN
122typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
123typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/dmnfs/cont003/p24data}
124
125#====================================================
126#- R_OUT
127typeset -r R_OUT=${ARCHIVE}/IGCM_OUT
128
129#====================================================
130#- R_FIG (hosting figures : monitoring and atlas, and/or small files)
131typeset -r R_FIG=${STORAGE}/IGCM_OUT
132
133#====================================================
134#- R_BUF (ONLY FOR double copy an scratch)
135typeset -r R_BUF=${SCRATCHDIR}/IGCM_OUT
136
137#====================================================
138#- OUT_POST
139typeset -r R_OUT_POST=${SCRATCHDIR}/IGCM_OUT
140
141#====================================================
142#- RUN_DIR_PATH : Temporary working directory (=> TMP)
143PBS_JOBID=${PBS_JOBID:=TEMPDIR$$}
144typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/${PBS_JOBID}}
145
146#====================================================
147#- Max number of arguments passed to nco operator or demigration command
148UNIX_MAX_LIMIT=360
149
150#- set PackDefault true on NEC front-end
151PackDefault=true
152
153#D-#==================================================
154#D-function IGCM_sys_RshMaster
155#D-* Purpose: Just a fake command to wrapp
156#D-           IGCM_card call in post-treatment
157#D-           Ulam do not see brodie filesystem
158#D-           Cesium do not see all mercure filesystem
159#D-           That's why we need this hack.
160#D-* Examples:
161#D-
162function IGCM_sys_RshMaster {
163  IGCM_debug_PushStack "IGCM_sys_RshMaster" $@
164  typeset NB_ESSAI DELAI status i
165  # number of tentative
166  NB_ESSAI=10
167  # time delay between tentative
168  DELAI=10
169  #
170  (( i = 0 ))
171  while [ $i -lt $NB_ESSAI ] ; do
172    ssh ${MASTER} exec /bin/ksh <<-EOF
173    export libIGCM=${libIGCM_SX}
174    export DEBUG_debug=${DEBUG_debug}
175    . ${libIGCM_SX}/libIGCM_debug/libIGCM_debug.ksh
176    . ${libIGCM_SX}/libIGCM_card/libIGCM_card.ksh
177    ${@}
178EOF
179    status=$?
180    if [ ${status} -gt 0 ]; then
181      IGCM_debug_Print 2 "IGCM_sys_RshMaster : ssh failed ${i}/${NB_ESSAI}"
182      IGCM_debug_Print 2 "IGCM_sys_RshMaster : sleep ${DELAI} seconds and try again."
183      sleep $DELAI
184    else
185      break
186    fi
187    (( i = i + 1 ))
188  done
189
190  IGCM_debug_PopStack "IGCM_sys_RshMaster"
191}
192
193#D-#==================================================
194#D-function IGCM_sys_RshArchive
195#D-* Purpose: Archive rsh command
196#D-* Examples:
197#D-
198function IGCM_sys_RshArchive {
199  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
200  /bin/ksh <<-EOF
201    ${@}
202EOF
203  if [ $? -gt 0 ] ; then
204    echo "IGCM_sys_RshArchive : erreur."
205    IGCM_debug_Exit "IGCM_sys_RshArchive"
206  fi
207  IGCM_debug_PopStack "IGCM_sys_RshArchive"
208}
209
210#D-#==================================================
211#D-function IGCM_sys_RshPost
212#D-* Purpose: Post-process rsh command
213#D-* Examples:
214#D-
215function IGCM_sys_RshPost {
216  IGCM_debug_PushStack "IGCM_sys_RshPost" $@
217  if ( $DEBUG_sys ) ; then
218    echo "IGCM_sys_RshPost :" $@
219  fi
220
221  #echo cat tmp_IGCM_sys_RshPost_$$ INITIAL
222  #cat tmp_IGCM_sys_RshPost_$$
223  # keep standard input (stdin) for the loop onto temporary file
224  cat >tmp_IGCM_sys_RshPost_$$
225
226  /bin/ksh <tmp_IGCM_sys_RshPost_$$
227  if [ $? -gt 0 ] ; then
228    echo "IGCM_sys_RshPost : erreur."
229    IGCM_debug_Exit "IGCM_sys_RshPost"
230    cat tmp_IGCM_sys_RshPost_$$
231  fi
232  # delete temporary file
233  \rm tmp_IGCM_sys_RshPost_$$
234
235  IGCM_debug_PopStack "IGCM_sys_RshPost"
236}
237
238#D-#==================================================
239#D-function IGCM_sys_SendMail
240#D-* Purpose: Send mail when simulation is over
241#D-* Examples:
242#D-
243function IGCM_sys_SendMail {
244  IGCM_debug_PushStack "IGCM_sys_SendMail" $@
245  if ( $DEBUG_sys ) ; then
246    echo "IGCM_sys_SendMail :" $@
247  fi
248
249  if ( ${ExitFlag} ) ; then
250    status=failed
251  else
252    status=completed
253  fi
254
255  ssh mercure <<-EOF
256    export LOGIN=${LOGIN}
257    export config_UserChoices_JobName=${config_UserChoices_JobName}
258    export config_UserChoices_MailName=${config_UserChoices_MailName}
259    export DateBegin=${DateBegin}
260    export DateEnd=${DateEnd}
261    export R_SAVE=${R_SAVE}
262    export SUBMIT_DIR=${SUBMIT_DIR}
263    export status=${status}
264
265    cat  << END_MAIL > job_end.mail
266Dear ${LOGIN},
267
268  Simulation ${config_UserChoices_JobName} is ${status} on supercomputer `hostname`.
269  Job started : ${DateBegin}
270  Job ended   : ${DateEnd}
271  Output files are available in ${R_SAVE}
272  Script files, Script Outputs and Debug files (if necessary) are available in ${SUBMIT_DIR}
273END_MAIL
274
275    if [ ! -z ${config_UserChoices_MailName} ] ; then
276       mailx -s "${config_UserChoices_JobName} ${status}" ${config_UserChoices_MailName} <  job_end.mail
277    elif [ -f ~/.forward ] ; then
278       mailx -s "${config_UserChoices_JobName} ${status}" $( cat ~/.forward ) < job_end.mail
279    fi
280    sleep 10
281    rm -f job_end.mail
282EOF
283
284  if [ $? -gt 0 ] ; then
285    echo "IGCM_sys_SendMail : erreur."
286    IGCM_debug_Exit "IGCM_sys_SendMail"
287  fi
288  IGCM_debug_PopStack "IGCM_sys_SendMail"
289}
290
291#D-#==================================================
292#D-function IGCM_sys_Mkdir
293#D-* Purpose: Master locale mkdir command
294#D-* Examples:
295#D-
296function IGCM_sys_Mkdir {
297  IGCM_debug_PushStack "IGCM_sys_Mkdir" $@
298  if ( $DEBUG_sys ) ; then
299    echo "IGCM_sys_Mkdir :" $@
300  fi
301  if [ ! -d ${1} ]; then
302    \mkdir -p $1
303    if [ $? -gt 0 ] ; then
304      echo "IGCM_sys_Mkdir : erreur."
305      IGCM_debug_Exit "IGCM_sys_Mkdir"
306    fi
307  fi
308  # vérification :
309  if [ ! -d ${1} ] ; then
310    echo "IGCM_sys_Mkdir : erreur."
311    IGCM_debug_Exit "IGCM_sys_Mkdir"
312  fi
313  IGCM_debug_PopStack "IGCM_sys_Mkdir"
314}
315
316#D-#==================================================
317#D-function IGCM_sys_MkdirArchive
318#D-* Purpose: Mkdir on Archive
319#D-* Examples:
320#D-
321function IGCM_sys_MkdirArchive {
322  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
323  if ( $DEBUG_sys ) ; then
324    echo "IGCM_sys_MkdirArchive :" $@
325  fi
326  #- creation de repertoire sur le serveur fichier
327  if [ ! -d ${1} ]; then
328    \mkdir -p $1
329    if [ $? -gt 0 ] ; then
330      echo "IGCM_sys_MkdirArchive : erreur."
331      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
332    fi
333  fi
334  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
335}
336
337#D-#==================================================
338#D-function IGCM_sys_MkdirWork
339#D-* Purpose: Mkdir on Work
340#D-* Examples:
341#D-
342function IGCM_sys_MkdirWork {
343  IGCM_debug_PushStack "IGCM_sys_MkdirWork" $@
344  if ( $DEBUG_sys ) ; then
345    echo "IGCM_sys_MkdirWork :" $@
346  fi
347  #- creation de repertoire sur le serveur fichier
348  if [ ! -d ${1} ]; then
349    \mkdir -p $1
350    if [ $? -gt 0 ] ; then
351      echo "IGCM_sys_MkdirWork : erreur."
352      IGCM_debug_Exit "IGCM_sys_MkdirWork"
353    fi
354  fi
355  IGCM_debug_PopStack "IGCM_sys_MkdirWork"
356}
357
358#D-#==================================================
359#D-function IGCM_sys_Cd
360#D-* Purpose: master cd command
361#D-* Examples:
362#D-
363function IGCM_sys_Cd {
364  IGCM_debug_PushStack "IGCM_sys_Cd" $@
365  if ( $DEBUG_sys ) ; then
366    echo "IGCM_sys_Cd :" $@
367  fi
368  \cd $1
369  if [ $? -gt 0 ] ; then
370    echo "IGCM_sys_Cd : erreur."
371    IGCM_debug_Exit "IGCM_sys_Cd"
372  fi
373  IGCM_debug_PopStack "IGCM_sys_Cd"
374}
375
376#D-#==================================================
377#D-function IGCM_sys_Chmod
378#D-* Purpose: Chmod
379#D-* Examples:
380#D-
381function IGCM_sys_Chmod {
382  IGCM_debug_PushStack "IGCM_sys_Chmod" -- $@
383  if ( $DEBUG_sys ) ; then
384    echo "IGCM_sys_Chmod :" $@
385  fi
386  if [ $DRYRUN -le 1 ]; then
387    \chmod $@
388    if [ $? -gt 0 ] ; then
389      echo "IGCM_sys_Chmod : erreur."
390      IGCM_debug_Exit "IGCM_sys_Chmod"
391    fi
392  else
393    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
394  fi
395  IGCM_debug_PopStack "IGCM_sys_Chmod"
396}
397
398#D-#==================================================
399#D-function IGCM_sys_FileSize
400#D-* Purpose: Filesize
401#D-* Examples:
402#D-
403function IGCM_sys_FileSize {
404  IGCM_debug_PushStack "IGCM_sys_FileSize" $@
405
406  typeset sizeF
407  set +A sizeF -- $( ls -la ${1} )
408  if [ $? -gt 0 ] ; then
409    IGCM_debug_Exit "IGCM_sys_FileSize"
410  fi
411  eval ${2}=${sizeF[4]}
412
413  IGCM_debug_PopStack "IGCM_sys_FileSize"
414}
415
416#D-#==================================================
417#D-function IGCM_sys_TestDir
418#D-* Purpose: Test Directory that must exists
419#D-* Examples:
420#D-
421function IGCM_sys_TestDir {
422  IGCM_debug_PushStack "IGCM_sys_TestDir" $@
423  if ( $DEBUG_sys ) ; then
424    echo "IGCM_sys_TestDir :" $@
425  fi
426  typeset ExistFlag
427  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
428  IGCM_debug_PopStack "IGCM_sys_TestDir"
429
430  return ${ExistFlag}
431}
432
433#D-#==================================================
434#D-function IGCM_sys_TestDirArchive
435#D-* Purpose: Test Directory that must exists on Archive
436#D-* Examples:
437#D-
438function IGCM_sys_TestDirArchive {
439  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
440  if ( $DEBUG_sys ) ; then
441    echo "IGCM_sys_TestDirArchive :" $@
442  fi
443  typeset ExistFlag
444  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
445  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
446
447  return ${ExistFlag}
448}
449
450#D-#==================================================
451#D-function IGCM_sys_IsFileArchived
452#D-* Purpose: Test file that must NOT EXISTS on Archive
453#D-* Examples:
454#D-
455function IGCM_sys_IsFileArchived {
456  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
457  if ( $DEBUG_sys ) ; then
458    echo "IGCM_sys_IsFileArchived :" $@
459  fi
460  typeset IsArchivedFlag
461  IsArchivedFlag=$( [ X$( echo $1 | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
462  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
463
464  return ${IsArchivedFlag}
465}
466
467#D-#==================================================
468#D-function IGCM_sys_TestFileArchive
469#D-* Purpose: Test file that must NOT EXISTS on Archive
470#D-* Examples:
471#D-
472function IGCM_sys_TestFileArchive {
473  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
474  typeset ExistFlag
475  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
476  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
477
478  return ${ExistFlag}
479}
480
481#D-#==================================================
482#D-function IGCM_sys_TestFileBuffer
483#D-* Purpose: Test file that must NOT EXISTS on Buffer
484#D-* Examples:
485#D-
486function IGCM_sys_TestFileBuffer {
487  IGCM_debug_PushStack "IGCM_sys_TestFileBuffer" $@
488  typeset ExistFlag
489  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
490  IGCM_debug_PopStack "IGCM_sys_TestFileBuffer"
491
492  return ${ExistFlag}
493}
494
495#D-#==================================================
496#D-function IGCM_sys_CountFileArchive
497#D-* Purpose: Count files on Archive filesystem
498#D-* Examples:
499#D-
500function IGCM_sys_CountFileArchive {
501  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
502  ls ${@} 2>/dev/null | wc -l
503  if [ $? -gt 0 ] ; then
504    echo "IGCM_sys_CountFileArchive : erreur."
505  fi
506  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
507}
508
509#D-#==================================================
510#D-function IGCM_sys_CountFileBuffer
511#D-* Purpose: Count files on Scratch filesystem
512#D-* Examples:
513#D-
514function IGCM_sys_CountFileBuffer {
515  IGCM_debug_PushStack "IGCM_sys_CountFileBuffer" $@
516  ls ${@} 2>/dev/null | wc -l
517  if [ $? -gt 0 ] ; then
518    echo "IGCM_sys_CountFileBuffer : erreur."
519  fi
520  IGCM_debug_PopStack "IGCM_sys_CountFileBuffer"
521}
522
523#D-#==================================================
524#D-function IGCM_sys_Tree
525#D-* Purpose: Tree directories with files on ${ARCHIVE}
526#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
527#D-
528function IGCM_sys_Tree {
529  IGCM_debug_PushStack "IGCM_sys_Tree" $@
530  if ( $DEBUG_sys ) ; then
531    echo "IGCM_sys_Tree :" $@
532  fi
533
534  \tree -f $@
535
536  IGCM_debug_PopStack "IGCM_sys_Tree"
537}
538
539#D-#==================================================
540#D-function IGCM_sys_Tar
541#D-* Purpose: master tar command
542#D-* Examples:
543#D-
544function IGCM_sys_Tar {
545  IGCM_debug_PushStack "IGCM_sys_Tar" $@
546  if ( $DEBUG_sys ) ; then
547    echo "IGCM_sys_Tar :" $@
548  fi
549  \tar cf $@
550  if [ $? -gt 0 ] ; then
551    echo "IGCM_sys_Tar : erreur."
552    IGCM_debug_Exit "IGCM_sys_Tar"
553  fi
554  IGCM_debug_PopStack "IGCM_sys_Tar"
555}
556
557#D-#==================================================
558#D-function IGCM_sys_UnTar
559#D-* Purpose: master un-tar command
560#D-* Examples:
561#D-
562function IGCM_sys_UnTar {
563  IGCM_debug_PushStack "IGCM_sys_UnTar" $@
564  if ( $DEBUG_sys ) ; then
565    echo "IGCM_sys_UnTar :" $@
566  fi
567  \tar xvf $1
568  if [ $? -gt 0 ] ; then
569    echo "IGCM_sys_UnTar : erreur."
570    IGCM_debug_Exit "IGCM_sys_UnTar"
571  fi
572  IGCM_debug_PopStack "IGCM_sys_UnTar"
573}
574
575#D-#==================================================
576#D-function IGCM_sys_Qsub
577#D-* Purpose: Qsub new job
578#D-* Examples:
579#D-
580function IGCM_sys_Qsub {
581  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
582  if ( $DEBUG_sys ) ; then
583    echo "IGCM_sys_Qsub :" $@
584  fi
585  /usr/local/bin/ccc_msub -o ${Script_Output} -e ${Script_Output}.e < $1
586  if [ $? -gt 0 ] ; then
587    echo "IGCM_sys_Qsub : erreur  -o ${Script_Output} -e ${Script_Output}.e $@"
588    IGCM_debug_Exit "IGCM_sys_Qsub"
589  fi
590  IGCM_debug_PopStack "IGCM_sys_Qsub"
591}
592
593#D-#==================================================
594#D-function IGCM_sys_QsubPost
595#D-* Purpose: Qsub new job on scalaire
596#D-* Examples:
597#D-
598function IGCM_sys_QsubPost {
599  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
600  if ( $DEBUG_sys ) ; then
601    echo "IGCM_sys_QsubPost :" $@
602  fi
603  /usr/local/bin/ccc_msub -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.e.out -E "-v ${listVarEnv}" ${libIGCM}/$1.job
604  if [ $? -gt 0 ] ; then
605    echo "IGCM_sys_QsubPost : erreur " $@
606    IGCM_debug_Exit "IGCM_sys_QsubPost"
607  fi
608  IGCM_debug_PopStack "IGCM_sys_QsubPost"
609}
610
611#D-*************************
612#D- File transfer functions
613#D-*************************
614#D-
615
616#D-#==================================================
617#D-function IGCM_sys_Rsync_out
618#D-* Purpose: treat return val of rsync
619#D-* Examples: IGCM_sys_Rsync_out out_RET_rsync
620#D-  Error values and explanations can depend on your system version.
621function IGCM_sys_Rsync_out {
622  RET=$1
623  if [ ! $RET ] ; then
624    echo "rsync error !"
625  fi
626
627  if [ $MYLANG = "fr" ]; then
628    case $RET in
629    0)  return ;;
630    1)  echo "Erreur de rsync ; RERR_SYNTAX : "
631      echo "Erreur de syntaxe ou d'utilisation."
632      return;;
633    2)  echo "Erreur de rsync ; RERR_PROTOCOL : "
634      echo "Incompatibilité de protocole."
635      return;;
636    3)  echo "Erreur de rsync ; RERR_FILESELECT 3"
637      echo "Erreurs  lors  de  la  sélection des fichiers d'entrée sortie et"
638      echo "répertoires"
639      return;;
640    4)  echo "Erreur de rsync ; RERR_UNSUPPORTED"
641      echo "Action demandée non supportée : une tentative de manipulation de"
642      echo "fichiers  64-bits  sur une plate-forme qui ne les supporte pas a"
643      echo "été faite ; ou une option qui est supportée par le  client  mais"
644      echo "pas par le serveur a été spécifiée."
645      return;;
646    10) echo "Erreur de rsync ; RERR_SOCKETIO"
647      echo "Erreur dans le socket d'entrée sortie"
648      return;;
649    11) echo "Erreur de rsync ; RERR_FILEIO"
650      echo "Erreur d'entrée sortie fichier"
651      return;;
652    12) echo "Erreur de rsync ; RERR_STREAMIO"
653      echo "Erreur dans flux de donnée du protocole rsync"
654      return;;
655    13) echo "Erreur de rsync ; RERR_MESSAGEIO"
656      echo "Erreur avec les diagnostics du programme"
657      return;;
658    14) echo "Erreur de rsync ; RERR_IPC"
659      echo "Erreur dans le code IPC"
660      return;;
661    20) echo "Erreur de rsync ; RERR_SIGNAL"
662      echo "SIGUSR1 ou SIGINT reçu"
663      return;;
664    21) echo "Erreur de rsync ; RERR_WAITCHILD"
665      echo "Une erreur retournée par waitpid()"
666      return;;
667    22) echo "Erreur de rsync ; RERR_MALLOC"
668      echo "Erreur lors de l'allocation des tampons de mémoire de coeur"
669      return;;
670    23) echo ""
671      echo "Erreur fichier inexistant"
672      return;;
673    30) echo "Erreur de rsync ; RERR_TIMEOUT"
674      echo "Temps d'attente écoulé dans l'envoi/réception de données"
675      return;;
676    *)  echo "Erreur de rsync : code de retour de rsync inconnu :" $RET
677      return;;
678    esac
679  elif [ $MYLANG = "en" ] ; then
680    case $RET in
681    0)  return;;
682    1)  echo "rsync error : Syntax or usage error "
683      return;;
684    2)  echo "rsync error : Protocol incompatibility "
685      return;;
686    3)  echo "rsync error : Errors selecting input/output files, dirs"
687      return;;
688    4)  echo "rsync error : Requested action not supported: an attempt"
689      echo "was made to manipulate 64-bit files on a platform that cannot support"
690      echo "them; or an option was specified that is supported by the client and"
691      echo "not by the server."
692      return;;
693    5)  echo "rsync error : Error starting client-server protocol"
694      return;;
695    10) echo "rsync error : Error in socket I/O "
696      return;;
697    11) echo "rsync error : Error in file I/O "
698      return;;
699    12) echo "rsync error : Error in rsync protocol data stream "
700      return;;
701    13) echo "rsync error : Errors with program diagnostics "
702      return;;
703    14) echo "rsync error : Error in IPC code "
704      return;;
705    20) echo "rsync error : Received SIGUSR1 or SIGINT "
706      return;;
707    21) echo "rsync error : Some error returned by waitpid() "
708      return;;
709    22) echo "rsync error : Error allocating core memory buffers "
710      return;;
711    23) echo "rsync error : Partial transfer due to error"
712      return;;
713    24) echo "rsync error : Partial transfer due to vanished source files"
714      return;;
715    30) echo "rsync error : Timeout in data send/receive "
716      return;;
717    *)  echo "rsync error : return code of rsync unknown :" $RET
718      return;;
719    esac
720  else
721    echo "unknown language $MYLANG."
722    return
723  fi
724}
725
726#D-#==================================================
727#D-function IGCM_sys_Cp
728#D-* Purpose: generic cp
729#D-* Examples:
730#D-
731function IGCM_sys_Cp {
732  IGCM_debug_PushStack "IGCM_sys_Cp" $@
733  if ( $DEBUG_sys ) ; then
734    echo "IGCM_sys_Cp :" $@
735  fi
736
737  typeset RET
738
739  echo cp $@ > out_rsync 2>&1
740  \cp $@ >> out_rsync 2>&1
741  RET=$?
742
743  if [ ${RET} -gt 0 ] ; then
744    echo "IGCM_sys_Cp : error."
745    cat out_rsync
746    IGCM_debug_Exit "IGCM_sys_Cp"
747  else
748    \rm out_rsync
749  fi
750  IGCM_debug_PopStack "IGCM_sys_Cp"
751}
752
753#D-#==================================================
754#D-function IGCM_sys_Rm
755#D-* Purpose: generic rm
756#D-* Examples:
757#D-
758function IGCM_sys_Rm {
759  IGCM_debug_PushStack "IGCM_sys_Rm" -- $@
760  if ( $DEBUG_sys ) ; then
761    echo "IGCM_sys_Rm :" $@
762  fi
763
764  typeset RET
765
766  echo rm $@ > out_rsync 2>&1
767  \rm $@ >> out_rsync 2>&1
768  RET=$?
769
770  if [ ${RET} -gt 0 ] ; then
771    echo "IGCM_sys_Rm : error."
772    cat out_rsync
773    IGCM_debug_Exit "IGCM_sys_Rm"
774  else
775    \rm out_rsync
776  fi
777  IGCM_debug_PopStack "IGCM_sys_Rm"
778}
779
780#D-#==================================================
781#D-function IGCM_sys_RmRunDir
782#D-* Purpose: rm tmpdir (dummy function most of the time batch
783#D-                      scheduler will do the job)
784#D-* Examples:
785#D-
786function IGCM_sys_RmRunDir {
787  IGCM_debug_PushStack "IGCM_sys_RmRunDir" -- $@
788  if ( $DEBUG_sys ) ; then
789    echo "IGCM_sys_RmRunDir :" $@
790  fi
791
792  typeset RET
793
794  echo rm $@ > out_rsync 2>&1
795  \rm $@ >> out_rsync 2>&1
796  RET=$?
797
798  if [ ${RET} -gt 0 ] ; then
799    echo "IGCM_sys_RmRunDir : error."
800    cat out_rsync
801    IGCM_debug_Exit "IGCM_sys_RmRunDir"
802  else
803    \rm out_rsync
804  fi
805  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
806}
807
808#D-#==================================================
809#D-function IGCM_sys_Mv
810#D-* Purpose: generic move
811#D-* Examples:
812#D-
813function IGCM_sys_Mv {
814  IGCM_debug_PushStack "IGCM_sys_Mv" $@
815  if ( $DEBUG_sys ) ; then
816    echo "IGCM_sys_Mv :" $@
817  fi
818
819  if [ $DRYRUN = 0 ]; then
820
821    typeset RET
822
823    echo mv $@ > out_rsync 2>&1
824    \mv $@ >> out_rsync 2>&1
825    RET=$?
826
827    if [ ${RET} -gt 0 ] ; then
828      echo "IGCM_sys_Mv : error in mv."
829      cat out_rsync
830      IGCM_debug_Exit "IGCM_sys_Mv"
831    else
832      \rm out_rsync
833    fi
834  else
835    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
836  fi
837
838  IGCM_debug_PopStack "IGCM_sys_Mv"
839}
840
841#D-#==================================================
842#D-function IGCM_sys_Put_Dir
843#D-* Purpose: Copy a complete directory on $(ARCHIVE)
844#D-* Examples:
845#D-
846function IGCM_sys_Put_Dir {
847  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
848  if ( $DEBUG_sys ) ; then
849    echo "IGCM_sys_Put_Dir :" $@
850  fi
851  if [ $DRYRUN = 0 ]; then
852    if [ ! -d ${1} ] ; then
853      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
854      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
855      return
856    fi
857
858    typeset RET
859
860    # Only if we use rsync
861    #IGCM_sys_TestDirArchive $( dirname $2 )
862    #
863    #USUAL WAY
864    \cp -r $1 $2 > out_rsync 2>&1
865    RET=$?
866
867    if [ ${RET} -gt 0 ] ; then
868      echo "IGCM_sys_Put_Dir : error."
869      cat out_rsync
870      IGCM_debug_Exit "IGCM_sys_Put_Dir"
871    else
872      \rm out_rsync
873    fi
874  else
875    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
876  fi
877  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
878}
879
880#D-#==================================================
881#D-function IGCM_sys_Get_Dir
882#D-* Purpose: Copy a complete directory from ${ARCHIVE}
883#D-* Examples:
884#D-
885function IGCM_sys_Get_Dir {
886  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
887  if ( $DEBUG_sys ) ; then
888    echo "IGCM_sys_Get_Dir :" $@
889  fi
890  if [ $DRYRUN = 0 ]; then
891#    if [ ! -d ${1} ] ; then
892#      echo "WARNING : IGCM_sys_Get_Dir ${1} DOES NOT EXIST ."
893#      IGCM_debug_PopStack "IGCM_sys_Get_Dir"
894#      return
895#    fi
896
897    typeset RET
898
899    # Only if we use rsync
900    #IGCM_sys_TestDirArchive $( dirname $2 )
901    #
902    # USUAL WAY
903    # add dmfind/dmget (to demigrate all offline files) :
904    #dmfind $1 -state MIG -o -state OFL -o -state PAR | dmget
905    \cp -r $1 $2 > out_rsync 2>&1
906    RET=$?
907
908    if [ ${RET} -gt 0 ] ; then
909      echo "IGCM_sys_Get_Dir : error."
910      cat out_rsync
911      IGCM_debug_Exit "IGCM_sys_Get_Dir"
912    else
913      \rm out_rsync
914    fi
915  else
916    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
917  fi
918  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
919}
920
921#D-#==================================================
922#D-function IGCM_sys_Get_Master
923#D-* Purpose: Copy a complete directory from MASTER filesystem
924#D-* Examples:
925#D-
926function IGCM_sys_Get_Master {
927  IGCM_debug_PushStack "IGCM_sys_Get_Master" $@
928  if ( $DEBUG_sys ) ; then
929    echo "IGCM_sys_Get_Master :" $@
930  fi
931  if [ $DRYRUN = 0 ]; then
932
933    typeset TEST NB_ESSAI DELAI status i
934
935    TEST=$( IGCM_sys_RshMaster ( [ -d $1 ] || [ -f $1 ] ) && echo 1 || echo 0 )
936    if [ ${TEST} -ne 1 ] ; then
937      echo "WARNING : IGCM_sys_Get_Master ${1} DOES NOT EXIST ON ${MASTER}."
938      IGCM_debug_PopStack "IGCM_sys_Get_Master"
939      return
940    fi
941
942    # number of tentative
943    NB_ESSAI=10
944    # time delay between tentative
945    DELAI=10
946    #
947    (( i = 0 ))
948    while [ $i -lt $NB_ESSAI ] ; do
949            #USUAL WAY
950      scp -r ${MASTER}:$1 $2 > out_rsync 2>&1
951      status=$?
952      if [ ${status} -gt 0 ]; then
953        IGCM_debug_Print 2 "IGCM_sys_Get_Master : scp failed ${i}/${NB_ESSAI}"
954        IGCM_debug_Print 2 "IGCM_sys_Get_Master : sleep ${DELAI} seconds and try again."
955        sleep $DELAI
956      else
957        break
958      fi
959      (( i = i + 1 ))
960    done
961
962    if [ ${status} -gt 0 ] ; then
963      echo "IGCM_sys_Get_Master : error."
964      cat out_rsync
965      IGCM_debug_Exit "IGCM_sys_Get_Master"
966    else
967      \rm out_rsync
968    fi
969  else
970    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
971  fi
972  IGCM_debug_PopStack "IGCM_sys_Get_Master"
973}
974
975#D-#==================================================
976#D-function IGCM_sys_Put_Rest
977#D-* Purpose: Put computied restarts on ${ARCHIVE}.
978#D-           File and target directory must exist.
979#D-* Examples:
980#D-
981function IGCM_sys_Put_Rest {
982  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
983  if ( $DEBUG_sys ) ; then
984    echo "IGCM_sys_Put_Rest :" $@
985  fi
986  if [ $DRYRUN = 0 ]; then
987    if [ ! -f ${1} ] ; then
988      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
989      IGCM_debug_Exit "IGCM_sys_Put_Rest"
990    fi
991
992    typeset RET
993    #
994    if [ X${JobType} = XRUN ] ; then
995      IGCM_sys_Chmod 444 ${1}
996    fi
997
998    #
999    # USUAL WAY
1000    \cp $1 $2 > out_rsync 2>&1
1001    RET=$?
1002
1003#       #RSYNC WITH NETWORK SSH CALL
1004#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > out_rsync 2>&1
1005#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> out_rsync 2>&1
1006
1007#       #RSYNC WITH NFS USE
1008#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1009#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1010
1011#       RET=$?
1012#       IGCM_sys_Rsync_out $RET
1013
1014#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1015#       (( RET=RET+$? ))
1016
1017    if [ ${RET} -gt 0 ] ; then
1018      echo "IGCM_sys_Put_Rest : error."
1019      cat out_rsync
1020      IGCM_debug_Exit "IGCM_sys_Put_Rest"
1021    else
1022      \rm out_rsync
1023    fi
1024  else
1025    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1026  fi
1027  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
1028}
1029
1030#D-#==================================================
1031#D-function IGCM_sys_Put_Out
1032#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
1033#D-* Examples:
1034#D-
1035function IGCM_sys_Put_Out {
1036  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
1037  if ( $DEBUG_sys ) ; then
1038    echo "IGCM_sys_Put_Out :" $@
1039  fi
1040
1041  typeset NB_ESSAI DELAI RET i exist skip
1042
1043  # number of tentative
1044  NB_ESSAI=3
1045  # time delay between tentative
1046  DELAI=2
1047
1048  if [ $DRYRUN = 0 ]; then
1049    if [ ! -f ${1} ] ; then
1050      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
1051      IGCM_debug_PopStack "IGCM_sys_Put_Out"
1052      return 1
1053    fi
1054    #
1055    IGCM_sys_MkdirArchive $( dirname $2 )
1056    #
1057
1058    #=====================================================
1059    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1060    #=====================================================
1061
1062    #echo ${2} | grep "${R_OUT}" > /dev/null 2>&1
1063    #if [ $? -eq 0 ] ; then
1064    #    typeset WORKPATH FILEPATH
1065    #    WORKPATH=$( dirname $2 | sed -e "s|${R_OUT}|${R_BUF}|" )
1066    #    IGCM_sys_MkdirWork ${WORKPATH}
1067    #    FILEPATH=${WORKPATH}/$( basename $2 )
1068    #    #
1069    #    IGCM_sys_Cp ${1} ${FILEPATH}
1070    #fi
1071
1072    if [ X${JobType} = XRUN ] ; then
1073      if [ X${3} = X ] ; then
1074        IGCM_sys_Chmod 444 ${1}
1075      fi
1076    fi
1077    #
1078
1079    exist=false
1080    skip=false
1081    if [ -f $2 ] ; then
1082      IGCM_debug_Print 1 "$2 already exist"
1083      #dmget $2
1084      ccc_hsm get $2
1085      exist=true
1086      if [ "X$( diff $1 $2 )" = X ] ; then
1087        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
1088        skip=true
1089      else
1090        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
1091        skip=false
1092      fi
1093    fi
1094    #
1095    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
1096      IGCM_sys_Chmod u+w $2
1097    fi
1098
1099    if [ X${skip} = Xfalse ] ; then
1100      (( i = 0 ))
1101      while [ $i -lt $NB_ESSAI ] ; do
1102        # USUAL WAY
1103        \cp $1 $2 > out_rsync 2>&1
1104        RET=$?
1105        if [ ${RET} -gt 0 ]; then
1106          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed ${i}/${NB_ESSAI}"
1107          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
1108          sleep $DELAI
1109        else
1110          break
1111        fi
1112        (( i = i + 1 ))
1113      done
1114    fi
1115
1116#       #RSYNC WITH NETWORK RSH CALL
1117#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${RHOST}:${2} > out_rsync 2>&1
1118#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${RHOST}:${2} >> out_rsync 2>&1
1119
1120#       #RSYNC WITH NFS USE
1121#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > out_rsync 2>&1
1122#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> out_rsync 2>&1
1123
1124#       RET=$?
1125#       IGCM_sys_Rsync_out $RET
1126
1127#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1128#       (( RET=RET+$? ))
1129
1130    if [ ${RET} -gt 0 ] ; then
1131      echo "IGCM_sys_Put_Out : error."
1132      cat out_rsync
1133      IGCM_debug_Exit "IGCM_sys_Put_Out"
1134    else
1135      \rm out_rsync
1136    fi
1137  else
1138    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1139  fi
1140  IGCM_debug_PopStack "IGCM_sys_Put_Out"
1141  return 0
1142}
1143
1144#D-#==================================================
1145#D-function IGCM_sys_Get
1146#D-* Purpose: Get a file from ${ARCHIVE}
1147#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
1148#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
1149function IGCM_sys_Get {
1150  IGCM_debug_PushStack "IGCM_sys_Get" $@
1151
1152  typeset DEST RET dm_liste ifile target file_work
1153  typeset NB_ESSAI DELAI status i
1154
1155  if ( $DEBUG_sys ) ; then
1156    echo "IGCM_sys_Get :" $@
1157  fi
1158
1159  # number of tentative
1160  NB_ESSAI=3
1161  # time delay between tentative
1162  DELAI=2
1163
1164  if [ $DRYRUN -le 2 ]; then
1165    if [ X${1} = X'/l' ] ; then
1166      # test if the first file is present in the old computation :
1167      eval set +A dm_liste \${${2}}
1168    else
1169      eval set +A dm_liste ${1}
1170    fi
1171    eval DEST=\${${#}}
1172
1173    #=====================================================
1174    #         COMMENT OUT DOUBLE COPY ON SCRATCHDIR
1175    #=====================================================
1176
1177    # Is it an R_OUT file (not R_IN) ?
1178    #echo ${dm_liste[0]} | grep "${R_OUT}" > /dev/null 2>&1
1179    #if [ $? -eq 0 ] ; then
1180    #    # Yes  ? then we try to get it in SCRATCHDIR
1181    #    set +A file_work $( echo ${dm_liste[*]} | sed -e "s|${R_OUT}|${R_BUF}|g" )
1182    #    if [ -f ${file_work[0]} ] ; then
1183    #   IGCM_sys_Cp ${file_work[*]} ${DEST}
1184    #   IGCM_debug_PopStack "IGCM_sys_Get"
1185    #   return
1186    #    fi
1187    #fi
1188
1189    ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1190    RET=$?
1191    if [ ${RET} -gt 0 ] ; then
1192      echo "WARNING IGCM_sys_Get : demigration error."
1193      cat out_rsync
1194      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
1195    fi
1196
1197    #if [ ${RET} -gt 0 ] ; then
1198    #    if [ ! "X$( grep "Lost dmusrcmd connection" out_rsync )" = "X" ] ; then
1199    #   cat out_rsync
1200    #   echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
1201    #   sleep 30
1202    #   echo "We try another time"
1203    ##  dmget ${dm_liste[*]} > out_rsync 2>&1
1204    #   ccc_hsm get ${dm_liste[*]} > out_rsync 2>&1
1205    #   RET=$?
1206    #   if [ ${RET} -gt 0 ] ; then
1207    #       echo "ERROR IGCM_sys_Get : again demigration error :"
1208    #       cat out_rsync
1209    #       IGCM_debug_Exit "IGCM_sys_Get"
1210    #   fi
1211    #    else
1212    #   echo "ERROR IGCM_sys_Get : demigration error :"
1213    #   cat out_rsync
1214    #   IGCM_debug_Exit "IGCM_sys_Get"
1215    #    fi
1216    #fi
1217
1218    #USUAL WAY
1219    if [ X${1} = X'/l' ] ; then
1220      (( RET=0 ))
1221      for target in ${dm_liste[*]} ; do
1222        local_file=$( basename ${target} )
1223        (( i = 0 ))
1224        # test if the target file is present before the loop
1225        IGCM_sys_TestFileArchive ${target}
1226        RET=$?
1227        if [ ${RET} -gt 0 ] ; then
1228          echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
1229          IGCM_debug_Exit "IGCM_sys_Get"
1230        else
1231          while [ $i -lt $NB_ESSAI ] ; do
1232            \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1233            status=$?
1234            if [ ${status} -gt 0 ]; then
1235              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed ${i}/${NB_ESSAI}"
1236              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
1237              sleep $DELAI
1238            else
1239              break
1240            fi
1241            (( i = i + 1 ))
1242          done
1243          (( RET = RET + status ))
1244        fi
1245      done
1246    else
1247      \cp ${dm_liste} ${DEST} >> out_rsync 2>&1
1248      RET=$?
1249    fi
1250
1251#       #RSYNC WITH NETWORK SSH CALL
1252#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RHOST}:"${dm_liste}" ${RHOST}:${RUN_DIR}/${DEST} > out_rsync 2>&1
1253#       ${RSYNC} ${RSYNC_opt} -e ssh ${RHOST}:"${dm_liste}" ${RHOST}:${RUN_DIR}/${DEST} >> out_rsync 2>&1
1254
1255#       #RSYNC WITH NFS USE
1256#       echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > out_rsync 2>&1
1257#       ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> out_rsync 2>&1
1258
1259#       RET=$?
1260#       IGCM_sys_Rsync_out $RET
1261
1262#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk out_rsync
1263#       (( RET=RET+$? ))
1264
1265    if [ ${RET} -gt 0 ] ; then
1266      echo "IGCM_sys_Get : copy error."
1267      cat out_rsync
1268      IGCM_debug_Exit "IGCM_sys_Get"
1269    else
1270      \rm out_rsync
1271    fi
1272  else
1273    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1274  fi
1275  IGCM_debug_PopStack "IGCM_sys_Get"
1276}
1277
1278#D-#==================================================
1279#D-function IGCM_sys_GetBuffer
1280#D-* Purpose: Get a file from ${SCRATCHDIR}
1281#D-* Examples: IGCM_sys_GetBuffer myfile /destpath/myfile_with_PREFIX
1282#D-            IGCM_sys_GetBuffer /l Array_contain_myfiles /destpath/
1283function IGCM_sys_GetBuffer {
1284  IGCM_debug_PushStack "IGCM_sys_GetBuffer" $@
1285
1286  typeset DEST RET buf_liste ifile target file_work
1287
1288  if ( $DEBUG_sys ) ; then
1289    echo "IGCM_sys_GetBuffer :" $@
1290  fi
1291  if [ $DRYRUN -le 2 ]; then
1292    if [ X${1} = X'/l' ] ; then
1293      # test if the first file is present in the old computation :
1294      eval set +A buf_liste \${${2}}
1295    else
1296      eval set +A buf_liste ${1}
1297    fi
1298    eval DEST=\${${#}}
1299
1300    #USUAL WAY
1301    if [ X${1} = X'/l' ] ; then
1302      (( RET=0 ))
1303      for target in ${buf_liste[*]} ; do
1304        local_file=$( basename ${target} )
1305        \cp ${target} ${DEST}/${local_file} >> out_rsync 2>&1
1306        (( RET = RET + $? ))
1307      done
1308    else
1309      \cp ${buf_liste} ${DEST} >> out_rsync 2>&1
1310      RET=$?
1311    fi
1312
1313    if [ ${RET} -gt 0 ] ; then
1314      echo "IGCM_sys_GetBuffer : copy error."
1315      cat out_rsync
1316      IGCM_debug_Exit "IGCM_sys_GetBuffer"
1317    else
1318      \rm out_rsync
1319    fi
1320  else
1321    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1322  fi
1323  IGCM_debug_PopStack "IGCM_sys_GetBuffer"
1324}
1325
1326#D-#==================================================
1327#D-function IGCM_sys_GetDate_FichWork
1328#D-* Purpose: donne la date filesys d'un fichier sur le filesystem WORK
1329#D-* Examples:
1330#D-
1331function IGCM_sys_GetDate_FichWork {
1332  IGCM_debug_PushStack "IGCM_sys_FichWork" $@
1333  if ( $DEBUG_sys ) ; then
1334    echo "IGCM_sys_GetDate_FichWork :" $@
1335  fi
1336  # donne la date filesys d'un fichier sur la machine work
1337  IGCM_debug_PopStack "IGCM_sys_FichWork"
1338}
1339
1340#D-#==================================================
1341#D-function IGCM_sys_GetDate_FichArchive
1342#D-* Purpose: donne la date filesys d'un fichier sur le filesystem ARCHIVE
1343#D-* Examples:
1344#D-
1345function IGCM_sys_GetDate_FichArchive {
1346  IGCM_debug_PushStack "IGCM_sys_FichArchive" $@
1347  if ( $DEBUG_sys ) ; then
1348    echo "IGCM_sys_GetDate_FichArchive :" $@
1349  fi
1350  IGCM_debug_PopStack "IGCM_sys_FichArchive"
1351}
1352
1353#D-#==================================================
1354#D-function IGCM_sys_Dods_Rm
1355#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
1356#D-* Examples:
1357#D-
1358function IGCM_sys_Dods_Rm {
1359  if ( $DEBUG_sys ) ; then
1360    echo "IGCM_sys_Dods_Rm :" $@
1361  fi
1362  typeset RET
1363  RET=0
1364  if [ $DRYRUN = 0 ]; then
1365
1366#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
1367#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
1368#      echo "Nothing has been done."
1369#      return
1370#    fi
1371
1372    /ccc/cont003/home/dsm/p86maf/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
1373    RET=$?
1374
1375#       if [ ${RET} -gt 0 ] ; then
1376#           echo "IGCM_sys_Dods_Rm : error."
1377#           cat out_dods_rm
1378#           IGCM_debug_Exit "IGCM_sys_Dods_Rm"
1379#       else
1380#           rm out_dods_rm
1381#       fi
1382
1383  else
1384    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1385  fi
1386  return $RET
1387}
1388
1389#D-#==================================================
1390#D-function IGCM_sys_Dods_Cp
1391#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
1392#D-* Examples:
1393#D-
1394function IGCM_sys_Dods_Cp {
1395  if ( $DEBUG_sys ) ; then
1396    echo "IGCM_sys_Dods_Cp :" $@
1397  fi
1398  typeset RET
1399  RET=0
1400  if [ $DRYRUN = 0 ]; then
1401
1402#    if [ ! -d ${R_SAVE}/${1} ] ; then
1403#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
1404#      echo "Nothing has been done."
1405#      return
1406#    fi
1407
1408    /ccc/cont003/home/dsm/p86maf/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
1409    RET=$?
1410
1411#       if [ ${RET} -gt 0 ] ; then
1412#           echo "IGCM_sys_Dods_Cp : error."
1413#           cat out_dods_cp
1414#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
1415#       else
1416#           rm out_dods_cp
1417#       fi
1418
1419  else
1420    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1421  fi
1422  return $RET
1423}
1424
1425#D-#==================================================
1426#D-function IGCM_sys_Put_Dods
1427#D-* Purpose: Put $(ARCHIVE) files on DODS internet protocole.
1428#D-* Examples:
1429#D-
1430function IGCM_sys_Put_Dods {
1431  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1432  if ( $DEBUG_sys ) ; then
1433    echo "IGCM_sys_Put_Dods :" $@
1434  fi
1435  typeset RET
1436  if [ $DRYRUN = 0 ]; then
1437    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1438      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1439      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1440      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1441      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1442      return
1443    fi
1444
1445    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1446      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1447      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1448      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1449      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1450      return
1451    fi
1452    #
1453    if [ -d ${R_SAVE}/${1} ] ; then
1454      cd ${R_SAVE}
1455    elif [ -d ${R_FIGR}/${1} ] ; then
1456      cd ${R_FIGR}
1457    fi
1458
1459    IGCM_sys_Dods_Rm ${1}
1460    IGCM_sys_Dods_Cp ${1}
1461    RET=0
1462
1463    if [ ${RET} -gt 0 ] ; then
1464      echo "IGCM_sys_Put_Dods : error."
1465      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1466    fi
1467  else
1468    ( ${DEBUG_debug} ) && echo "DRYRUN mode = " $DRYRUN >> stack
1469  fi
1470  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1471}
1472
1473##############################################################
1474# REBUILD OPERATOR
1475
1476function IGCM_sys_rebuild {
1477  IGCM_debug_PushStack "IGCM_sys_rebuild" -- $@
1478  if ( $DEBUG_sys ) ; then
1479    echo "IGCM_sys_rebuild :" $@
1480  fi
1481  /home/cont003/p86ipsl/CESIUM/bin/rebuild -f -o $@
1482  if [ $? -gt 0 ] ; then
1483    echo "IGCM_sys_rebuild : erreur ${@}."
1484    IGCM_debug_Exit "rebuild"
1485  fi
1486
1487  IGCM_debug_PopStack "IGCM_sys_rebuild"
1488}
1489
1490function IGCM_sys_rebuild_station {
1491  IGCM_debug_PushStack "IGCM_sys_rebuild_station" -- $@
1492  typeset i list_opt file_in file_out prefix_invert list_invert
1493  if ( $DEBUG_sys ) ; then
1494    echo "IGCM_sys_rebuild_station :" $@
1495  fi
1496  list_opt=$@
1497
1498  # Invert Axis : t,x -> x,t
1499  #               t,pres,x -> x,t,pres
1500  # So that we can concatenate along x
1501  i=0
1502  for file_in in ${list_opt} ; do
1503    (( i = i + 1))
1504    [ ${i} = 1 ] && file_out=${file_in} && continue
1505    prefix_invert=$( basename ${file_in} .nc )
1506    IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs ${file_in} ${prefix_invert}_xt.nc
1507    list_invert[${#list_invert[*]}]=${prefix_invert}_xt.nc
1508  done
1509
1510  # Concatenate
1511  IGCM_sys_ncrcat ${list_invert[*]} histstn_xt.nc
1512
1513  # Re-ivert file
1514  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x histstn_xt.nc ${file_out}
1515
1516  # Station re-ordering is too expansive to be run within libICGM
1517  # This is due to (ncpdq - nrcat - ncpdq) I/O sequence.
1518  # This re-ordering must be done "in memory" by the cmorization process
1519  # Anyway this is the best sequence using (ncpdq - nrcat - ncpdq)
1520  # BEGIN reordering
1521
1522  # Only LMDZ text output contains the exact ordering of the station.
1523  # We isolate this in the code below:
1524  #  0  38  -157.5000000000000  70.98591549295774
1525  #  0  54  27.49999999999999   67.18309859154928
1526  #  0  56  -62.50000000000001  82.39436619718309
1527  #  0  79  12.49999999999999   78.59154929577466
1528  #  0  116 -165.0000000000000  76.05633802816901
1529  #  0  117 130.0000000000000   70.98591549295774
1530  #  0  118 110.0000000000000   87.46478873239437
1531  #  1  40  4.999999999999995   51.97183098591550
1532#  typeset iStation iProc list_opt file_in file_out prefix_invert
1533#  typeset -Z4 j4
1534#  typeset -Z3 j3
1535
1536#  unset list_opt
1537#  set +A list_opt $@
1538
1539  # Filename after rebuild
1540#  file_out=${list_opt[0]}
1541  # Prefix of output files
1542#  prefix_invert=$( basename ${file_out} .nc )
1543  # Number of procs
1544#  num_proc=$( grep -i mpi_size ${PREFIX}_${Exe_Output} | wc -l )
1545
1546#  iProc=0
1547#  while [ ${iProc} -lt ${num_proc} ] ; do
1548    # Array containing Station as a number
1549#    unset proc_stn
1550#    set +A proc_stn $( grep "iophy_mpi rank ip lon lat  $iProc" ${PREFIX}_${Exe_Output} | sed -e "s/iophy_mpi rank ip lon lat //g" | awk ' {print $2}' )
1551    # Number of stations produced by processor proc
1552#    stationLast=${#proc_stn[*]}
1553    # Proc number on 4 digits
1554#    j4=${iProc}
1555    # Init
1556#    iStation=0
1557#    while [ ${iStation} -lt ${stationLast} ] ; do
1558      # Station number on 3 digits
1559#      j3=${proc_stn[${iStation}]}
1560      # Extract station
1561      # Invert Axis : t,x -> x,t
1562      #               t,pres,x -> x,t,pres
1563      # So that we can concatenate along x
1564#      IGCM_sys_ncpdq -a x,time_counter -a x,time_counter,presnivs -d x,$iStation,$iStation ${prefix_invert}_${j4}.nc ${prefix_invert}_stn_${j3}.nc
1565#      (( iStation = iStation + 1 ))
1566#    done
1567#    (( iProc = iProc + 1 ))
1568#  done
1569
1570  # Concatenate all station along x
1571#  IGCM_sys_ncrcat ${prefix_invert}_stn_???.nc ${prefix_invert}_xt.nc
1572
1573  # Re-invert file
1574#  IGCM_sys_ncpdq -a time_counter,x -a time_counter,presnivs,x ${prefix_invert}_xt.nc ${file_out}
1575
1576  # END reordering
1577
1578  IGCM_debug_PopStack "IGCM_sys_rebuild_station"
1579}
1580
1581############################################################
1582# Activate Running Environnment Variables
1583
1584function IGCM_sys_activ_variables {
1585  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1586  if ( $DEBUG_sys ) ; then
1587    echo "IGCM_sys_activ_variables"
1588  fi
1589  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1590}
1591
1592############################################################
1593# Desactivate Running Environnment Variables
1594
1595function IGCM_sys_desactiv_variables {
1596  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1597  if ( $DEBUG_sys ) ; then
1598    echo "IGCM_sys_desactiv_variables"
1599  fi
1600  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1601}
1602
1603############################################################
1604# Build run file
1605
1606function IGCM_sys_build_run_file {
1607  IGCM_debug_PushStack "IGCM_sys_build_run_file" $@
1608  if ( $DEBUG_sys ) ; then
1609    echo "IGCM_sys_build_run_file"
1610  fi
1611  IGCM_debug_PopStack "IGCM_sys_build_run_file"
1612}
1613
1614############################################################
1615# Check of space available on temporary filesytems
1616function IGCM_sys_check_quota {
1617    IGCM_debug_PushStack "IGCM_sys_check_quota"
1618    if ( $DEBUG_sys ) ; then
1619        echo "IGCM_sys_check_quota"
1620    fi
1621    IGCM_debug_PopStack "IGCM_sys_check_quota"
1622}
1623
1624##############################################################
1625# NCO OPERATOR
1626
1627function IGCM_sys_ncap2 {
1628  IGCM_debug_PushStack "IGCM_sys_ncap2" -- $@
1629  if ( $DEBUG_sys ) ; then
1630    echo "IGCM_sys_ncap2 :" $@
1631  fi
1632  ncap2 "$@"
1633  if [ $? -gt 0 ] ; then
1634    echo "IGCM_sys_ncap2 : erreur ${@}."
1635    IGCM_debug_Exit "ncap2"
1636  fi
1637
1638  IGCM_debug_PopStack "IGCM_sys_ncap2"
1639}
1640
1641function IGCM_sys_ncatted {
1642  IGCM_debug_PushStack "IGCM_sys_ncatted" -- $@
1643  if ( $DEBUG_sys ) ; then
1644    echo "IGCM_sys_ncatted :" $@
1645  fi
1646  ncatted "$@"
1647  if [ $? -gt 0 ] ; then
1648    echo "IGCM_sys_ncatted : erreur ${@}."
1649    IGCM_debug_Exit "ncatted"
1650  fi
1651
1652  IGCM_debug_PopStack "IGCM_sys_ncatted"
1653}
1654
1655function IGCM_sys_ncbo {
1656  IGCM_debug_PushStack "IGCM_sys_ncbo" -- $@
1657  if ( $DEBUG_sys ) ; then
1658    echo "IGCM_sys_ncbo :" $@
1659  fi
1660  ncbo $@
1661  if [ $? -gt 0 ] ; then
1662    echo "IGCM_sys_ncbo : erreur ${@}."
1663    IGCM_debug_Exit "ncbo"
1664  fi
1665
1666  IGCM_debug_PopStack "IGCM_sys_ncbo"
1667}
1668
1669function IGCM_sys_ncdiff {
1670  IGCM_debug_PushStack "IGCM_sys_ncdiff" -- $@
1671  if ( $DEBUG_sys ) ; then
1672    echo "IGCM_sys_ncdiff :" $@
1673  fi
1674  ncdiff $@
1675  if [ $? -gt 0 ] ; then
1676    echo "IGCM_sys_ncdiff : erreur ${@}."
1677    IGCM_debug_Exit "ncdiff"
1678  fi
1679
1680  IGCM_debug_PopStack "IGCM_sys_ncdiff"
1681}
1682
1683function IGCM_sys_ncea {
1684  IGCM_debug_PushStack "IGCM_sys_ncea" -- $@
1685  if ( $DEBUG_sys ) ; then
1686    echo "IGCM_sys_ncea :" $@
1687  fi
1688  ncea $@
1689  if [ $? -gt 0 ] ; then
1690    echo "IGCM_sys_ncea : erreur ${@}."
1691    IGCM_debug_Exit "ncea"
1692  fi
1693
1694  IGCM_debug_PopStack "IGCM_sys_ncea"
1695}
1696
1697function IGCM_sys_ncecat {
1698  IGCM_debug_PushStack "IGCM_sys_ncecat" -- $@
1699  if ( $DEBUG_sys ) ; then
1700    echo "IGCM_sys_ncecat :" $@
1701  fi
1702  ncecat $@
1703  if [ $? -gt 0 ] ; then
1704    echo "IGCM_sys_ncecat : erreur ${@}."
1705    IGCM_debug_Exit "ncecat"
1706  fi
1707
1708  IGCM_debug_PopStack "IGCM_sys_ncecat"
1709}
1710
1711function IGCM_sys_ncflint {
1712  IGCM_debug_PushStack "IGCM_sys_ncflint" -- $@
1713  if ( $DEBUG_sys ) ; then
1714    echo "IGCM_sys_ncflint :" $@
1715  fi
1716  ncflint $@
1717  if [ $? -gt 0 ] ; then
1718    echo "IGCM_sys_ncflint : erreur ${@}."
1719    IGCM_debug_Exit "ncflint"
1720  fi
1721
1722  IGCM_debug_PopStack "IGCM_sys_ncflint"
1723}
1724
1725function IGCM_sys_ncks {
1726  IGCM_debug_PushStack "IGCM_sys_ncks" -- $@
1727  if ( $DEBUG_sys ) ; then
1728    echo "IGCM_sys_ncks :" $@
1729  fi
1730  ncks $@
1731  if [ $? -gt 0 ] ; then
1732    echo "IGCM_sys_ncks : erreur ${@}."
1733    IGCM_debug_Exit "ncks"
1734  fi
1735
1736  IGCM_debug_PopStack "IGCM_sys_ncks"
1737}
1738
1739function IGCM_sys_ncpdq {
1740  IGCM_debug_PushStack "IGCM_sys_ncpdq" -- $@
1741  if ( $DEBUG_sys ) ; then
1742    echo "IGCM_sys_ncpdq :" $@
1743  fi
1744  ncpdq $@
1745  if [ $? -gt 0 ] ; then
1746    echo "IGCM_sys_ncpdq : erreur ${@}."
1747    IGCM_debug_Exit "ncpdq"
1748  fi
1749
1750  IGCM_debug_PopStack "IGCM_sys_ncpdq"
1751}
1752
1753function IGCM_sys_ncra {
1754  IGCM_debug_PushStack "IGCM_sys_ncra" -- $@
1755  if ( $DEBUG_sys ) ; then
1756    echo "IGCM_sys_ncra :" $@
1757  fi
1758  ncra $@
1759  if [ $? -gt 0 ] ; then
1760    echo "IGCM_sys_ncra : erreur ${@}."
1761    IGCM_debug_Exit "ncra"
1762  fi
1763
1764  IGCM_debug_PopStack "IGCM_sys_ncra"
1765}
1766
1767function IGCM_sys_ncrcat {
1768  IGCM_debug_PushStack "IGCM_sys_ncrcat" -- $@
1769  if ( $DEBUG_sys ) ; then
1770    echo "IGCM_sys_ncrcat :" $@
1771  fi
1772  ncrcat $@
1773  if [ $? -gt 0 ] ; then
1774    echo "IGCM_sys_ncrcat : erreur ${@}."
1775#       IGCM_debug_Exit "ncrcat"
1776  fi
1777
1778  IGCM_debug_PopStack "IGCM_sys_ncrcat"
1779}
1780
1781function IGCM_sys_ncrename {
1782  IGCM_debug_PushStack "IGCM_sys_ncrename" -- $@
1783  if ( $DEBUG_sys ) ; then
1784    echo "IGCM_sys_ncrename :" $@
1785  fi
1786  ncrename $@
1787  if [ $? -gt 0 ] ; then
1788    echo "IGCM_sys_ncrename : erreur ${@}."
1789    IGCM_debug_Exit "ncrename"
1790  fi
1791
1792  IGCM_debug_PopStack "IGCM_sys_ncrename"
1793}
1794
1795function IGCM_sys_ncwa {
1796  IGCM_debug_PushStack "IGCM_sys_ncwa" -- $@
1797  if ( $DEBUG_sys ) ; then
1798    echo "IGCM_sys_ncwa :" $@
1799  fi
1800  ncwa $@
1801  if [ $? -gt 0 ] ; then
1802    echo "IGCM_sys_ncwa : erreur ${@}."
1803    IGCM_debug_Exit "ncwa"
1804  fi
1805
1806  IGCM_debug_PopStack "IGCM_sys_ncwa"
1807}
1808
1809##############################################################
1810# CDO OPERATOR
1811
1812function IGCM_sys_cdo {
1813  IGCM_debug_PushStack "IGCM_sys_cdo" -- $@
1814  if ( $DEBUG_sys ) ; then
1815    echo "IGCM_sys_cdo :" $@
1816  fi
1817  \cdo $@
1818  if [ $? -gt 0 ] ; then
1819    echo "IGCM_sys_cdo : erreur ${@}."
1820    IGCM_debug_PopStack "IGCM_sys_cdo"
1821    return 1
1822  else
1823    IGCM_debug_PopStack "IGCM_sys_cdo"
1824    return 0
1825  fi
1826
1827  IGCM_debug_PopStack "IGCM_sys_cdo"
1828}
Note: See TracBrowser for help on using the repository browser.