source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_ciclad.ksh @ 1188

Last change on this file since 1188 was 1188, checked in by sdipsl, 9 years ago
  • /usr/bin/qsub is the right path. ciclad-ng removed /usr/local/bin/qsub that should not have been here.
  • Property svn:keywords set to Revision Date Author
File size: 35.6 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil
5# Contact: Sebastien.Denvil__at__ipsl.jussieu.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for ciclad
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host and user names
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=${MASTER:=$( hostname )}
71
72#D-
73#D-#==================================================
74#D-Program used in libIGCM
75#D-#==================================================
76
77# Submit command
78typeset SUBMIT=${SUBMIT:=qsub}
79# rsync with path
80typeset -r RSYNC=/usr/bin/rsync
81# RSYNC_opt args to rsync
82typeset -r RSYNC_opt="-va"
83# ie storage filesystem
84typeset -r STOREHOST=${MASTER}
85
86#====================================================
87# Access to module command
88#====================================================
89. /usr/share/Modules/init/ksh
90
91#====================================================
92# Set environment tools (ferret, nco, cdo, rebuild, ...)
93#====================================================
94. /home/igcmg/.atlas_env_ciclad_ksh
95export PATH=${PATH}:/home/igcmg/rebuild/src_X64_CICLAD/modipsl_v2_2_3_netcdf4.2/bin/
96
97#====================================================
98# Host specific DIRECTORIES
99#====================================================
100
101#====================================================
102#- MirrorlibIGCM for frontend
103typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
104
105#====================================================
106#- libIGCM_POST for frontend
107typeset -r libIGCM_POST=${libIGCM}
108
109#====================================================
110#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
111typeset -r R_EXE="${MODIPSL}/bin"
112
113#====================================================
114#- SUBMIT_DIR : submission dir
115if [ X${PBS_O_WORKDIR} != X ] ; then
116  typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PBS_O_WORKDIR}}
117else
118  typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${PWD}}
119fi
120
121#====================================================
122#- IN
123typeset -r R_IN=${R_IN:=/prodigfs/ipslfs/igcmg/IGCM}
124typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/prodigfs/ipslfs/igcmg/IGCM}
125
126#====================================================
127#- RUN_DIR_PATH : Temporary working directory (=> TMP)
128if [ X${PBS_JOBID} != X ] ; then
129  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/data/${LOGIN}/RUN_DIR/${PBS_JOBID}_${$}}
130else
131  typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=/tmp/tmp$$}
132fi
133
134#====================================================
135#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
136typeset -r OUTCOMMAND_PATH=/tmp
137
138#====================================================
139#- HOST_MPIRUN_COMMAND
140typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="time mpirun"}
141
142#====================================================
143#- Max number of arguments passed to nco operator or demigration command
144UNIX_MAX_LIMIT=360
145
146#====================================================
147#- set PackDefault to false on ciclad
148PackDefault=false
149
150#====================================================
151#- Number of core per node (max number of OpenMP task)
152NUM_COREPERNODE=1
153
154#====================================================
155#- Default number of MPI task for IPSL coupled model
156#- required for backward compatibility
157#-
158DEFAULT_NUM_PROC_OCE=1
159DEFAULT_NUM_PROC_CPL=1
160(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
161DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
162
163#D-#==================================================
164#D-function IGCM_sys_defineArchives
165#D-* Purpose:
166#D-* Define ARCHIVE : Dedicated to large files
167#D-* Define STORAGE : Dedicated to small/medium files
168#D-* Define R_OUT   : Output tree located on ARCHIVE
169#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
170#D-* Define R_BUF   : Output tree located on STORAGE hosting files waiting for rebuild or pack processes
171#D-* if SpaceName=TEST nothing special will hapen
172#D-* Examples:
173#D-
174function IGCM_sys_defineArchives {
175  IGCM_debug_PushStack "IGCM_sys_defineArchives"
176
177  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
178    #====================================================
179    #- ARCHIVE (dedicated to large files)
180    ARCHIVE=${config_UserChoices_ARCHIVE}
181  else
182    #====================================================
183    #- ARCHIVE (dedicated to large files)
184    ARCHIVE=${ARCHIVE:=/data/${LOGIN}}
185  fi
186
187  if [ ! X${config_UserChoices_STORAGE} = X ]; then
188    #====================================================
189    #- STORAGE (dedicated to small/medium files)
190    STORAGE=${config_UserChoices_STORAGE}
191  else
192    #====================================================
193    #- STORAGE (dedicated to small/medium files)
194    STORAGE=${ARCHIVE}
195  fi
196
197  # ON CICLAD NO SPECIAL CASE WHEN X${config_UserChoices_SpaceName} = XTEST
198
199  #====================================================
200  #- R_OUT
201  R_OUT=${ARCHIVE}/IGCM_OUT
202
203  #====================================================
204  #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
205  R_FIG=${STORAGE}/IGCM_OUT
206
207  #====================================================
208  #- R_BUF (ONLY FOR double copy an scratch)
209  R_BUF=${STORAGE}/IGCM_OUT
210
211  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
212  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
213  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
214
215  IGCM_debug_PopStack "IGCM_sys_defineArchives"
216}
217
218#D-#==================================================
219#D-function IGCM_sys_RshArchive
220#D-* Purpose: Archive rsh command
221#D-* Examples:
222#D-
223function IGCM_sys_RshArchive {
224  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
225  /bin/ksh <<-EOF
226    ${@}
227EOF
228  status=$?
229  if [ ${status} -gt 0 ] ; then
230    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
231    IGCM_debug_Exit "IGCM_sys_RshArchive"
232  fi
233  IGCM_debug_PopStack "IGCM_sys_RshArchive"
234}
235
236#D-#==================================================
237#D-function IGCM_sys_RshArchive_NoError
238#D-* Purpose: Archive rsh command, without error
239#D-*          used only in monitoring.job
240#D-* Examples:
241#D-
242function IGCM_sys_RshArchive_NoError {
243  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
244  /bin/ksh <<-EOF
245    ${@} 2> /dev/null
246EOF
247  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
248}
249
250#D-#==================================================
251#D-function IGCM_sys_MkdirArchive
252#D-* Purpose: Mkdir on Archive
253#D-* Examples:
254#D-
255function IGCM_sys_MkdirArchive {
256  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
257  if ( $DEBUG_sys ) ; then
258    echo "IGCM_sys_MkdirArchive :" $@
259  fi
260  #- creation de repertoire sur le serveur fichier
261  if [ ! -d ${1} ]; then
262    \mkdir -p $1
263    status=$?
264
265    if [ ${status} -gt 0 ] ; then
266      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
267      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
268    fi
269  fi
270  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
271}
272
273#D-#==================================================
274#D-function IGCM_sys_TestDirArchive
275#D-* Purpose: Test Directory that must exists on Archive
276#D-* Examples:
277#D-
278function IGCM_sys_TestDirArchive {
279  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
280  if ( $DEBUG_sys ) ; then
281    echo "IGCM_sys_TestDirArchive :" $@
282  fi
283  typeset ExistFlag
284  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
285  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
286  return ${ExistFlag}
287}
288
289#D-#==================================================
290#D-function IGCM_sys_IsFileArchived
291#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
292#D-* Examples:
293#D-
294function IGCM_sys_IsFileArchived {
295  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
296  if ( $DEBUG_sys ) ; then
297    echo "IGCM_sys_IsFileArchived :" $@
298  fi
299  typeset IsArchivedFlag
300  # Never archived for this system
301  IsArchivedFlag=1
302  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
303
304  return ${IsArchivedFlag}
305}
306
307#D-#==================================================
308#D-function IGCM_sys_TestFileArchive
309#D-* Purpose: Test file that must NOT EXISTS on Archive
310#D-* Examples:
311#D-
312function IGCM_sys_TestFileArchive {
313  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
314  typeset ExistFlag
315  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
316  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
317
318  return ${ExistFlag}
319}
320
321#D-#==================================================
322#D-function IGCM_sys_CountFileArchive
323#D-* Purpose: Count files on Archive filesystem
324#D-* Examples:
325#D-
326function IGCM_sys_CountFileArchive {
327  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
328  ls ${@} 2>/dev/null | wc -l
329  if [ $? -gt 0 ] ; then
330    echo "IGCM_sys_CountFileArchive : erreur."
331  fi
332  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
333}
334
335#D-#==================================================
336#D-function IGCM_sys_Tree
337#D-* Purpose: Tree directories with files on ${ARCHIVE}
338#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
339#D-
340function IGCM_sys_Tree {
341  IGCM_debug_PushStack "IGCM_sys_Tree" $@
342  if ( $DEBUG_sys ) ; then
343    echo "IGCM_sys_Tree :" $@
344  fi
345
346  \tree -f $@
347
348  IGCM_debug_PopStack "IGCM_sys_Tree"
349}
350
351#D-#==================================================
352#D-function IGCM_sys_Qsub
353#D-* Purpose: Qsub new job
354#D-* Examples:
355#D-
356function IGCM_sys_Qsub {
357  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
358  if ( $DEBUG_sys ) ; then
359    echo "IGCM_sys_Qsub :" $@
360  fi
361  typeset options status
362  options="-o ${SUBMIT_DIR}/${Script_Output}"
363  /usr/bin/qsub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
364  status=$?
365
366  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
367  if [ ${status} -gt 0 ] ; then
368    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
369    IGCM_debug_Exit "IGCM_sys_Qsub"
370  else
371    JobID=$( gawk {'print $1'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
372  fi
373  IGCM_debug_PopStack "IGCM_sys_Qsub"
374}
375
376#D-#==================================================
377#D-function IGCM_sys_QsubPost
378#D-* Purpose: Qsub new job on scalaire
379#D-* Examples:
380#D-
381function IGCM_sys_QsubPost {
382  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
383  if ( $DEBUG_sys ) ; then
384    echo "IGCM_sys_QsubPost :" $@
385  fi
386  typeset options status
387  options="-o ${POST_DIR}/${Script_Post_Output}.out -v ${listVarEnv}"
388  /usr/bin/qsub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
389  status=$?
390
391  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
392  if [ ${status} -gt 0 ] ; then
393    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
394    IGCM_debug_Exit "IGCM_sys_QsubPost"
395  else
396    JobID=$( gawk {'print $1'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
397  fi
398  IGCM_debug_PopStack "IGCM_sys_QsubPost"
399}
400
401#D-*************************
402#D- File transfer functions
403#D-*************************
404#D-
405
406#D-#==================================================
407#D-function IGCM_sys_RmRunDir
408#D-* Purpose: rm tmpdir (dummy function most of the time batch
409#D-                      scheduler will do the job)
410#D-* Examples:
411#D-
412function IGCM_sys_RmRunDir {
413  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
414  if ( $DEBUG_sys ) ; then
415    echo "IGCM_sys_RmRunDir :" $@
416  fi
417
418  typeset status
419
420  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
421  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
422  status=$?
423
424  if [ ${status} -gt 0 ] ; then
425    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
426    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
427    IGCM_debug_Exit "IGCM_sys_RmRunDir"
428  else
429    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
430  fi
431  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
432}
433
434#D-#==================================================
435#D-function IGCM_sys_Put_Dir
436#D-* Purpose: Copy a complete directory on $(ARCHIVE)
437#D-* Examples:
438#D-
439function IGCM_sys_Put_Dir {
440  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
441  if ( $DEBUG_sys ) ; then
442    echo "IGCM_sys_Put_Dir :" $@
443  fi
444  if [ $DRYRUN = 0 ]; then
445    if [ ! -d ${1} ] ; then
446      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
447      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
448      return
449    fi
450
451    typeset status
452
453    # Only if we use rsync
454    #IGCM_sys_TestDirArchive $( dirname $2 )
455    #
456    #USUAL WAY
457    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
458    status=$?
459
460    if [ ${status} -gt 0 ] ; then
461      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
462      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
463      IGCM_debug_Exit "IGCM_sys_Put_Dir"
464    else
465      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
466    fi
467  fi
468  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
469}
470
471#D-#==================================================
472#D-function IGCM_sys_Get_Dir
473#D-* Purpose: Copy a complete directory from ${ARCHIVE}
474#D-* Examples:
475#D-
476function IGCM_sys_Get_Dir {
477  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
478  if ( $DEBUG_sys ) ; then
479    echo "IGCM_sys_Get_Dir :" $@
480  fi
481  if [ $DRYRUN = 0 ]; then
482    typeset NB_ESSAI DELAI status i
483    # number of tentative
484    NB_ESSAI=3
485    # time delay between tentative
486    DELAI=2
487
488    #
489    # USUAL WAY
490    i=0
491    while [ $i -lt $NB_ESSAI ] ; do
492      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
493      status=$?
494      if [ ${status} -gt 0 ] ; then
495        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
496        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
497        sleep $DELAI
498      else
499        break
500      fi
501      (( i = i + 1 ))
502    done
503
504    if [ ${status} -gt 0 ] ; then
505      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
506      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
507      IGCM_debug_Exit "IGCM_sys_Get_Dir"
508    else
509      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
510    fi
511  fi
512  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
513}
514
515#D-#==================================================
516#D-function IGCM_sys_Put_Rest
517#D-* Purpose: Put computied restarts on ${ARCHIVE}.
518#D-           File and target directory must exist.
519#D-* Examples:
520#D-
521function IGCM_sys_Put_Rest {
522  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
523  if ( $DEBUG_sys ) ; then
524    echo "IGCM_sys_Put_Rest :" $@
525  fi
526  if [ $DRYRUN = 0 ]; then
527    if [ ! -f ${1} ] ; then
528      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
529      IGCM_debug_Exit "IGCM_sys_Put_Rest"
530    fi
531
532    typeset status
533    #
534    # USUAL WAY
535    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
536    status=$?
537
538#       #RSYNC WITH NETWORK SSH CALL
539#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
540#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
541
542#       #RSYNC WITH NFS USE
543#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
544#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
545
546#       status=$?
547#       IGCM_sys_Rsync_out $status
548
549#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
550#       (( status=status+$? ))
551
552    if [ ${status} -gt 0 ] ; then
553      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
554      [ -f ${1} ] && ls -l ${1}
555      [ -f ${2} ] && ls -l ${2}
556      [ -f ${2}/${1} ] && ls -l ${2}/${1}
557      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
558      IGCM_debug_Exit "IGCM_sys_Put_Rest"
559    else
560
561      if [ X${JobType} = XRUN ] ; then
562        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
563        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
564      fi
565
566      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
567    fi
568  fi
569  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
570}
571
572#D-#==================================================
573#D-function IGCM_sys_Put_Out
574#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
575#D-* Examples:
576#D-
577function IGCM_sys_Put_Out {
578  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
579  if ( $DEBUG_sys ) ; then
580    echo "IGCM_sys_Put_Out :" $@
581  fi
582
583  typeset status
584
585  if [ $DRYRUN = 0 ]; then
586    if [ ! -f ${1} ] ; then
587      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
588      IGCM_debug_PopStack "IGCM_sys_Put_Out"
589      return 1
590    fi
591    #
592    IGCM_sys_MkdirArchive $( dirname $2 )
593    #
594    if [ X${JobType} = XRUN ] ; then
595      if [ X${3} = X ] ; then
596        IGCM_sys_Chmod 444 ${1}
597      fi
598    fi
599    #
600
601    echo ${RSYNC} ${RSYNC_opt} $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
602    ${RSYNC} ${RSYNC_opt} $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
603    status=$?
604    IGCM_sys_Rsync_out $status
605
606    ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
607    (( status=status+$? ))
608
609    if [ ${status} -gt 0 ] ; then
610      IGCM_debug_Print 2 "IGCM_sys_Put_Out : rsync failed error code ${status}"
611      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
612      IGCM_debug_Exit "IGCM_sys_Put_Out"
613    else
614      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
615    fi
616  fi
617  IGCM_debug_PopStack "IGCM_sys_Put_Out"
618  return 0
619}
620
621#D-#==================================================
622#D-function IGCM_sys_Get
623#D-* Purpose: Get a file from ${ARCHIVE}
624#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
625#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
626function IGCM_sys_Get {
627  IGCM_debug_PushStack "IGCM_sys_Get" $@
628
629  typeset DEST status dm_liste
630
631  if ( $DEBUG_sys ) ; then
632    echo "IGCM_sys_Get :" $@
633  fi
634  if [ $DRYRUN -le 2 ]; then
635    if [ X${1} = X'/l' ] ; then
636      eval set +A dm_liste \${${2}}
637    else
638      dm_liste=${1}
639    fi
640    eval DEST=\${${#}}
641
642    # test if the (first) file is present in the old computation :
643    IGCM_sys_TestFileArchive ${dm_liste[0]}
644    status=$?
645    if [ ${status} -gt 0 ] ; then
646      echo "IGCM_sys_Get, ERROR : regular file ${dm_liste[0]} DOES NOT EXIST ."
647      IGCM_debug_Exit "IGCM_sys_Get"
648      return
649    fi
650
651    #USUAL WAY
652    \cp ${dm_liste[*]} ${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
653    status=$?
654
655    if [ ${status} -gt 0 ] ; then
656      IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status}"
657      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
658      IGCM_debug_Exit "IGCM_sys_Get"
659    else
660      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
661    fi
662  fi
663  IGCM_debug_PopStack "IGCM_sys_Get"
664}
665
666#D-#==================================================
667#D-function IGCM_sys_GetDate_Monitoring
668#D-* Purpose: get the last year for which the monitoring has been computed
669#D-* Examples:
670#D-
671function IGCM_sys_GetDate_Monitoring {
672  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
673  if ( $DEBUG_sys ) ; then
674    echo "IGCM_sys_GetDate_Monitoring :" $@
675  fi
676
677  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
678
679  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
680}
681
682#D-#==================================================
683#D-function IGCM_sys_Dods_Rm
684#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
685#D-* Examples:
686#D-
687function IGCM_sys_Dods_Rm {
688  if ( $DEBUG_sys ) ; then
689    echo "IGCM_sys_Dods_Rm :" $@
690  fi
691  return 0
692}
693
694#D-#==================================================
695#D-function IGCM_sys_Dods_Cp
696#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
697#D-* Examples:
698#D-
699function IGCM_sys_Dods_Cp {
700  if ( $DEBUG_sys ) ; then
701    echo "IGCM_sys_Dods_Cp :" $@
702  fi
703  return 0
704}
705
706#D-#==================================================
707#D-function IGCM_sys_Put_Dods
708#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole. Dummy function here
709#D-* Examples:
710#D-
711function IGCM_sys_Put_Dods {
712  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
713  if ( $DEBUG_sys ) ; then
714    echo "IGCM_sys_Put_Dods :" $@
715  fi
716  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
717}
718
719##############################################################
720# REBUILD OPERATOR
721
722#D-#==================================================
723#D-function IGCM_sys_sync
724#D-* Purpose: flush buffer on disk (dummy function on Ada)
725#D-* Examples:
726#D-
727function IGCM_sys_sync {
728  IGCM_debug_PushStack "IGCM_sys_sync" $@
729  if ( $DEBUG_sys ) ; then
730    echo "IGCM_sys_sync :" $@
731    echo "Dummy call, let the system do that."
732  fi
733  IGCM_debug_PopStack "IGCM_sys_sync"
734}
735
736############################################################
737# Activate Running Environnment Variables
738
739#D-#==================================================
740#D-function IGCM_sys_desactiv_variables
741#D-* Purpose: set environement variables prior to execution
742#D-* Examples:
743#D-
744function IGCM_sys_activ_variables {
745  IGCM_debug_PushStack "IGCM_sys_activ_variables"
746  if ( $DEBUG_sys ) ; then
747    echo "IGCM_sys_activ_variables"
748  fi
749
750# --------------------------------------------------------------------
751#D- MPI specifications
752# --------------------------------------------------------------------
753
754# --------------------------------------------------------------------
755#D- Other specifications
756# --------------------------------------------------------------------
757
758  ulimit -s unlimited
759
760  IGCM_debug_PopStack "IGCM_sys_activ_variables"
761}
762
763############################################################
764# Desactivate Running Environnment Variables
765
766#D-#==================================================
767#D-function IGCM_sys_desactiv_variables
768#D-* Purpose: unset environement variables after execution
769#D-* Examples:
770#D-
771function IGCM_sys_desactiv_variables {
772  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
773  if ( $DEBUG_sys ) ; then
774    echo "IGCM_sys_desactiv_variables"
775  fi
776# --------------------------------------------------------------------
777#D- MPI specifications
778# --------------------------------------------------------------------
779
780# --------------------------------------------------------------------
781#D- Other specifications
782# --------------------------------------------------------------------
783
784  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
785}
786
787############################################################
788# Build MPI/OMP scripts run file (dummy function)
789
790#D-#==================================================
791#D-function IGCM_sys_build_run_file
792#D-* Purpose: build run file (deprecated)
793#D-* Examples:
794#D-
795function IGCM_sys_build_run_file {
796
797  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
798
799}
800
801############################################################
802# Build MPI/OMP scripts
803
804#D-#==================================================
805#D-function IGCM_sys_build_execution_scripts
806#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
807#D-* Examples:
808#D-
809function IGCM_sys_build_execution_scripts
810{
811  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
812  if ( $DEBUG_sys ) ; then
813    echo "IGCM_sys_build_execution_scripts " $@
814  fi
815
816  typeset nodes listnodes init_node start_num init_exec comp ExeNameIn ExeNameOut
817  typeset node_num_current node_current comp_proc_mpi_loc comp_proc_omp_loc
818  typeset num_corempi nombre_restant_node nombre_restant_comp
819
820  if ( ${OK_PARA_MPMD} ) ; then
821
822    if [ -f run_file ] ; then
823      IGCM_sys_Rm -f run_file
824    fi
825    touch run_file
826
827    if ( ${OK_PARA_OMP} ) ; then
828
829      #  Hosts treatment
830
831      ${HOST_MPIRUN_COMMAND} hostname | sort | uniq > hosts.tmp
832
833      i=0
834      rm -f hosts
835      IGCM_debug_Print 1 "sys Ciclad, Hosts avaible :"
836      for nodes in `cat hosts.tmp` ; do
837        host[$i]=$nodes
838        echo "${host[$i]} slots=1 max_slots=1" >> hosts
839        IGCM_debug_Print 1 ${host[$i]}
840        i=$((i+1))
841      done
842      rm -f hosts.tmp
843
844      listnodes=${host[*]}
845
846      EXECUTION="${HOST_MPIRUN_COMMAND} -hostfile hosts"
847
848      # Initialisation
849
850      init_node=y
851      node_num_current=0
852      start_num=0
853      init_exec=n
854
855      # Build run_file
856
857      # First loop on the components for the coupler ie oasis (only if oasis3)
858      # the coupler ie oasis3 must be the first one
859      for comp in ${config_ListOfComponents[*]} ; do
860
861        if [ "X${comp}" = "XCPL" ]  ; then
862
863          eval ExeNameIn=\${config_Executable_${comp}[0]}
864          eval ExeNameOut=\${config_Executable_${comp}[1]}
865
866          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
867          echo ""  >> script_${ExeNameOut}.ksh
868          #echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
869          #echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
870          #echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
871          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err"  >> script_${ExeNameOut}.ksh
872          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
873
874          init_node=n
875
876          (( nombre_restant_node = NUM_COREPERNODE - 1 ))
877          node_num_current=0
878          node_current=${host[${node_num_current}]}
879
880          EXECUTION="${EXECUTION} -H ${node_current} -np 1 ./script_${ExeNameOut}.ksh"
881
882          init_exec=y
883          start_num=1
884
885        fi
886
887      done
888
889      # Then loop on the components (except for oasis)
890
891      for comp in ${config_ListOfComponents[*]} ; do
892
893        eval ExeNameIn=\${config_Executable_${comp}[0]}
894        eval ExeNameOut=\${config_Executable_${comp}[1]}
895
896        # Only if we really have an executable for the component :
897        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" != "XCPL" ] ) ; then
898
899          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
900          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
901
902          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
903          # echo "set -vx" >> script_${ExeNameOut}.ksh
904          echo ""  >> script_${ExeNameOut}.ksh
905          #echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
906          #echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
907          #echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
908          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
909          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
910          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
911          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
912
913          node_num=0
914
915          # We define the number of MPI process to be assigned for the component
916
917          nombre_restant_comp=${comp_proc_mpi_loc}
918
919          # Loop on the allocated nodes
920
921          for node in ${listnodes} ; do
922
923            # We go to the current node
924            if [ ${node_num} = ${node_num_current} ] ; then
925
926              node_current=${host[${node_num_current}]}
927
928              # If first time on the node : initialisation
929
930              if [ ${init_node} = y ] ; then
931                nombre_restant_node=${NUM_COREPERNODE}
932              fi
933
934              # Test on the number of OMP threads
935
936              if [ ${comp_proc_omp_loc} -gt ${nombre_restant_node} ] ; then
937                (( node_num = node_num + 1 ))
938                node_num_current=${node_num}
939                init_node=y
940                continue
941              fi
942
943              # Number of MPI process to assign
944
945              (( num_corempi = nombre_restant_node / comp_proc_omp_loc ))
946
947              if [ ${num_corempi} -gt ${nombre_restant_comp} ] ; then
948                num_corempi=${nombre_restant_comp}
949              fi
950
951              (( nombre_restant_node = nombre_restant_node - num_corempi * comp_proc_omp_loc ))
952              (( nombre_restant_comp = nombre_restant_comp - num_corempi ))
953
954              if [ ${init_exec} = y ] ; then
955                EXECUTION="${EXECUTION} : -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
956              else
957                EXECUTION="${EXECUTION} -H ${node_current} -np ${num_corempi} ./script_${ExeNameOut}.ksh"
958                init_exec=y
959              fi
960
961              ((  start_num = num_corempi + start_num ))
962
963            else
964
965              (( node_num = node_num + 1 ))
966              continue
967            fi
968
969            # Test on the number of core/process remaining on the node/component
970
971            if [ ${nombre_restant_node} = 0 ] ; then
972              (( node_num = node_num + 1 ))
973              node_num_current=${node_num}
974              init_node=y
975
976              if [ ${nombre_restant_comp} = 0 ] ; then
977                break 1
978              fi
979            else
980
981              node_num_current=${node_num}
982              init_node=n
983
984              if [ ${nombre_restant_comp} = 0 ] ; then
985                break 1
986              fi
987            fi
988          done
989        fi
990      done
991
992    else
993
994      # Then first loop on the components for the coupler ie oasis
995
996      ## the coupler ie oasis must be the first one
997      for comp in ${config_ListOfComponents[*]} ; do
998
999        eval ExeNameOut=\${config_Executable_${comp}[1]}
1000
1001        # for CPL component only
1002        if [ "X${comp}" = "XCPL" ] ; then
1003          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1004          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut} " >> run_file
1005        fi
1006      done
1007
1008      # Then second loop on the components
1009
1010      for comp in ${config_ListOfComponents[*]} ; do
1011
1012        eval ExeNameOut=\${config_Executable_${comp}[1]}
1013
1014        # Only if we really have an executable for the component and not the coupler ie oasis:
1015        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1016          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1017          echo "-np ${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1018        fi
1019      done
1020      IGCM_sys_Chmod u+x run_file
1021
1022      EXECUTION="${HOST_MPIRUN_COMMAND} --app ./run_file"
1023
1024    fi
1025
1026  else # Only one executable. launch it.
1027
1028    for comp in ${config_ListOfComponents[*]} ; do
1029
1030      # Only if we really have an executable for the component :
1031      eval ExeNameOut=\${config_Executable_${comp}[1]}
1032      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1033
1034        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1035        echo ""  >> script_${ExeNameOut}.ksh
1036        if ( ${OK_PARA_OMP} ) ; then
1037          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1038          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1039        fi
1040        if  ( ${OK_PARA_MPI} ) ; then
1041          # Default : mpirun used if nb_proc gt 1
1042          # pour sortie out/err par process
1043          # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${OMPI_COMM_WORLD_RANK} 2>out_${ExeNameOut}.err.\${OMPI_COMM_WORLD_RANK}"  >> script_${ExeNameOut}.ksh
1044          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1045          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1046          EXECUTION="${HOST_MPIRUN_COMMAND} ./script_${ExeNameOut}.ksh"
1047        else
1048          # Default : mpirun is NOT used if nb_proc eq 1
1049          # pour sortie out/err par process
1050          # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1051          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1052          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1053          EXECUTION="time ./script_${ExeNameOut}.ksh"
1054        fi
1055
1056        IGCM_debug_Print 1 "sys Ciclad : script_${ExeNameOut}.ksh contains"
1057        cat script_${ExeNameOut}.ksh
1058
1059      fi
1060
1061    done
1062
1063  fi
1064
1065  IGCM_debug_Print 1 "sys Ciclad : execution command is"
1066  IGCM_debug_Print 1 "$EXECUTION"
1067
1068  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1069}
1070
1071#D-#==================================================
1072#D-function IGCM_sys_check_path
1073#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1074#D-* do not point to an important use directory. Stop immediately in that case.
1075#D-* Examples:
1076#D-
1077function IGCM_sys_check_path {
1078  IGCM_debug_PushStack "IGCM_sys_check_path"
1079  if ( $DEBUG_sys ) ; then
1080    echo "IGCM_sys_check_path"
1081  fi
1082
1083  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${ARCHIVE} ] ) ; then
1084    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1085    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1086    IGCM_debug_Exit "This will stop the job"
1087  fi
1088  IGCM_debug_PopStack "IGCM_sys_check_path"
1089}
1090
1091#D-#==================================================
1092#D-function IGCM_sys_check_quota. Dummy call here
1093#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1094#D-* Examples:
1095#D-
1096function IGCM_sys_check_quota {
1097  IGCM_debug_PushStack "IGCM_sys_check_quota"
1098  if ( $DEBUG_sys ) ; then
1099    echo "IGCM_sys_check_quota"
1100  fi
1101
1102  # TO BE IMPLEMENTED
1103
1104  IGCM_debug_PopStack "IGCM_sys_check_quota"
1105}
1106
1107#D-#==================================================
1108#D-function IGCM_sys_GetJobID
1109#D-* Purpose: Check if job_name is currently
1110#D-  running or in queue
1111#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1112#D-
1113function IGCM_sys_GetJobID {
1114  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1115  if ( $DEBUG_sys ) ; then
1116    echo "IGCM_sys_GetJobID"
1117  fi
1118
1119  # With -f option, the full job name is given in the last column
1120  ID="$( qstat -u $2 | grep -w $1 | gawk '-F ' '{print $10}' )"
1121
1122  eval ${3}=${ID}
1123  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1124}
1125
1126#D-#==================================================
1127#D-function IGCM_sys_CountJobInQueue
1128#D-* Purpose: Check if job_name is currently
1129#D-  running or in queue
1130#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1131#D-
1132function IGCM_sys_CountJobInQueue {
1133  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1134  if ( $DEBUG_sys ) ; then
1135    echo "IGCM_sys_CountJobInQueue"
1136  fi
1137  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1138}
Note: See TracBrowser for help on using the repository browser.