source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 1232

Last change on this file since 1232 was 1230, checked in by sdipsl, 9 years ago
  • MPI/OMP handling refactoring
  • IGCM_config_ConfigureExexution will define the MPMD/SPMDO/MPI/OMP context
  • 6 execution types are introduced and will be documented later
  • prerequisite to have ins_job editing the headers
  • IGCM_sys_build_execution_scripts will be adapted later
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 51.2 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Curie
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host user names project maxCpuTime
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=curie
71# add default project on curie
72typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} | cut -d@ -f1 )
73# jobWarningDelay in seconds
74typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=ccc_msub}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Set environment tools (ferret, nco, cdo, rebuild, ...)
93#====================================================
94if [ X${TaskType} = Xcomputing ] ; then
95  . /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4.3.3.1_hdf5_parallel_curie_ksh > /dev/null 2>&1
96# to run with netcdf 3.6.3 ie compilation done before 17/2/2014
97# uncomment 2 lines :
98#  module unload netcdf
99#  module load netcdf/3.6.3
100else
101  . /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4.3.3.1_hdf5_parallel_curie_ksh > /dev/null 2>&1
102  PCMDI_MP=/ccc/work/cont003/dsm/p86ipsl/PCMDI-MP
103fi
104[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
105[ ! X${TaskType} = Xchecking ] && module list
106
107export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
108export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
109export PATH=${PATH}:$( ccc_home -u p86ipsl )/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/
110
111#====================================================
112# Specific for ocean additionnal diagnostic
113export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
114export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
115
116#====================================================
117# Host specific DIRECTORIES
118#====================================================
119
120# ============ CESIUM START ============ #
121
122#====================================================
123#- Mirror libIGCM from titane to cesium if needed
124#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
125#if [ ! ${ROOTSYS} = "home" ] ; then
126#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
127#else
128#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
129#fi
130
131#====================================================
132#- libIGCM_POST
133#if ( ${MirrorlibIGCM} ) ; then
134#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
135#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
136#else
137#  typeset -r libIGCM_POST=${libIGCM}
138#fi
139
140# ============ CESIUM  END  ============ #
141
142#====================================================
143#- MirrorlibIGCM for frontend
144typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
145
146#====================================================
147#- libIGCM_POST for frontend
148typeset -r libIGCM_POST=${libIGCM}
149
150#====================================================
151#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
152typeset -r R_EXE="${MODIPSL}/bin"
153
154#====================================================
155#- SUBMIT_DIR : submission dir
156typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
157
158#====================================================
159#- IN
160typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
161typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
162
163#====================================================
164#- RUN_DIR_PATH : Temporary working directory (=> TMP)
165typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
166
167#====================================================
168#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
169typeset -r OUTCOMMAND_PATH=/tmp
170
171#====================================================
172#- HOST_MPIRUN_COMMAND
173typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1"}
174
175#====================================================
176#- Max number of arguments passed to nco operator or demigration command
177UNIX_MAX_LIMIT=360
178
179#====================================================
180#- set PackDefault to true on curie
181PackDefault=true
182
183#====================================================
184#- Number of core per node (max number of OpenMP task)
185NUM_COREPERNODE=8
186
187#====================================================
188#- Default number of MPI task for IPSL coupled model
189#- required for backward compatibility
190#-
191DEFAULT_NUM_PROC_OCE=5
192DEFAULT_NUM_PROC_CPL=1
193(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
194
195#D-#==================================================
196#D-function IGCM_sys_defineArchives
197#D-* Purpose:
198#D-* Define ARCHIVE : Dedicated to large files
199#D-* Define STORAGE : Dedicated to small/medium files
200#D-* Define R_OUT   : Output tree located on ARCHIVE
201#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
202#D-* Define R_BUF   : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
203#D-* if SpaceName=TEST everything is stored on SCRATCHDIR
204#D-* Examples:
205#D-
206function IGCM_sys_defineArchives {
207  IGCM_debug_PushStack "IGCM_sys_defineArchives"
208
209  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
210    #====================================================
211    #- ARCHIVE (dedicated to large files)
212    ARCHIVE=${config_UserChoices_ARCHIVE}
213  else
214    #====================================================
215    #- ARCHIVE (dedicated to large files)
216    ARCHIVE=${CCCSTOREDIR}
217  fi
218
219  if [ ! X${config_UserChoices_STORAGE} = X ]; then
220    #====================================================
221    #- STORAGE (dedicated to small/medium files)
222    STORAGE=${config_UserChoices_STORAGE}
223  else
224    #====================================================
225    #- STORAGE (dedicated to small/medium files)
226    STORAGE=${CCCWORKDIR}
227  fi
228
229  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
230    #====================================================
231    #- R_OUT
232    R_OUT=${SCRATCHDIR}/IGCM_OUT
233
234    #====================================================
235    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
236    R_FIG=${SCRATCHDIR}/IGCM_OUT
237
238    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
239
240  else
241    #====================================================
242    #- R_OUT
243    R_OUT=${ARCHIVE}/IGCM_OUT
244
245    #====================================================
246    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
247    R_FIG=${STORAGE}/IGCM_OUT
248  fi
249
250  #====================================================
251  #- R_BUF (ONLY FOR double copy an scratch)
252  R_BUF=${SCRATCHDIR}/IGCM_OUT
253
254  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
255  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
256  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
257
258  IGCM_debug_PopStack "IGCM_sys_defineArchives"
259}
260
261#D-#==================================================
262#D-function IGCM_sys_RshArchive
263#D-* Purpose: Archive rsh command
264#D-* Examples:
265#D-
266function IGCM_sys_RshArchive {
267  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
268  /bin/ksh <<-EOF
269    ${@}
270EOF
271  status=$?
272  if [ ${status} -gt 0 ] ; then
273    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
274    IGCM_debug_Exit "IGCM_sys_RshArchive"
275  fi
276  IGCM_debug_PopStack "IGCM_sys_RshArchive"
277}
278
279#D-#==================================================
280#D-function IGCM_sys_RshArchive_NoError
281#D-* Purpose: Archive rsh command, without error
282#D-*          used only in monitoring.job
283#D-* Examples:
284#D-
285function IGCM_sys_RshArchive_NoError {
286  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
287  /bin/ksh <<-EOF
288    ${@} 2> /dev/null
289EOF
290  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
291}
292
293#D-#==================================================
294#D-function IGCM_sys_MkdirArchive
295#D-* Purpose: Mkdir on Archive
296#D-* Examples:
297#D-
298function IGCM_sys_MkdirArchive {
299  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
300  if ( $DEBUG_sys ) ; then
301    echo "IGCM_sys_MkdirArchive :" $@
302  fi
303  #- creation de repertoire sur le serveur fichier
304  if [ ! -d ${1} ]; then
305    \mkdir -p $1
306    status=$?
307
308    if [ ${status} -gt 0 ] ; then
309      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
310      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
311    fi
312  fi
313  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
314}
315
316#D-#==================================================
317#D-function IGCM_sys_TestDirArchive
318#D-* Purpose: Test Directory that must exists on Archive
319#D-* Examples:
320#D-
321function IGCM_sys_TestDirArchive {
322  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
323  if ( $DEBUG_sys ) ; then
324    echo "IGCM_sys_TestDirArchive :" $@
325  fi
326  typeset ExistFlag
327  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
328  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
329  return ${ExistFlag}
330}
331
332#D-#==================================================
333#D-function IGCM_sys_IsFileArchived
334#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
335#D-* Examples:
336#D-
337function IGCM_sys_IsFileArchived {
338  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
339  if ( $DEBUG_sys ) ; then
340    echo "IGCM_sys_IsFileArchived :" $@
341  fi
342  typeset IsArchivedFlag
343  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
344  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
345
346  return ${IsArchivedFlag}
347}
348
349#D-#==================================================
350#D-function IGCM_sys_TestFileArchive
351#D-* Purpose: Test file that must NOT EXISTS on Archive
352#D-* Examples:
353#D-
354function IGCM_sys_TestFileArchive {
355  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
356  typeset ExistFlag
357  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
358  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
359
360  return ${ExistFlag}
361}
362
363#D-#==================================================
364#D-function IGCM_sys_CountFileArchive
365#D-* Purpose: Count files on Archive filesystem
366#D-* Examples:
367#D-
368function IGCM_sys_CountFileArchive {
369  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
370  ls ${@} 2>/dev/null | wc -l
371  if [ $? -gt 0 ] ; then
372    echo "IGCM_sys_CountFileArchive : erreur."
373  fi
374  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
375}
376
377#D-#==================================================
378#D-function IGCM_sys_Tree
379#D-* Purpose: Tree directories with files on ${ARCHIVE}
380#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
381#D-
382function IGCM_sys_Tree {
383  IGCM_debug_PushStack "IGCM_sys_Tree" $@
384  if ( $DEBUG_sys ) ; then
385    echo "IGCM_sys_Tree :" $@
386  fi
387
388  \ls -lR ${@}
389
390  IGCM_debug_PopStack "IGCM_sys_Tree"
391}
392
393#D-#==================================================
394#D-function IGCM_sys_Qsub
395#D-* Purpose: Qsub new job
396#D-* Examples:
397#D-
398function IGCM_sys_Qsub {
399  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
400  if ( $DEBUG_sys ) ; then
401    echo "IGCM_sys_Qsub :" $@
402  fi
403  typeset options status
404  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
405
406  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
407  status=$?
408
409  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
410  if [ ${status} -gt 0 ] ; then
411    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
412    IGCM_debug_Exit "IGCM_sys_Qsub"
413  else
414    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
415  fi
416  IGCM_debug_PopStack "IGCM_sys_Qsub"
417}
418
419#D-#==================================================
420#D-function IGCM_sys_QsubPost
421#D-* Purpose: Qsub new job on scalaire
422#D-* Examples:
423#D-
424function IGCM_sys_QsubPost {
425  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
426  if ( $DEBUG_sys ) ; then
427    echo "IGCM_sys_QsubPost :" $@
428  fi
429  typeset options status
430
431  # EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
432  case $( hostname -s ) in
433  curie*)
434    options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
435  airain*)
436    options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
437  esac
438
439  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
440  status=$?
441
442  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
443  if [ ${status} -gt 0 ] ; then
444    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
445    IGCM_debug_Exit "IGCM_sys_QsubPost"
446  else
447    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
448  fi
449  IGCM_debug_PopStack "IGCM_sys_QsubPost"
450}
451
452#D-*************************
453#D- File transfer functions
454#D-*************************
455#D-
456
457#D-#==================================================
458#D-function IGCM_sys_RmRunDir
459#D-* Purpose: rm tmpdir (dummy function most of the time batch
460#D-                      scheduler will do the job)
461#D-* Examples:
462#D-
463function IGCM_sys_RmRunDir {
464  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
465  if ( $DEBUG_sys ) ; then
466    echo "IGCM_sys_RmRunDir :" $@
467  fi
468
469  typeset status
470
471  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
472  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
473  status=$?
474
475  if [ ${status} -gt 0 ] ; then
476    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
477    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
478    IGCM_debug_Exit "IGCM_sys_RmRunDir"
479  else
480    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
481  fi
482  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
483}
484
485#D-#==================================================
486#D-function IGCM_sys_Put_Dir
487#D-* Purpose: Copy a complete directory on $(ARCHIVE)
488#D-* Examples:
489#D-
490function IGCM_sys_Put_Dir {
491  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
492  if ( $DEBUG_sys ) ; then
493    echo "IGCM_sys_Put_Dir :" $@
494  fi
495  if [ $DRYRUN = 0 ]; then
496    if [ ! -d ${1} ] ; then
497      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
498      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
499      return
500    fi
501
502    typeset status
503
504    # Only if we use rsync
505    #IGCM_sys_TestDirArchive $( dirname $2 )
506    #
507    #USUAL WAY
508    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
509    status=$?
510
511    if [ ${status} -gt 0 ] ; then
512      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
513      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
514      IGCM_debug_Exit "IGCM_sys_Put_Dir"
515    else
516      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
517    fi
518  fi
519  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
520}
521
522#D-#==================================================
523#D-function IGCM_sys_Get_Dir
524#D-* Purpose: Copy a complete directory from ${ARCHIVE}
525#D-* Examples:
526#D-
527function IGCM_sys_Get_Dir {
528  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
529  if ( $DEBUG_sys ) ; then
530    echo "IGCM_sys_Get_Dir :" $@
531  fi
532  if [ $DRYRUN = 0 ]; then
533    typeset NB_ESSAI DELAI status i
534    # number of tentative
535    NB_ESSAI=3
536    # time delay between tentative
537    DELAI=2
538
539    #
540    # USUAL WAY
541    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
542    ccc_hsm get -r $1
543
544    i=0
545    while [ $i -lt $NB_ESSAI ] ; do
546      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
547      status=$?
548      if [ ${status} -gt 0 ] ; then
549        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
550        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
551        sleep $DELAI
552      else
553        break
554      fi
555      (( i = i + 1 ))
556    done
557
558    if [ ${status} -gt 0 ] ; then
559      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
560      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
561      IGCM_debug_Exit "IGCM_sys_Get_Dir"
562    else
563      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
564    fi
565  fi
566  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
567}
568
569#D-#==================================================
570#D-function IGCM_sys_Put_Rest
571#D-* Purpose: Put computied restarts on ${ARCHIVE}.
572#D-           File and target directory must exist.
573#D-* Examples:
574#D-
575function IGCM_sys_Put_Rest {
576  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
577  if ( $DEBUG_sys ) ; then
578    echo "IGCM_sys_Put_Rest :" $@
579  fi
580  if [ $DRYRUN = 0 ]; then
581    if [ ! -f ${1} ] ; then
582      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
583      IGCM_debug_Exit "IGCM_sys_Put_Rest"
584    fi
585
586    typeset status
587    #
588    # USUAL WAY
589    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
590    status=$?
591
592#       #RSYNC WITH NETWORK SSH CALL
593#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
594#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
595
596#       #RSYNC WITH NFS USE
597#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
598#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
599
600#       status=$?
601#       IGCM_sys_Rsync_out $status
602
603#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
604#       (( status=status+$? ))
605
606    if [ ${status} -gt 0 ] ; then
607      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
608      [ -f ${1} ] && ls -l ${1}
609      [ -f ${2} ] && ls -l ${2}
610      [ -f ${2}/${1} ] && ls -l ${2}/${1}
611      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
612      IGCM_debug_Exit "IGCM_sys_Put_Rest"
613    else
614
615      if [ X${JobType} = XRUN ] ; then
616        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
617        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
618      fi
619
620      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
621    fi
622  fi
623  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
624}
625
626#D-#==================================================
627#D-function IGCM_sys_Put_Out
628#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
629#D-* Examples:
630#D-
631function IGCM_sys_Put_Out {
632  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
633  if ( $DEBUG_sys ) ; then
634    echo "IGCM_sys_Put_Out :" $@
635  fi
636
637  typeset NB_ESSAI DELAI status i exist skip
638
639  # number of tentative
640  NB_ESSAI=3
641  # time delay between tentative
642  DELAI=2
643
644  if [ $DRYRUN = 0 ]; then
645    if [ ! -f ${1} ] ; then
646      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
647      IGCM_debug_PopStack "IGCM_sys_Put_Out"
648      return 1
649    fi
650    #
651    IGCM_sys_MkdirArchive $( dirname $2 )
652    #
653    exist=false
654    skip=false
655    if [ -f $2 ] ; then
656      IGCM_debug_Print 1 "$2 already exist"
657      ccc_hsm get $2
658      exist=true
659      if [ "X$( diff $1 $2 )" = X ] ; then
660        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
661        skip=true
662      else
663        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
664        skip=false
665      fi
666    fi
667    #
668    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
669      IGCM_sys_Chmod u+w $2
670    fi
671
672    if [ X${skip} = Xfalse ] ; then
673      i=0
674      while [ $i -lt $NB_ESSAI ] ; do
675        if [ $( stat -c %d $1 ) -ne $( stat -c %d $( dirname $2 ) ) ] ; then
676          # USUAL WAY
677          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
678          status=$?
679        else
680          # NOT SO USUAL WAY
681          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
682          status=$?
683        fi
684        if [ ${status} -gt 0 ]; then
685          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
686          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
687          [ -f ${1} ] && ls -l ${1}
688          [ -f ${2} ] && ls -l ${2}
689          [ -f ${2}/${1} ] && ls -l ${2}/${1}
690          sleep $DELAI
691        else
692          break
693        fi
694        (( i = i + 1 ))
695      done
696    fi
697
698#       #RSYNC WITH NETWORK SSH CALL
699#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
700#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
701
702#       #RSYNC WITH NFS USE
703#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
704#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
705
706#       status=$?
707#       IGCM_sys_Rsync_out $status
708
709#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
710#       (( status=status+$? ))
711
712    if [ ${status} -gt 0 ] ; then
713      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
714      [ -f ${1} ] && ls -l ${1}
715      [ -f ${2} ] && ls -l ${2}
716      [ -f ${2}/${1} ] && ls -l ${2}/${1}
717      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
718      IGCM_debug_Exit "IGCM_sys_Put_Out"
719    else
720
721      if [ X${JobType} = XRUN ] ; then
722        if [ X${3} = X ] ; then
723          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
724          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
725        fi
726      fi
727
728      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
729    fi
730  fi
731  IGCM_debug_PopStack "IGCM_sys_Put_Out"
732  return 0
733}
734
735#D-#==================================================
736#D-function IGCM_sys_Get
737#D-* Purpose: Get a file from ${ARCHIVE}
738#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
739#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
740function IGCM_sys_Get {
741  IGCM_debug_PushStack "IGCM_sys_Get" $@
742
743  typeset DEST dm_liste target file_work
744  typeset NB_ESSAI DELAI status i
745
746  if ( $DEBUG_sys ) ; then
747    echo "IGCM_sys_Get :" $@
748  fi
749
750  # number of tentative
751  NB_ESSAI=3
752  # time delay between tentative
753  DELAI=2
754
755  if [ $DRYRUN -le 2 ]; then
756    if [ X${1} = X'/l' ] ; then
757      eval set +A dm_liste \${${2}}
758    else
759      eval set +A dm_liste ${1}
760    fi
761    eval DEST=\${${#}}
762    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
763    status=$?
764    if [ ${status} -gt 0 ] ; then
765      echo "WARNING IGCM_sys_Get : error code ${status}"
766      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
767      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
768    fi
769
770    #if [ ${status} -gt 0 ] ; then
771    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
772    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
773    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
774    #    sleep 30
775    #    echo "We try another time"
776    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
777    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
778    #    status=$?
779    #    if [ ${status} -gt 0 ] ; then
780    #      echo "ERROR IGCM_sys_Get : again demigration error :"
781    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
782    #      IGCM_debug_Exit "IGCM_sys_Get"
783    #    fi
784    #  else
785    #    echo "ERROR IGCM_sys_Get : demigration error :"
786    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
787    #    IGCM_debug_Exit "IGCM_sys_Get"
788    #  fi
789    #fi
790
791    #   #RSYNC WITH NETWORK SSH CALL
792    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
793    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
794
795    #   #RSYNC WITH NFS USE
796    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
797    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
798
799    #   status=$?
800    #   IGCM_sys_Rsync_out $status
801
802    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
803    #   (( status=status+$? ))
804
805    #USUAL WAY
806    if [ X${1} = X'/l' ] ; then
807      for target in ${dm_liste[*]} ; do
808        local_file=$( basename ${target} )
809        # test if the target file is present before the loop
810        IGCM_sys_TestFileArchive ${target}
811        status=$?
812        if [ ${status} -gt 0 ] ; then
813          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
814          IGCM_debug_Exit "IGCM_sys_Get"
815        else
816          i=0
817          while [ $i -lt $NB_ESSAI ] ; do
818            #if [ X${DoLink} = Xtrue ] ; then
819            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
820            #  status=$?
821            #  else
822            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
823            #  status=$?
824            #fi
825            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
826            status=$?
827            if [ ${status} -gt 0 ]; then
828              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
829              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
830              sleep $DELAI
831            else
832              break
833            fi
834            (( i = i + 1 ))
835          done
836          if [ ${status} -gt 0 ] ; then
837            echo "IGCM_sys_Get : error"
838            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
839            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
840            IGCM_debug_Exit "IGCM_sys_Get"
841          else
842            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
843          fi
844        fi
845      done
846    else
847      i=0
848      while [ $i -lt $NB_ESSAI ] ; do
849        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
850        status=$?
851        if [ ${status} -gt 0 ]; then
852          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
853          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
854          sleep $DELAI
855        else
856          break
857        fi
858        (( i = i + 1 ))
859      done
860      if [ ${status} -gt 0 ] ; then
861        echo "IGCM_sys_Get : error"
862        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
863        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
864        IGCM_debug_Exit "IGCM_sys_Get"
865      else
866        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
867      fi
868    fi
869  fi
870  IGCM_debug_PopStack "IGCM_sys_Get"
871}
872
873#D-#==================================================
874#D-function IGCM_sys_GetDate_Monitoring
875#D-* Purpose: get the last year for which the monitoring has been computed
876#D-* Examples:
877#D-
878function IGCM_sys_GetDate_Monitoring {
879  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
880  if ( $DEBUG_sys ) ; then
881    echo "IGCM_sys_GetDate_Monitoring :" $@
882  fi
883
884  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
885
886  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
887}
888
889#D-#==================================================
890#D-function IGCM_sys_Dods_Rm
891#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
892#D-* Examples:
893#D-
894function IGCM_sys_Dods_Rm {
895  if ( $DEBUG_sys ) ; then
896    echo "IGCM_sys_Dods_Rm :" $@
897  fi
898  typeset status
899  if [ $DRYRUN = 0 ]; then
900
901#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
902#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
903#      echo "Nothing has been done."
904#      return
905#    fi
906
907    /ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
908    status=$?
909
910#    if [ ${status} -gt 0 ] ; then
911#      echo "IGCM_sys_Dods_Rm : error."
912#      cat out_dods_rm
913#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
914#    else
915#      rm out_dods_rm
916#    fi
917
918  fi
919  return $status
920}
921
922#D-#==================================================
923#D-function IGCM_sys_Dods_Cp
924#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
925#D-* Examples:
926#D-
927function IGCM_sys_Dods_Cp {
928  if ( $DEBUG_sys ) ; then
929    echo "IGCM_sys_Dods_Cp :" $@
930  fi
931  typeset status
932  if [ $DRYRUN = 0 ]; then
933
934#    if [ ! -d ${R_SAVE}/${1} ] ; then
935#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
936#      echo "Nothing has been done."
937#      return
938#    fi
939
940    /ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
941    status=$?
942
943#       if [ ${status} -gt 0 ] ; then
944#           echo "IGCM_sys_Dods_Cp : error."
945#           cat out_dods_cp
946#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
947#       else
948#           rm out_dods_cp
949#       fi
950
951  fi
952  return $status
953}
954
955#D-#==================================================
956#D-function IGCM_sys_Put_Dods
957#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
958#D-* Examples:
959#D-
960function IGCM_sys_Put_Dods {
961  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
962  if ( $DEBUG_sys ) ; then
963    echo "IGCM_sys_Put_Dods :" $@
964  fi
965  typeset status
966  if [ $DRYRUN = 0 ]; then
967    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
968      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
969      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
970      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
971      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
972      return
973    fi
974
975    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
976      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
977      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
978      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
979      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
980      return
981    fi
982    #
983    if [ -d ${R_SAVE}/${1} ] ; then
984      cd ${R_SAVE}
985    elif [ -d ${R_FIGR}/${1} ] ; then
986      cd ${R_FIGR}
987    fi
988
989    IGCM_sys_Dods_Rm ${1}
990    IGCM_sys_Dods_Cp ${1}
991    status=0
992
993    if [ ${status} -gt 0 ] ; then
994      echo "IGCM_sys_Put_Dods : error."
995      IGCM_debug_Exit "IGCM_sys_Put_Dods"
996    fi
997  fi
998  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
999}
1000
1001##############################################################
1002# REBUILD OPERATOR
1003
1004#D-#==================================================
1005#D-function IGCM_sys_sync
1006#D-* Purpose: flush buffer on disk
1007#D-* Examples:
1008#D-
1009function IGCM_sys_sync {
1010  IGCM_debug_PushStack "IGCM_sys_sync" $@
1011  if ( $DEBUG_sys ) ; then
1012    echo "IGCM_sys_sync :" $@
1013  fi
1014
1015  /bin/sync
1016
1017  IGCM_debug_PopStack "IGCM_sys_sync"
1018}
1019
1020############################################################
1021# Activate Running Environnment Variables
1022
1023#D-#==================================================
1024#D-function IGCM_sys_desactiv_variables
1025#D-* Purpose: set environement variables prior to execution
1026#D-* Examples:
1027#D-
1028function IGCM_sys_activ_variables {
1029  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1030  if ( $DEBUG_sys ) ; then
1031    echo "IGCM_sys_activ_variables"
1032  fi
1033
1034# --------------------------------------------------------------------
1035#D- MPI specifications
1036# --------------------------------------------------------------------
1037
1038# --------------------------------------------------------------------
1039#D- Other specifications
1040# --------------------------------------------------------------------
1041
1042  ulimit -s unlimited
1043
1044  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1045}
1046
1047############################################################
1048# Desactivate Running Environnment Variables
1049
1050#D-#==================================================
1051#D-function IGCM_sys_desactiv_variables
1052#D-* Purpose: unset environement variables after execution
1053#D-* Examples:
1054#D-
1055function IGCM_sys_desactiv_variables {
1056  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1057  if ( $DEBUG_sys ) ; then
1058    echo "IGCM_sys_desactiv_variables"
1059  fi
1060# --------------------------------------------------------------------
1061#D- MPI specifications
1062# --------------------------------------------------------------------
1063
1064# --------------------------------------------------------------------
1065#D- Other specifications
1066# --------------------------------------------------------------------
1067
1068  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1069}
1070
1071############################################################
1072# Build MPI/OMP scripts run file (dummy function)
1073
1074#D-#==================================================
1075#D-function IGCM_sys_build_run_file
1076#D-* Purpose: build run file (deprecated)
1077#D-* Examples:
1078#D-
1079function IGCM_sys_build_run_file {
1080
1081  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1082
1083}
1084
1085############################################################
1086# Build MPI/OMP scripts
1087
1088#D-#==================================================
1089#D-function IGCM_sys_build_execution_scripts
1090#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1091#D-* Examples:
1092#D-
1093function IGCM_sys_build_execution_scripts
1094{
1095  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1096  if ( $DEBUG_sys ) ; then
1097    echo "IGCM_sys_build_execution_scripts " $@
1098  fi
1099
1100  EXECUTION=${HOST_MPIRUN_COMMAND}
1101
1102  # MPMD mode
1103  if ( ${OK_PARA_MPMD} ) ; then
1104
1105    # Only MPI (MPMD)
1106    if  ( ! ${OK_PARA_OMP} ) ; then
1107
1108      if [ -f run_file ] ; then
1109        IGCM_sys_Rm -f run_file
1110      fi
1111      touch run_file
1112
1113      # Build run_file
1114
1115      # First loop on the components for the coupler ie oasis (only if oasis3)
1116      # the coupler ie oasis3 must be the first one
1117      for comp in ${config_ListOfComponents[*]} ; do
1118
1119        eval ExeNameIn=\${config_Executable_${comp}[0]}
1120        eval ExeNameOut=\${config_Executable_${comp}[1]}
1121
1122        # for CPL component only
1123        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1124
1125          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1126          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1127          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1128        fi
1129      done
1130
1131      # Then second loop on the components
1132      for comp in ${config_ListOfComponents[*]} ; do
1133
1134        eval ExeNameIn=\${config_Executable_${comp}[0]}
1135        eval ExeNameOut=\${config_Executable_${comp}[1]}
1136
1137        # Only if we really have an executable for the component and not the coupler ie oasis:
1138        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1139
1140          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1141          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1142          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1143        fi
1144      done
1145
1146      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1147
1148      IGCM_sys_Chmod u+x run_file
1149      if ( $DEBUG_sys ) ; then
1150        echo "run_file contains : "
1151        cat run_file
1152      fi
1153
1154    # MPI-OpenMP (MPMD)
1155    else
1156
1157      # Use of mpirun instead of ccc_mprun
1158      EXECUTION="time mpirun"
1159
1160      #  Hosts treatment
1161      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1162
1163      i=0
1164      rm -f hosts rankfile
1165      IGCM_debug_Print 1 "sys Curie, Hosts available :"
1166      for nodes in `cat hosts.tmp`
1167      do
1168        host[$i]=$nodes
1169        echo "${host[$i]}" >> hosts
1170        IGCM_debug_Print 1 ${host[$i]}
1171        i=$((i+1))
1172      done
1173      rm -f hosts.tmp
1174
1175      listnodes=${host[*]}
1176
1177      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1178
1179      # Initialisation
1180      rank=0
1181      current_core=0
1182      core_per_node=16
1183      init_exec=n
1184
1185      # Loop on the components
1186      for comp in ${config_ListOfComponents[*]} ; do
1187
1188        eval ExeNameIn=\${config_Executable_${comp}[0]}
1189        eval ExeNameOut=\${config_Executable_${comp}[1]}
1190
1191        # Not possible if oasis has an executable (i.e old version of oasis3)
1192        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1193          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1194          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1195          IGCM_debug_Verif_Exit
1196        fi
1197
1198        # Only if we really have an executable for the component :
1199        if [ "X${ExeNameOut}" != X\"\" ] ; then
1200
1201          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1202          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1203
1204          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1205          echo ""  >> script_${ExeNameOut}.ksh
1206          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1207
1208            # Check if the number of threads is correct
1209            case ${comp_proc_omp_loc} in
1210            2|4|8|16)
1211              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1212              ;;
1213            *)
1214              IGCM_debug_Exit "ERROR with OMP parameters !"
1215              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1216              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1217              IGCM_debug_Verif_Exit
1218              ;;
1219            esac
1220            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1221            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1222            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1223            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1224          fi
1225          #echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
1226          #echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
1227          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1228          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1229
1230          if [ ${init_exec} = y ] ; then
1231            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1232          else
1233            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1234            init_exec=y
1235          fi
1236
1237# Build rankfile : method used to assign cores and nodes for the MPI process
1238# Ex :
1239#rank 0=curie5296 slot=0,1,2,3
1240#rank 1=curie5296 slot=4,5,6,7
1241# Example of final command :
1242# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1243# with script_lmdz.x.ksh :
1244# #!/bin/ksh
1245#export KMP_STACKSIZE=3g
1246#export KMP_LIBRARY=turnaround
1247#export MKL_SERIAL=YES
1248#OMP_NUM_THREADS=4
1249#./lmdz.x
1250#
1251
1252          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1253            (( index_host = current_core / core_per_node ))
1254            host_value=${host[${index_host}]}
1255            (( slot =  current_core % core_per_node ))
1256            virg=","
1257            string_final=""
1258            for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1259              string=$index$virg
1260              string_final=$string_final$string
1261            done
1262            string_final=$( echo $string_final | sed "s/.$//" )
1263            echo "rank $rank=$host_value slot=$string_final" >> rankfile
1264            (( rank = rank + 1 ))
1265            (( current_core = current_core + comp_proc_omp_loc ))
1266          done
1267        fi
1268
1269      done
1270    fi
1271
1272  # Only one executable (SPMD mode).
1273  else
1274
1275    for comp in ${config_ListOfComponents[*]} ; do
1276
1277      # Only if we really have an executable for the component :
1278      eval ExeNameOut=\${config_Executable_${comp}[1]}
1279      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1280
1281        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1282        echo ""  >> script_${ExeNameOut}.ksh
1283        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1284
1285        if ( ${OK_PARA_OMP} ) ; then
1286          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1287          echo ""  >> script_${ExeNameOut}.ksh
1288          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1289          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1290          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1291          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1292        fi
1293
1294        if  ( ${OK_PARA_MPI} ) ; then
1295          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1296          # Default : ccc_mprun used if nb_proc gt 1
1297          # to have out/err per process on different files
1298          # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1299          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1300          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1301        else
1302          # Default : ccc_mprun is NOT used if nb_proc eq 1
1303          # to have out/err per process on different files
1304          # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1305          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1306          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1307        fi
1308
1309        IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
1310        cat script_${ExeNameOut}.ksh
1311
1312      fi
1313
1314    done
1315
1316  fi
1317
1318  IGCM_debug_Print 1 "sys Curie : execution command is "
1319  IGCM_debug_Print 1 "$EXECUTION"
1320
1321  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1322}
1323
1324#D-#==================================================
1325#D-function IGCM_sys_check_path
1326#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1327#D-* do not point to an important use directory. Stop immediately in that case.
1328#D-* Examples:
1329#D-
1330function IGCM_sys_check_path {
1331  IGCM_debug_PushStack "IGCM_sys_check_path"
1332  if ( $DEBUG_sys ) ; then
1333    echo "IGCM_sys_check_path"
1334  fi
1335
1336  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORKDIR} ] || [ X${RUN_DIR_PATH} = X${SCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1337    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1338    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1339    IGCM_debug_Exit "This will stop the job"
1340  fi
1341  IGCM_debug_PopStack "IGCM_sys_check_path"
1342}
1343
1344#D-#==================================================
1345#D-function IGCM_sys_check_quota
1346#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1347#D-* Examples:
1348#D-
1349function IGCM_sys_check_quota {
1350  IGCM_debug_PushStack "IGCM_sys_check_quota"
1351  if ( $DEBUG_sys ) ; then
1352    echo "IGCM_sys_check_quota"
1353  fi
1354  # Limit of quota (in %)
1355  limit_quota=90
1356
1357  # Check of the volume
1358  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1359  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1360
1361  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1362
1363    unit_avail=${volume_avail: -1}
1364    unit_quota=${volume_quota: -1}
1365
1366    if [ "${unit_quota}" = "*" ] ; then
1367      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1368      IGCM_debug_Print 1 "More than 100% of your quota is used"
1369      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1370      IGCM_debug_Print 1 "You must have more than 10% available to run"
1371      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1372      IGCM_debug_Verif_Exit
1373    fi
1374
1375    temp_avail=${volume_avail%%${volume_avail: -1}*}
1376    temp_quota=${volume_quota%%${volume_quota: -1}*}
1377
1378    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1379
1380    # Convertion
1381      if [ ${volume_avail: -1} = "T" ] ; then
1382        (( temp_avail = temp_avail * 1000000000000 ))
1383      elif [ ${volume_avail: -1} = "G" ] ; then
1384        (( temp_avail = temp_avail * 1000000000 ))
1385      elif [ ${volume_avail: -1} = "M" ] ; then
1386        (( temp_avail = temp_avail * 1000000 ))
1387      elif [ ${volume_avail: -1} = "k" ] ; then
1388        (( temp_avail = temp_avail * 1000 ))
1389      else
1390        (( temp_avail = volume_avail ))
1391      fi
1392      if [ ${volume_quota: -1} = "T" ] ; then
1393        (( temp_quota = temp_quota * 1000000000000 ))
1394      elif [ ${volume_quota: -1} = "G" ] ; then
1395        (( temp_quota = temp_quota * 1000000000 ))
1396      elif [ ${volume_quota: -1} = "M" ] ; then
1397        (( temp_quota = temp_quota * 1000000 ))
1398      elif [ ${volume_quota: -1} = "k" ] ; then
1399        (( temp_quota = temp_quota * 1000 ))
1400      else
1401        (( temp_quota = volume_quota ))
1402      fi
1403    fi
1404
1405    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1406#    echo "volume ratio is " $quota_volume
1407
1408    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1409      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1410      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1411      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1412      IGCM_debug_Print 1 "You must have more than 10% available to run"
1413      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1414      IGCM_debug_Verif_Exit
1415    fi
1416
1417  fi
1418
1419# Check of the number of inodes
1420
1421  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1422  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1423
1424  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1425
1426    unit_avail=${inode_avail: -1}
1427    unit_quota=${inode_quota: -1}
1428
1429    if [ "${unit_quota}" = "*" ] ; then
1430      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1431      IGCM_debug_Print 1 "More than 100% of your quota is used"
1432      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1433      IGCM_debug_Print 1 "You must have more than 10% available to run"
1434      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1435      IGCM_debug_Verif_Exit
1436    fi
1437
1438    temp_avail=${inode_avail%%${inode_avail: -1}*}
1439    temp_quota=${inode_quota%%${inode_quota: -1}*}
1440
1441    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1442
1443    # Convertion
1444      if [ ${inode_avail: -1} = "T" ] ; then
1445        (( temp_avail = temp_avail * 1000000000000 ))
1446      elif [ ${inode_avail: -1} = "G" ] ; then
1447        (( temp_avail = temp_avail * 1000000000 ))
1448      elif [ ${inode_avail: -1} = "M" ] ; then
1449        (( temp_avail = temp_avail * 1000000 ))
1450      elif [ ${inode_avail: -1} = "k" ] ; then
1451        (( temp_avail = temp_avail * 1000 ))
1452      else
1453        (( temp_avail = inode_avail ))
1454      fi
1455
1456      if [ ${inode_quota: -1} = "T" ] ; then
1457        (( temp_quota = temp_quota * 1000000000000 ))
1458      elif [ ${inode_quota: -1} = "G" ] ; then
1459        (( temp_quota = temp_quota * 1000000000 ))
1460      elif [ ${inode_quota: -1} = "M" ] ; then
1461        (( temp_quota = temp_quota * 1000000 ))
1462      elif [ ${inode_quota: -1} = "k" ] ; then
1463        (( temp_quota = temp_quota * 1000 ))
1464      else
1465        (( temp_quota = inode_quota ))
1466      fi
1467    fi
1468    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1469#    echo "inode ratio is " $quota_inode
1470
1471    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1472      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1473      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1474      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1475      IGCM_debug_Print 1 "You must have more than 10% available to run"
1476      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1477      IGCM_debug_Verif_Exit
1478    fi
1479  fi
1480  IGCM_debug_PopStack "IGCM_sys_check_quota"
1481}
1482
1483#D-#==================================================
1484#D-function IGCM_sys_GetJobID
1485#D-* Purpose: Check if job_name is currently
1486#D-  running or in queue
1487#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1488#D-
1489function IGCM_sys_GetJobID {
1490  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1491  if ( $DEBUG_sys ) ; then
1492    echo "IGCM_sys_GetJobID"
1493  fi
1494
1495  # With -f option, the full job name is given in the last column
1496  ID=$( ccc_mstat -f -u $2 | \
1497        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1498
1499  eval ${3}=${ID}
1500  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1501}
1502
1503#D-#==================================================
1504#D-function IGCM_sys_CountJobInQueue
1505#D-* Purpose: Check if job_name is currently
1506#D-  running or in queue
1507#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1508#D-
1509function IGCM_sys_CountJobInQueue {
1510  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1511  if ( $DEBUG_sys ) ; then
1512    echo "IGCM_sys_CountJobInQueue"
1513  fi
1514
1515  # With -f option, the full job name is given in the last column
1516  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1517
1518  eval ${2}=${NbRun}
1519
1520  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1521}
1522
1523#D-#==================================================
1524#D-function IGCM_sys_ListJobInQueue
1525#D-* Purpose: Check if job_name is currently
1526#D-  running or in queue
1527#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1528#D-
1529function IGCM_sys_ListJobInQueue {
1530  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1531  if ( $DEBUG_sys ) ; then
1532    echo "IGCM_sys_ListJobInQueue"
1533  fi
1534
1535  # With -f option, the full job name is given in the last column
1536  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1537                                        '( $2  == User      && \
1538                                           $NF != /TS/      && \
1539                                           $NF !~ /PACK/    && \
1540                                           $NF !~ /REBUILD/ && \
1541                                           $NF !~ /pack/ )     \
1542                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1543
1544  eval set -A ${2} ${JobList[*]}
1545
1546  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1547}
Note: See TracBrowser for help on using the repository browser.