source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 1204

Last change on this file since 1204 was 1201, checked in by sdipsl, 9 years ago
  • add jobWarningDelay in seconds to supervision messages see #253 (threshold for heartbeat)
  • add accountingProject to supervision messages see #255 (may ease gencmip6 data production tracking)
  • Property licence set to
    The following licence information concerns ONLY the libIGCM tools
    ==================================================================

    Copyright © Centre National de la Recherche Scientifique CNRS
    Commissariat à l'Énergie Atomique CEA

    libIGCM : Library for Portable Models Computation of IGCM Group.

    IGCM Group is the french IPSL Global Climate Model Group.

    This library is a set of shell scripts and functions whose purpose is
    the management of the initialization, the launch, the transfer of
    output files, the post-processing and the monitoring of datas produce
    by any numerical program on any plateforme.

    This software is governed by the CeCILL license under French law and
    abiding by the rules of distribution of free software. You can use,
    modify and/ or redistribute the software under the terms of the CeCILL
    license as circulated by CEA, CNRS and INRIA at the following URL
    "http://www.cecill.info".

    As a counterpart to the access to the source code and rights to copy,
    modify and redistribute granted by the license, users are provided only
    with a limited warranty and the software's author, the holder of the
    economic rights, and the successive licensors have only limited
    liability.

    In this respect, the user's attention is drawn to the risks associated
    with loading, using, modifying and/or developing or reproducing the
    software by the user in light of its specific status of free software,
    that may mean that it is complicated to manipulate, and that also
    therefore means that it is reserved for developers and experienced
    professionals having in-depth computer knowledge. Users are therefore
    encouraged to load and test the software's suitability as regards their
    requirements in conditions enabling the security of their systems and/or
    data to be ensured and, more generally, to use and operate it in the
    same conditions as regards security.

    The fact that you are presently reading this means that you have had
    knowledge of the CeCILL license and that you accept its terms.
  • Property svn:keywords set to Revision Author Date
File size: 51.1 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Curie
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host user names project maxCpuTime
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=curie
71# add default project on curie
72typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} | cut -d@ -f1 )
73# jobWarningDelay in seconds
74typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=ccc_msub}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Set environment tools (ferret, nco, cdo, rebuild, ...)
93#====================================================
94if [ X${TaskType} = Xcomputing ] ; then
95  . /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4.3.3.1_hdf5_parallel_curie_ksh > /dev/null 2>&1
96# to run with netcdf 3.6.3 ie compilation done before 17/2/2014
97# uncomment 2 lines :
98#  module unload netcdf
99#  module load netcdf/3.6.3
100else
101  . /ccc/cont003/home/dsm/p86ipsl/.atlas_env_netcdf4.3.3.1_hdf5_parallel_curie_ksh > /dev/null 2>&1
102  PCMDI_MP=/ccc/work/cont003/dsm/p86ipsl/PCMDI-MP
103fi
104export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddNoise/src_X64_CURIE/bin
105export PATH=${PATH}:/ccc/cont003/home/dsm/p86ipsl/AddPerturbation/src_X64_CURIE/bin
106export PATH=${PATH}:$( ccc_home -u p86ipsl )/rebuild/src_X64_CURIE/modipsl_v2_2_2_netcdf4.2/bin/
107
108#====================================================
109# Specific for ocean additionnal diagnostic
110export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
111export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
112
113#====================================================
114# Host specific DIRECTORIES
115#====================================================
116
117# ============ CESIUM START ============ #
118
119#====================================================
120#- Mirror libIGCM from titane to cesium if needed
121#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
122#if [ ! ${ROOTSYS} = "home" ] ; then
123#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
124#else
125#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
126#fi
127
128#====================================================
129#- libIGCM_POST
130#if ( ${MirrorlibIGCM} ) ; then
131#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
132#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
133#else
134#  typeset -r libIGCM_POST=${libIGCM}
135#fi
136
137# ============ CESIUM  END  ============ #
138
139#====================================================
140#- MirrorlibIGCM for frontend
141typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
142
143#====================================================
144#- libIGCM_POST for frontend
145typeset -r libIGCM_POST=${libIGCM}
146
147#====================================================
148#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
149typeset -r R_EXE="${MODIPSL}/bin"
150
151#====================================================
152#- SUBMIT_DIR : submission dir
153typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
154
155#====================================================
156#- IN
157typeset -r R_IN=${R_IN:=/ccc/work/cont003/dsm/p86ipsl/IGCM}
158typeset -r R_IN_ECMWF=${R_IN_ECMWF:=/ccc/work/cont003/dsm/p24data}
159
160#====================================================
161#- RUN_DIR_PATH : Temporary working directory (=> TMP)
162typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
163
164#====================================================
165#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
166typeset -r OUTCOMMAND_PATH=/tmp
167
168#====================================================
169#- HOST_MPIRUN_COMMAND
170typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1"}
171
172#====================================================
173#- Max number of arguments passed to nco operator or demigration command
174UNIX_MAX_LIMIT=360
175
176#====================================================
177#- set PackDefault to true on curie
178PackDefault=true
179
180#====================================================
181#- Number of core per node (max number of OpenMP task)
182NUM_COREPERNODE=8
183
184#====================================================
185#- Default number of MPI task for IPSL coupled model
186#- required for backward compatibility
187#-
188DEFAULT_NUM_PROC_OCE=5
189DEFAULT_NUM_PROC_CPL=1
190(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
191DEFAULT_NUM_PROC_TOTAL=${BATCH_NUM_PROC_TOT}
192
193#D-#==================================================
194#D-function IGCM_sys_defineArchives
195#D-* Purpose:
196#D-* Define ARCHIVE : Dedicated to large files
197#D-* Define STORAGE : Dedicated to small/medium files
198#D-* Define R_OUT   : Output tree located on ARCHIVE
199#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
200#D-* Define R_BUF   : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
201#D-* if SpaceName=TEST everything is stored on SCRATCHDIR
202#D-* Examples:
203#D-
204function IGCM_sys_defineArchives {
205  IGCM_debug_PushStack "IGCM_sys_defineArchives"
206
207  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
208    #====================================================
209    #- ARCHIVE (dedicated to large files)
210    ARCHIVE=${config_UserChoices_ARCHIVE}
211  else
212    #====================================================
213    #- ARCHIVE (dedicated to large files)
214    ARCHIVE=${CCCSTOREDIR}
215  fi
216
217  if [ ! X${config_UserChoices_STORAGE} = X ]; then
218    #====================================================
219    #- STORAGE (dedicated to small/medium files)
220    STORAGE=${config_UserChoices_STORAGE}
221  else
222    #====================================================
223    #- STORAGE (dedicated to small/medium files)
224    STORAGE=${CCCWORKDIR}
225  fi
226
227  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
228    #====================================================
229    #- R_OUT
230    R_OUT=${SCRATCHDIR}/IGCM_OUT
231
232    #====================================================
233    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
234    R_FIG=${SCRATCHDIR}/IGCM_OUT
235
236    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
237
238  else
239    #====================================================
240    #- R_OUT
241    R_OUT=${ARCHIVE}/IGCM_OUT
242
243    #====================================================
244    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
245    R_FIG=${STORAGE}/IGCM_OUT
246  fi
247
248  #====================================================
249  #- R_BUF (ONLY FOR double copy an scratch)
250  R_BUF=${SCRATCHDIR}/IGCM_OUT
251
252  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
253  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
254  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
255
256  IGCM_debug_PopStack "IGCM_sys_defineArchives"
257}
258
259#D-#==================================================
260#D-function IGCM_sys_RshArchive
261#D-* Purpose: Archive rsh command
262#D-* Examples:
263#D-
264function IGCM_sys_RshArchive {
265  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
266  /bin/ksh <<-EOF
267    ${@}
268EOF
269  status=$?
270  if [ ${status} -gt 0 ] ; then
271    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
272    IGCM_debug_Exit "IGCM_sys_RshArchive"
273  fi
274  IGCM_debug_PopStack "IGCM_sys_RshArchive"
275}
276
277#D-#==================================================
278#D-function IGCM_sys_RshArchive_NoError
279#D-* Purpose: Archive rsh command, without error
280#D-*          used only in monitoring.job
281#D-* Examples:
282#D-
283function IGCM_sys_RshArchive_NoError {
284  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
285  /bin/ksh <<-EOF
286    ${@} 2> /dev/null
287EOF
288  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
289}
290
291#D-#==================================================
292#D-function IGCM_sys_MkdirArchive
293#D-* Purpose: Mkdir on Archive
294#D-* Examples:
295#D-
296function IGCM_sys_MkdirArchive {
297  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
298  if ( $DEBUG_sys ) ; then
299    echo "IGCM_sys_MkdirArchive :" $@
300  fi
301  #- creation de repertoire sur le serveur fichier
302  if [ ! -d ${1} ]; then
303    \mkdir -p $1
304    status=$?
305
306    if [ ${status} -gt 0 ] ; then
307      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
308      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
309    fi
310  fi
311  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
312}
313
314#D-#==================================================
315#D-function IGCM_sys_TestDirArchive
316#D-* Purpose: Test Directory that must exists on Archive
317#D-* Examples:
318#D-
319function IGCM_sys_TestDirArchive {
320  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
321  if ( $DEBUG_sys ) ; then
322    echo "IGCM_sys_TestDirArchive :" $@
323  fi
324  typeset ExistFlag
325  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
326  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
327  return ${ExistFlag}
328}
329
330#D-#==================================================
331#D-function IGCM_sys_IsFileArchived
332#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
333#D-* Examples:
334#D-
335function IGCM_sys_IsFileArchived {
336  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
337  if ( $DEBUG_sys ) ; then
338    echo "IGCM_sys_IsFileArchived :" $@
339  fi
340  typeset IsArchivedFlag
341  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
342  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
343
344  return ${IsArchivedFlag}
345}
346
347#D-#==================================================
348#D-function IGCM_sys_TestFileArchive
349#D-* Purpose: Test file that must NOT EXISTS on Archive
350#D-* Examples:
351#D-
352function IGCM_sys_TestFileArchive {
353  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
354  typeset ExistFlag
355  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
356  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
357
358  return ${ExistFlag}
359}
360
361#D-#==================================================
362#D-function IGCM_sys_CountFileArchive
363#D-* Purpose: Count files on Archive filesystem
364#D-* Examples:
365#D-
366function IGCM_sys_CountFileArchive {
367  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
368  ls ${@} 2>/dev/null | wc -l
369  if [ $? -gt 0 ] ; then
370    echo "IGCM_sys_CountFileArchive : erreur."
371  fi
372  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
373}
374
375#D-#==================================================
376#D-function IGCM_sys_Tree
377#D-* Purpose: Tree directories with files on ${ARCHIVE}
378#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
379#D-
380function IGCM_sys_Tree {
381  IGCM_debug_PushStack "IGCM_sys_Tree" $@
382  if ( $DEBUG_sys ) ; then
383    echo "IGCM_sys_Tree :" $@
384  fi
385
386  \ls -lR ${@}
387
388  IGCM_debug_PopStack "IGCM_sys_Tree"
389}
390
391#D-#==================================================
392#D-function IGCM_sys_Qsub
393#D-* Purpose: Qsub new job
394#D-* Examples:
395#D-
396function IGCM_sys_Qsub {
397  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
398  if ( $DEBUG_sys ) ; then
399    echo "IGCM_sys_Qsub :" $@
400  fi
401  typeset options status
402  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
403
404  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
405  status=$?
406
407  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
408  if [ ${status} -gt 0 ] ; then
409    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
410    IGCM_debug_Exit "IGCM_sys_Qsub"
411  else
412    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
413  fi
414  IGCM_debug_PopStack "IGCM_sys_Qsub"
415}
416
417#D-#==================================================
418#D-function IGCM_sys_QsubPost
419#D-* Purpose: Qsub new job on scalaire
420#D-* Examples:
421#D-
422function IGCM_sys_QsubPost {
423  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
424  if ( $DEBUG_sys ) ; then
425    echo "IGCM_sys_QsubPost :" $@
426  fi
427  typeset options status
428
429  # EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
430  case $( hostname -s ) in
431  curie*)
432    options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
433  airain*)
434    options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
435  esac
436
437  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
438  status=$?
439
440  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
441  if [ ${status} -gt 0 ] ; then
442    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
443    IGCM_debug_Exit "IGCM_sys_QsubPost"
444  else
445    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
446  fi
447  IGCM_debug_PopStack "IGCM_sys_QsubPost"
448}
449
450#D-*************************
451#D- File transfer functions
452#D-*************************
453#D-
454
455#D-#==================================================
456#D-function IGCM_sys_RmRunDir
457#D-* Purpose: rm tmpdir (dummy function most of the time batch
458#D-                      scheduler will do the job)
459#D-* Examples:
460#D-
461function IGCM_sys_RmRunDir {
462  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
463  if ( $DEBUG_sys ) ; then
464    echo "IGCM_sys_RmRunDir :" $@
465  fi
466
467  typeset status
468
469  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
470  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
471  status=$?
472
473  if [ ${status} -gt 0 ] ; then
474    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
475    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
476    IGCM_debug_Exit "IGCM_sys_RmRunDir"
477  else
478    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
479  fi
480  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
481}
482
483#D-#==================================================
484#D-function IGCM_sys_Put_Dir
485#D-* Purpose: Copy a complete directory on $(ARCHIVE)
486#D-* Examples:
487#D-
488function IGCM_sys_Put_Dir {
489  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
490  if ( $DEBUG_sys ) ; then
491    echo "IGCM_sys_Put_Dir :" $@
492  fi
493  if [ $DRYRUN = 0 ]; then
494    if [ ! -d ${1} ] ; then
495      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
496      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
497      return
498    fi
499
500    typeset status
501
502    # Only if we use rsync
503    #IGCM_sys_TestDirArchive $( dirname $2 )
504    #
505    #USUAL WAY
506    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
507    status=$?
508
509    if [ ${status} -gt 0 ] ; then
510      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
511      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
512      IGCM_debug_Exit "IGCM_sys_Put_Dir"
513    else
514      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
515    fi
516  fi
517  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
518}
519
520#D-#==================================================
521#D-function IGCM_sys_Get_Dir
522#D-* Purpose: Copy a complete directory from ${ARCHIVE}
523#D-* Examples:
524#D-
525function IGCM_sys_Get_Dir {
526  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
527  if ( $DEBUG_sys ) ; then
528    echo "IGCM_sys_Get_Dir :" $@
529  fi
530  if [ $DRYRUN = 0 ]; then
531    typeset NB_ESSAI DELAI status i
532    # number of tentative
533    NB_ESSAI=3
534    # time delay between tentative
535    DELAI=2
536
537    #
538    # USUAL WAY
539    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
540    ccc_hsm get -r $1
541
542    i=0
543    while [ $i -lt $NB_ESSAI ] ; do
544      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
545      status=$?
546      if [ ${status} -gt 0 ] ; then
547        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
548        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
549        sleep $DELAI
550      else
551        break
552      fi
553      (( i = i + 1 ))
554    done
555
556    if [ ${status} -gt 0 ] ; then
557      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
558      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
559      IGCM_debug_Exit "IGCM_sys_Get_Dir"
560    else
561      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
562    fi
563  fi
564  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
565}
566
567#D-#==================================================
568#D-function IGCM_sys_Put_Rest
569#D-* Purpose: Put computied restarts on ${ARCHIVE}.
570#D-           File and target directory must exist.
571#D-* Examples:
572#D-
573function IGCM_sys_Put_Rest {
574  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
575  if ( $DEBUG_sys ) ; then
576    echo "IGCM_sys_Put_Rest :" $@
577  fi
578  if [ $DRYRUN = 0 ]; then
579    if [ ! -f ${1} ] ; then
580      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
581      IGCM_debug_Exit "IGCM_sys_Put_Rest"
582    fi
583
584    typeset status
585    #
586    # USUAL WAY
587    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
588    status=$?
589
590#       #RSYNC WITH NETWORK SSH CALL
591#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
592#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
593
594#       #RSYNC WITH NFS USE
595#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
596#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
597
598#       status=$?
599#       IGCM_sys_Rsync_out $status
600
601#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
602#       (( status=status+$? ))
603
604    if [ ${status} -gt 0 ] ; then
605      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
606      [ -f ${1} ] && ls -l ${1}
607      [ -f ${2} ] && ls -l ${2}
608      [ -f ${2}/${1} ] && ls -l ${2}/${1}
609      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
610      IGCM_debug_Exit "IGCM_sys_Put_Rest"
611    else
612
613      if [ X${JobType} = XRUN ] ; then
614        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
615        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
616      fi
617
618      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
619    fi
620  fi
621  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
622}
623
624#D-#==================================================
625#D-function IGCM_sys_Put_Out
626#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
627#D-* Examples:
628#D-
629function IGCM_sys_Put_Out {
630  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
631  if ( $DEBUG_sys ) ; then
632    echo "IGCM_sys_Put_Out :" $@
633  fi
634
635  typeset NB_ESSAI DELAI status i exist skip
636
637  # number of tentative
638  NB_ESSAI=3
639  # time delay between tentative
640  DELAI=2
641
642  if [ $DRYRUN = 0 ]; then
643    if [ ! -f ${1} ] ; then
644      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
645      IGCM_debug_PopStack "IGCM_sys_Put_Out"
646      return 1
647    fi
648    #
649    IGCM_sys_MkdirArchive $( dirname $2 )
650    #
651    exist=false
652    skip=false
653    if [ -f $2 ] ; then
654      IGCM_debug_Print 1 "$2 already exist"
655      ccc_hsm get $2
656      exist=true
657      if [ "X$( diff $1 $2 )" = X ] ; then
658        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
659        skip=true
660      else
661        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
662        skip=false
663      fi
664    fi
665    #
666    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
667      IGCM_sys_Chmod u+w $2
668    fi
669
670    if [ X${skip} = Xfalse ] ; then
671      i=0
672      while [ $i -lt $NB_ESSAI ] ; do
673        if [ $( stat -c %d $1 ) -ne $( stat -c %d $( dirname $2 ) ) ] ; then
674          # USUAL WAY
675          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
676          status=$?
677        else
678          # NOT SO USUAL WAY
679          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
680          status=$?
681        fi
682        if [ ${status} -gt 0 ]; then
683          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
684          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
685          [ -f ${1} ] && ls -l ${1}
686          [ -f ${2} ] && ls -l ${2}
687          [ -f ${2}/${1} ] && ls -l ${2}/${1}
688          sleep $DELAI
689        else
690          break
691        fi
692        (( i = i + 1 ))
693      done
694    fi
695
696#       #RSYNC WITH NETWORK SSH CALL
697#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
698#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
699
700#       #RSYNC WITH NFS USE
701#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
702#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
703
704#       status=$?
705#       IGCM_sys_Rsync_out $status
706
707#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
708#       (( status=status+$? ))
709
710    if [ ${status} -gt 0 ] ; then
711      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
712      [ -f ${1} ] && ls -l ${1}
713      [ -f ${2} ] && ls -l ${2}
714      [ -f ${2}/${1} ] && ls -l ${2}/${1}
715      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
716      IGCM_debug_Exit "IGCM_sys_Put_Out"
717    else
718
719      if [ X${JobType} = XRUN ] ; then
720        if [ X${3} = X ] ; then
721          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
722          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
723        fi
724      fi
725
726      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
727    fi
728  fi
729  IGCM_debug_PopStack "IGCM_sys_Put_Out"
730  return 0
731}
732
733#D-#==================================================
734#D-function IGCM_sys_Get
735#D-* Purpose: Get a file from ${ARCHIVE}
736#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
737#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
738function IGCM_sys_Get {
739  IGCM_debug_PushStack "IGCM_sys_Get" $@
740
741  typeset DEST dm_liste target file_work
742  typeset NB_ESSAI DELAI status i
743
744  if ( $DEBUG_sys ) ; then
745    echo "IGCM_sys_Get :" $@
746  fi
747
748  # number of tentative
749  NB_ESSAI=3
750  # time delay between tentative
751  DELAI=2
752
753  if [ $DRYRUN -le 2 ]; then
754    if [ X${1} = X'/l' ] ; then
755      eval set +A dm_liste \${${2}}
756    else
757      eval set +A dm_liste ${1}
758    fi
759    eval DEST=\${${#}}
760    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
761    status=$?
762    if [ ${status} -gt 0 ] ; then
763      echo "WARNING IGCM_sys_Get : error code ${status}"
764      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
765      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
766    fi
767
768    #if [ ${status} -gt 0 ] ; then
769    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
770    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
771    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
772    #    sleep 30
773    #    echo "We try another time"
774    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
775    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
776    #    status=$?
777    #    if [ ${status} -gt 0 ] ; then
778    #      echo "ERROR IGCM_sys_Get : again demigration error :"
779    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
780    #      IGCM_debug_Exit "IGCM_sys_Get"
781    #    fi
782    #  else
783    #    echo "ERROR IGCM_sys_Get : demigration error :"
784    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
785    #    IGCM_debug_Exit "IGCM_sys_Get"
786    #  fi
787    #fi
788
789    #   #RSYNC WITH NETWORK SSH CALL
790    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
791    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
792
793    #   #RSYNC WITH NFS USE
794    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
795    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
796
797    #   status=$?
798    #   IGCM_sys_Rsync_out $status
799
800    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
801    #   (( status=status+$? ))
802
803    #USUAL WAY
804    if [ X${1} = X'/l' ] ; then
805      for target in ${dm_liste[*]} ; do
806        local_file=$( basename ${target} )
807        # test if the target file is present before the loop
808        IGCM_sys_TestFileArchive ${target}
809        status=$?
810        if [ ${status} -gt 0 ] ; then
811          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
812          IGCM_debug_Exit "IGCM_sys_Get"
813        else
814          i=0
815          while [ $i -lt $NB_ESSAI ] ; do
816            #if [ X${DoLink} = Xtrue ] ; then
817            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
818            #  status=$?
819            #  else
820            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
821            #  status=$?
822            #fi
823            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
824            status=$?
825            if [ ${status} -gt 0 ]; then
826              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
827              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
828              sleep $DELAI
829            else
830              break
831            fi
832            (( i = i + 1 ))
833          done
834          if [ ${status} -gt 0 ] ; then
835            echo "IGCM_sys_Get : error"
836            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
837            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
838            IGCM_debug_Exit "IGCM_sys_Get"
839          else
840            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
841          fi
842        fi
843      done
844    else
845      i=0
846      while [ $i -lt $NB_ESSAI ] ; do
847        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
848        status=$?
849        if [ ${status} -gt 0 ]; then
850          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
851          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
852          sleep $DELAI
853        else
854          break
855        fi
856        (( i = i + 1 ))
857      done
858      if [ ${status} -gt 0 ] ; then
859        echo "IGCM_sys_Get : error"
860        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
861        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
862        IGCM_debug_Exit "IGCM_sys_Get"
863      else
864        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
865      fi
866    fi
867  fi
868  IGCM_debug_PopStack "IGCM_sys_Get"
869}
870
871#D-#==================================================
872#D-function IGCM_sys_GetDate_Monitoring
873#D-* Purpose: get the last year for which the monitoring has been computed
874#D-* Examples:
875#D-
876function IGCM_sys_GetDate_Monitoring {
877  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
878  if ( $DEBUG_sys ) ; then
879    echo "IGCM_sys_GetDate_Monitoring :" $@
880  fi
881
882  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
883
884  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
885}
886
887#D-#==================================================
888#D-function IGCM_sys_Dods_Rm
889#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
890#D-* Examples:
891#D-
892function IGCM_sys_Dods_Rm {
893  if ( $DEBUG_sys ) ; then
894    echo "IGCM_sys_Dods_Rm :" $@
895  fi
896  typeset status
897  if [ $DRYRUN = 0 ]; then
898
899#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
900#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
901#      echo "Nothing has been done."
902#      return
903#    fi
904
905    /ccc/cont003/home/dsm/p86ipsl/bin/dods_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
906    status=$?
907
908#    if [ ${status} -gt 0 ] ; then
909#      echo "IGCM_sys_Dods_Rm : error."
910#      cat out_dods_rm
911#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
912#    else
913#      rm out_dods_rm
914#    fi
915
916  fi
917  return $status
918}
919
920#D-#==================================================
921#D-function IGCM_sys_Dods_Cp
922#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
923#D-* Examples:
924#D-
925function IGCM_sys_Dods_Cp {
926  if ( $DEBUG_sys ) ; then
927    echo "IGCM_sys_Dods_Cp :" $@
928  fi
929  typeset status
930  if [ $DRYRUN = 0 ]; then
931
932#    if [ ! -d ${R_SAVE}/${1} ] ; then
933#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
934#      echo "Nothing has been done."
935#      return
936#    fi
937
938    /ccc/cont003/home/dsm/p86ipsl/bin/dods_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
939    status=$?
940
941#       if [ ${status} -gt 0 ] ; then
942#           echo "IGCM_sys_Dods_Cp : error."
943#           cat out_dods_cp
944#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
945#       else
946#           rm out_dods_cp
947#       fi
948
949  fi
950  return $status
951}
952
953#D-#==================================================
954#D-function IGCM_sys_Put_Dods
955#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
956#D-* Examples:
957#D-
958function IGCM_sys_Put_Dods {
959  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
960  if ( $DEBUG_sys ) ; then
961    echo "IGCM_sys_Put_Dods :" $@
962  fi
963  typeset status
964  if [ $DRYRUN = 0 ]; then
965    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
966      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
967      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
968      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
969      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
970      return
971    fi
972
973    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
974      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
975      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
976      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
977      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
978      return
979    fi
980    #
981    if [ -d ${R_SAVE}/${1} ] ; then
982      cd ${R_SAVE}
983    elif [ -d ${R_FIGR}/${1} ] ; then
984      cd ${R_FIGR}
985    fi
986
987    IGCM_sys_Dods_Rm ${1}
988    IGCM_sys_Dods_Cp ${1}
989    status=0
990
991    if [ ${status} -gt 0 ] ; then
992      echo "IGCM_sys_Put_Dods : error."
993      IGCM_debug_Exit "IGCM_sys_Put_Dods"
994    fi
995  fi
996  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
997}
998
999##############################################################
1000# REBUILD OPERATOR
1001
1002#D-#==================================================
1003#D-function IGCM_sys_sync
1004#D-* Purpose: flush buffer on disk
1005#D-* Examples:
1006#D-
1007function IGCM_sys_sync {
1008  IGCM_debug_PushStack "IGCM_sys_sync" $@
1009  if ( $DEBUG_sys ) ; then
1010    echo "IGCM_sys_sync :" $@
1011  fi
1012
1013  /bin/sync
1014
1015  IGCM_debug_PopStack "IGCM_sys_sync"
1016}
1017
1018############################################################
1019# Activate Running Environnment Variables
1020
1021#D-#==================================================
1022#D-function IGCM_sys_desactiv_variables
1023#D-* Purpose: set environement variables prior to execution
1024#D-* Examples:
1025#D-
1026function IGCM_sys_activ_variables {
1027  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1028  if ( $DEBUG_sys ) ; then
1029    echo "IGCM_sys_activ_variables"
1030  fi
1031
1032# --------------------------------------------------------------------
1033#D- MPI specifications
1034# --------------------------------------------------------------------
1035
1036# --------------------------------------------------------------------
1037#D- Other specifications
1038# --------------------------------------------------------------------
1039
1040  ulimit -s unlimited
1041
1042  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1043}
1044
1045############################################################
1046# Desactivate Running Environnment Variables
1047
1048#D-#==================================================
1049#D-function IGCM_sys_desactiv_variables
1050#D-* Purpose: unset environement variables after execution
1051#D-* Examples:
1052#D-
1053function IGCM_sys_desactiv_variables {
1054  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1055  if ( $DEBUG_sys ) ; then
1056    echo "IGCM_sys_desactiv_variables"
1057  fi
1058# --------------------------------------------------------------------
1059#D- MPI specifications
1060# --------------------------------------------------------------------
1061
1062# --------------------------------------------------------------------
1063#D- Other specifications
1064# --------------------------------------------------------------------
1065
1066  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1067}
1068
1069############################################################
1070# Build MPI/OMP scripts run file (dummy function)
1071
1072#D-#==================================================
1073#D-function IGCM_sys_build_run_file
1074#D-* Purpose: build run file (deprecated)
1075#D-* Examples:
1076#D-
1077function IGCM_sys_build_run_file {
1078
1079  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1080
1081}
1082
1083############################################################
1084# Build MPI/OMP scripts
1085
1086#D-#==================================================
1087#D-function IGCM_sys_build_execution_scripts
1088#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1089#D-* Examples:
1090#D-
1091function IGCM_sys_build_execution_scripts
1092{
1093  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1094  if ( $DEBUG_sys ) ; then
1095    echo "IGCM_sys_build_execution_scripts " $@
1096  fi
1097
1098  EXECUTION=${HOST_MPIRUN_COMMAND}
1099
1100  # MPMD mode
1101  if ( ${OK_PARA_MPMD} ) ; then
1102
1103    # Only MPI (MPMD)
1104    if  ( ! ${OK_PARA_OMP} ) ; then
1105
1106      if [ -f run_file ] ; then
1107        IGCM_sys_Rm -f run_file
1108      fi
1109      touch run_file
1110
1111      # Build run_file
1112
1113      # First loop on the components for the coupler ie oasis (only if oasis3)
1114      # the coupler ie oasis3 must be the first one
1115      for comp in ${config_ListOfComponents[*]} ; do
1116
1117        eval ExeNameIn=\${config_Executable_${comp}[0]}
1118        eval ExeNameOut=\${config_Executable_${comp}[1]}
1119
1120        # for CPL component only
1121        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1122
1123          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1124          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1125          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1126        fi
1127      done
1128
1129      # Then second loop on the components
1130      for comp in ${config_ListOfComponents[*]} ; do
1131
1132        eval ExeNameIn=\${config_Executable_${comp}[0]}
1133        eval ExeNameOut=\${config_Executable_${comp}[1]}
1134
1135        # Only if we really have an executable for the component and not the coupler ie oasis:
1136        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1137
1138          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1139          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1140          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1141        fi
1142      done
1143
1144      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1145
1146      IGCM_sys_Chmod u+x run_file
1147      if ( $DEBUG_sys ) ; then
1148        echo "run_file contains : "
1149        cat run_file
1150      fi
1151
1152    # MPI-OpenMP (MPMD)
1153    else
1154
1155      # Use of mpirun instead of ccc_mprun
1156      EXECUTION="time mpirun"
1157
1158      #  Hosts treatment
1159      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1160
1161      i=0
1162      rm -f hosts rankfile
1163      IGCM_debug_Print 1 "sys Curie, Hosts available :"
1164      for nodes in `cat hosts.tmp`
1165      do
1166        host[$i]=$nodes
1167        echo "${host[$i]}" >> hosts
1168        IGCM_debug_Print 1 ${host[$i]}
1169        i=$((i+1))
1170      done
1171      rm -f hosts.tmp
1172
1173      listnodes=${host[*]}
1174
1175      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1176
1177      # Initialisation
1178      rank=0
1179      current_core=0
1180      core_per_node=16
1181      init_exec=n
1182
1183      # Loop on the components
1184      for comp in ${config_ListOfComponents[*]} ; do
1185
1186        eval ExeNameIn=\${config_Executable_${comp}[0]}
1187        eval ExeNameOut=\${config_Executable_${comp}[1]}
1188
1189        # Not possible if oasis has an executable (i.e old version of oasis3)
1190        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1191          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1192          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1193          IGCM_debug_Verif_Exit
1194        fi
1195
1196        # Only if we really have an executable for the component :
1197        if [ "X${ExeNameOut}" != X\"\" ] ; then
1198
1199          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1200          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1201
1202          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1203          echo ""  >> script_${ExeNameOut}.ksh
1204          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1205
1206            # Check if the number of threads is correct
1207            case ${comp_proc_omp_loc} in
1208            2|4|8|16)
1209              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1210              ;;
1211            *)
1212              IGCM_debug_Exit "ERROR with OMP parameters !"
1213              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1214              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1215              IGCM_debug_Verif_Exit
1216              ;;
1217            esac
1218            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1219            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1220            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1221            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1222          fi
1223          #echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK - ${start_num})) " >>  script_${ExeNameOut}.ksh
1224          #echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}"  >> script_${ExeNameOut}.ksh
1225          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1226          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1227
1228          if [ ${init_exec} = y ] ; then
1229            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1230          else
1231            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1232            init_exec=y
1233          fi
1234
1235# Build rankfile : method used to assign cores and nodes for the MPI process
1236# Ex :
1237#rank 0=curie5296 slot=0,1,2,3
1238#rank 1=curie5296 slot=4,5,6,7
1239# Example of final command :
1240# mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1241# with script_lmdz.x.ksh :
1242# #!/bin/ksh
1243#export KMP_STACKSIZE=3g
1244#export KMP_LIBRARY=turnaround
1245#export MKL_SERIAL=YES
1246#OMP_NUM_THREADS=4
1247#./lmdz.x
1248#
1249
1250          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1251            (( index_host = current_core / core_per_node ))
1252            host_value=${host[${index_host}]}
1253            (( slot =  current_core % core_per_node ))
1254            virg=","
1255            string_final=""
1256            for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1257              string=$index$virg
1258              string_final=$string_final$string
1259            done
1260            string_final=$( echo $string_final | sed "s/.$//" )
1261            echo "rank $rank=$host_value slot=$string_final" >> rankfile
1262            (( rank = rank + 1 ))
1263            (( current_core = current_core + comp_proc_omp_loc ))
1264          done
1265        fi
1266
1267      done
1268    fi
1269
1270  # Only one executable (SPMD mode).
1271  else
1272
1273    for comp in ${config_ListOfComponents[*]} ; do
1274
1275      # Only if we really have an executable for the component :
1276      eval ExeNameOut=\${config_Executable_${comp}[1]}
1277      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1278
1279        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1280        echo ""  >> script_${ExeNameOut}.ksh
1281        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1282
1283        if ( ${OK_PARA_OMP} ) ; then
1284          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1285          echo ""  >> script_${ExeNameOut}.ksh
1286          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1287          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1288          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1289          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1290        fi
1291
1292        if  ( ${OK_PARA_MPI} ) ; then
1293          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1294          # Default : ccc_mprun used if nb_proc gt 1
1295          # to have out/err per process on different files
1296          # echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1297          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1298          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1299        else
1300          # Default : ccc_mprun is NOT used if nb_proc eq 1
1301          # to have out/err per process on different files
1302          # echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1303          echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1304          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1305        fi
1306
1307        IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
1308        cat script_${ExeNameOut}.ksh
1309
1310      fi
1311
1312    done
1313
1314  fi
1315
1316  IGCM_debug_Print 1 "sys Curie : execution command is "
1317  IGCM_debug_Print 1 "$EXECUTION"
1318
1319  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1320}
1321
1322#D-#==================================================
1323#D-function IGCM_sys_check_path
1324#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1325#D-* do not point to an important use directory. Stop immediately in that case.
1326#D-* Examples:
1327#D-
1328function IGCM_sys_check_path {
1329  IGCM_debug_PushStack "IGCM_sys_check_path"
1330  if ( $DEBUG_sys ) ; then
1331    echo "IGCM_sys_check_path"
1332  fi
1333
1334  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORKDIR} ] || [ X${RUN_DIR_PATH} = X${SCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1335    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1336    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1337    IGCM_debug_Exit "This will stop the job"
1338  fi
1339  IGCM_debug_PopStack "IGCM_sys_check_path"
1340}
1341
1342#D-#==================================================
1343#D-function IGCM_sys_check_quota
1344#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1345#D-* Examples:
1346#D-
1347function IGCM_sys_check_quota {
1348  IGCM_debug_PushStack "IGCM_sys_check_quota"
1349  if ( $DEBUG_sys ) ; then
1350    echo "IGCM_sys_check_quota"
1351  fi
1352  # Limit of quota (in %)
1353  limit_quota=90
1354
1355  # Check of the volume
1356  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1357  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1358
1359  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1360
1361    unit_avail=${volume_avail: -1}
1362    unit_quota=${volume_quota: -1}
1363
1364    if [ "${unit_quota}" = "*" ] ; then
1365      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1366      IGCM_debug_Print 1 "More than 100% of your quota is used"
1367      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1368      IGCM_debug_Print 1 "You must have more than 10% available to run"
1369      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1370      IGCM_debug_Verif_Exit
1371    fi
1372
1373    temp_avail=${volume_avail%%${volume_avail: -1}*}
1374    temp_quota=${volume_quota%%${volume_quota: -1}*}
1375
1376    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1377
1378    # Convertion
1379      if [ ${volume_avail: -1} = "T" ] ; then
1380        (( temp_avail = temp_avail * 1000000000000 ))
1381      elif [ ${volume_avail: -1} = "G" ] ; then
1382        (( temp_avail = temp_avail * 1000000000 ))
1383      elif [ ${volume_avail: -1} = "M" ] ; then
1384        (( temp_avail = temp_avail * 1000000 ))
1385      elif [ ${volume_avail: -1} = "k" ] ; then
1386        (( temp_avail = temp_avail * 1000 ))
1387      else
1388        (( temp_avail = volume_avail ))
1389      fi
1390      if [ ${volume_quota: -1} = "T" ] ; then
1391        (( temp_quota = temp_quota * 1000000000000 ))
1392      elif [ ${volume_quota: -1} = "G" ] ; then
1393        (( temp_quota = temp_quota * 1000000000 ))
1394      elif [ ${volume_quota: -1} = "M" ] ; then
1395        (( temp_quota = temp_quota * 1000000 ))
1396      elif [ ${volume_quota: -1} = "k" ] ; then
1397        (( temp_quota = temp_quota * 1000 ))
1398      else
1399        (( temp_quota = volume_quota ))
1400      fi
1401    fi
1402
1403    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1404#    echo "volume ratio is " $quota_volume
1405
1406    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1407      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1408      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1409      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1410      IGCM_debug_Print 1 "You must have more than 10% available to run"
1411      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1412      IGCM_debug_Verif_Exit
1413    fi
1414
1415  fi
1416
1417# Check of the number of inodes
1418
1419  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1420  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1421
1422  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1423
1424    unit_avail=${inode_avail: -1}
1425    unit_quota=${inode_quota: -1}
1426
1427    if [ "${unit_quota}" = "*" ] ; then
1428      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1429      IGCM_debug_Print 1 "More than 100% of your quota is used"
1430      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1431      IGCM_debug_Print 1 "You must have more than 10% available to run"
1432      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1433      IGCM_debug_Verif_Exit
1434    fi
1435
1436    temp_avail=${inode_avail%%${inode_avail: -1}*}
1437    temp_quota=${inode_quota%%${inode_quota: -1}*}
1438
1439    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1440
1441    # Convertion
1442      if [ ${inode_avail: -1} = "T" ] ; then
1443        (( temp_avail = temp_avail * 1000000000000 ))
1444      elif [ ${inode_avail: -1} = "G" ] ; then
1445        (( temp_avail = temp_avail * 1000000000 ))
1446      elif [ ${inode_avail: -1} = "M" ] ; then
1447        (( temp_avail = temp_avail * 1000000 ))
1448      elif [ ${inode_avail: -1} = "k" ] ; then
1449        (( temp_avail = temp_avail * 1000 ))
1450      else
1451        (( temp_avail = inode_avail ))
1452      fi
1453
1454      if [ ${inode_quota: -1} = "T" ] ; then
1455        (( temp_quota = temp_quota * 1000000000000 ))
1456      elif [ ${inode_quota: -1} = "G" ] ; then
1457        (( temp_quota = temp_quota * 1000000000 ))
1458      elif [ ${inode_quota: -1} = "M" ] ; then
1459        (( temp_quota = temp_quota * 1000000 ))
1460      elif [ ${inode_quota: -1} = "k" ] ; then
1461        (( temp_quota = temp_quota * 1000 ))
1462      else
1463        (( temp_quota = inode_quota ))
1464      fi
1465    fi
1466    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1467#    echo "inode ratio is " $quota_inode
1468
1469    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1470      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1471      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1472      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1473      IGCM_debug_Print 1 "You must have more than 10% available to run"
1474      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1475      IGCM_debug_Verif_Exit
1476    fi
1477  fi
1478  IGCM_debug_PopStack "IGCM_sys_check_quota"
1479}
1480
1481#D-#==================================================
1482#D-function IGCM_sys_GetJobID
1483#D-* Purpose: Check if job_name is currently
1484#D-  running or in queue
1485#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1486#D-
1487function IGCM_sys_GetJobID {
1488  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1489  if ( $DEBUG_sys ) ; then
1490    echo "IGCM_sys_GetJobID"
1491  fi
1492
1493  # With -f option, the full job name is given in the last column
1494  ID=$( ccc_mstat -f -u $2 | \
1495        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1496
1497  eval ${3}=${ID}
1498  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1499}
1500
1501#D-#==================================================
1502#D-function IGCM_sys_CountJobInQueue
1503#D-* Purpose: Check if job_name is currently
1504#D-  running or in queue
1505#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1506#D-
1507function IGCM_sys_CountJobInQueue {
1508  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1509  if ( $DEBUG_sys ) ; then
1510    echo "IGCM_sys_CountJobInQueue"
1511  fi
1512
1513  # With -f option, the full job name is given in the last column
1514  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1515
1516  eval ${2}=${NbRun}
1517
1518  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1519}
1520
1521#D-#==================================================
1522#D-function IGCM_sys_ListJobInQueue
1523#D-* Purpose: Check if job_name is currently
1524#D-  running or in queue
1525#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1526#D-
1527function IGCM_sys_ListJobInQueue {
1528  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1529  if ( $DEBUG_sys ) ; then
1530    echo "IGCM_sys_ListJobInQueue"
1531  fi
1532
1533  # With -f option, the full job name is given in the last column
1534  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1535                                        '( $2  == User      && \
1536                                           $NF != /TS/      && \
1537                                           $NF !~ /PACK/    && \
1538                                           $NF !~ /REBUILD/ && \
1539                                           $NF !~ /pack/ )     \
1540                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1541
1542  eval set -A ${2} ${JobList[*]}
1543
1544  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1545}
Note: See TracBrowser for help on using the repository browser.