source: CONFIG_DEVT/IPSLCM6.2.2_ENSEMBLES/libIGCM/libIGCM_sys/libIGCM_sys_curie.ksh @ 6207

Last change on this file since 6207 was 6204, checked in by aclsce, 2 years ago

First import of IPSLCM6.2.2_work_ENSEMBLES working configuration

File size: 67.5 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision:: 1466                                     $ Revision of last commit
7# $Author:: aclsce                                     $ Author of last commit
8# $Date:: 2018-07-13 09:38:15 +0200 (Fri, 13 Jul 2018) $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Curie
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40typeset -r DRYRUN=${DRYRUN:=0}
41
42# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
43# -------------------------------------------------------------------------------------
44# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
45# |          |  Cp/Exe/param/files |            |         |                           |
46# |          |  Chmod Qsub         |            |         |                           |
47# -------------------------------------------------------------------------------------
48# |    0     |       yes           |    yes     |  yes    |      yes                  |
49# -------------------------------------------------------------------------------------
50# |    1     |       yes           |    yes     |  yes    |      no                   |
51# -------------------------------------------------------------------------------------
52# |    2     |       yes           |    yes     |  no     |      no                   |
53# -------------------------------------------------------------------------------------
54# |    3     |       yes           |    no      |  no     |      no                   |
55# -------------------------------------------------------------------------------------
56
57#=====================================================
58# Global Variables :
59#=====================================================
60# Language : "fr" or "en"
61typeset -r MYLANG="fr"
62
63#=====================================================
64# Host user names project maxCpuTime
65# $hostname ou hostname
66typeset HOST=${HOST:=$( hostname )}
67# $username ou whoami
68typeset LOGIN=${LOGIN:=$( whoami )}
69# $hostname of the MASTER job
70typeset MASTER=curie
71# add default project on curie
72typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=gen0826} | cut -d@ -f1 )
73# jobWarningDelay in seconds
74typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
75
76#D-
77#D-#==================================================
78#D-Program used in libIGCM
79#D-#==================================================
80
81# Submit command
82typeset SUBMIT=${SUBMIT:=ccc_msub}
83# rsync with path
84typeset -r RSYNC=/usr/bin/rsync
85# RSYNC_opt args to rsync
86typeset -r RSYNC_opt="-va"
87# ie storage filesystem
88typeset -r STOREHOST=${MASTER}
89typeset -r REMOTE_RSYNC=/usr/bin/rsync
90
91#====================================================
92# Set environment tools (ferret, nco, cdo, rebuild, ...)
93#====================================================
94if [ X${TaskType} = Xcomputing ] ; then
95  IGCM_debug_Print 1 "Module will be loaded once we know the compiler version"
96else
97  module purge > /dev/null 2>&1
98  ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/curie/env_intel17.0.2_curie   > /dev/null 2>&1
99  . $( ccc_home -u igcmg)/MachineEnvironment/curie/env_intel17.0.2_curie   > /dev/null 2>&1
100  ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/curie/env_netcdf4.3.3.1_curie > /dev/null 2>&1
101  . $( ccc_home -u igcmg)/MachineEnvironment/curie/env_netcdf4.3.3.1_curie > /dev/null 2>&1
102  ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/curie/env_atlas_curie         > /dev/null 2>&1
103  . $( ccc_home -u igcmg)/MachineEnvironment/curie/env_atlas_curie         > /dev/null 2>&1
104  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
105  export UVCDAT_ANONYMOUS_LOG=FALSE
106fi
107
108# Use CMIP6 storage space when using CMIP6 cpu ressources
109[[ X"$( echo ${PROJECT} | grep cmip6 )" != "X" && ! $(module list --terse 2>&1 | grep dfldatadir/gencmip6) ]] && module switch dfldatadir dfldatadir/gencmip6
110
111# FYI
112[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
113[ ! X${TaskType} = Xchecking ] && module list
114
115# for rebuild, AddNoise and AddPertu3DOCE
116export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/curie/bin
117##export PATH=${PATH}:/ccc/cont003/home/igcmg/igcmg/Tools/curie/bin
118
119#====================================================
120# Specific for ocean additionnal diagnostic
121export FER_GO="$FER_GO /home/cont003/p86denv/IGCM_POST_UTIL/JNL /home/cont003/p86denv/GRAF /home/cont003/p86denv/GRAF/GO"
122export FER_PALETTE="$FER_PALETTE /home/cont003/p86denv/GRAF/PALET"
123
124#====================================================
125# Host specific DIRECTORIES
126#====================================================
127
128# ============ CESIUM START ============ #
129
130#====================================================
131#- Mirror libIGCM from titane to cesium if needed
132#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
133#if [ ! ${ROOTSYS} = "home" ] ; then
134#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
135#else
136#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
137#fi
138
139#====================================================
140#- libIGCM_POST
141#if ( ${MirrorlibIGCM} ) ; then
142#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
143#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
144#else
145#  typeset -r libIGCM_POST=${libIGCM}
146#fi
147
148# ============ CESIUM  END  ============ #
149
150#====================================================
151#- MirrorlibIGCM for frontend
152typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
153
154#====================================================
155#- libIGCM_POST for frontend
156typeset -r libIGCM_POST=${libIGCM}
157
158#====================================================
159#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
160typeset -r R_EXE="${MODIPSL}/bin"
161
162#====================================================
163#- SUBMIT_DIR : submission dir
164typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
165
166#====================================================
167#- IN
168typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
169
170#====================================================
171#- RUN_DIR_PATH : Temporary working directory (=> TMP)
172typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${SCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
173
174#====================================================
175#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
176typeset -r OUTCOMMAND_PATH=/tmp
177
178#====================================================
179#- HOST_MPIRUN_COMMAND
180typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
181
182#====================================================
183#- Max number of arguments passed to nco operator or demigration command
184UNIX_MAX_LIMIT=360
185
186#====================================================
187#- set PackDefault to true on curie
188PackDefault=true
189
190#====================================================
191#- Default number of MPI task for IPSL coupled model
192#- required for backward compatibility
193#-
194DEFAULT_NUM_PROC_OCE=5
195DEFAULT_NUM_PROC_CPL=1
196(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
197
198#====================================================
199#- Number of computing cores per node
200#-
201NB_CORE_PER_NODE=16
202
203#D-#==================================================
204#D-function IGCM_sys_defineArchives
205#D-* Purpose:
206#D-* Define ARCHIVE : Dedicated to large files
207#D-* Define STORAGE : Dedicated to small/medium files
208#D-* Define R_OUT   : Output tree located on ARCHIVE
209#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
210#D-* Define R_BUF   : Output tree located on SCRATCHDIR hosting files waiting for rebuild or pack processes
211#D-* if SpaceName=TEST everything is stored on SCRATCHDIR
212#D-* Examples:
213#D-
214function IGCM_sys_defineArchives {
215  IGCM_debug_PushStack "IGCM_sys_defineArchives"
216
217  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
218    #====================================================
219    #- ARCHIVE (dedicated to large files)
220    ARCHIVE=${config_UserChoices_ARCHIVE}
221  else
222    #====================================================
223    #- ARCHIVE (dedicated to large files)
224    ARCHIVE=${CCCSTOREDIR}
225  fi
226
227  if [ ! X${config_UserChoices_STORAGE} = X ]; then
228    #====================================================
229    #- STORAGE (dedicated to small/medium files)
230    STORAGE=${config_UserChoices_STORAGE}
231  else
232    #====================================================
233    #- STORAGE (dedicated to small/medium files)
234    STORAGE=${CCCWORKDIR}
235  fi
236
237  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
238    #====================================================
239    #- R_OUT
240    R_OUT=${SCRATCHDIR}/IGCM_OUT
241
242    #====================================================
243    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
244    R_FIG=${SCRATCHDIR}/IGCM_OUT
245
246    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
247
248  else
249    #====================================================
250    #- R_OUT
251    R_OUT=${ARCHIVE}/IGCM_OUT
252
253    #====================================================
254    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
255    R_FIG=${STORAGE}/IGCM_OUT
256  fi
257
258  #====================================================
259  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
260  CMIP6_BUF=${STORAGE}/IGCM_OUT
261
262  #====================================================
263  #- R_BUF (ONLY FOR double copy an scratch)
264  R_BUF=${SCRATCHDIR}/IGCM_OUT
265
266  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
267  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
268  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
269
270  IGCM_debug_PopStack "IGCM_sys_defineArchives"
271}
272
273#D-#==================================================
274#D-function IGCM_sys_RshArchive
275#D-* Purpose: Archive rsh command
276#D-* Examples:
277#D-
278function IGCM_sys_RshArchive {
279  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
280  /bin/ksh <<-EOF
281    ${@}
282EOF
283  status=$?
284  if [ ${status} -gt 0 ] ; then
285    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
286    IGCM_debug_Exit "IGCM_sys_RshArchive"
287  fi
288  IGCM_debug_PopStack "IGCM_sys_RshArchive"
289}
290
291#D-#==================================================
292#D-function IGCM_sys_RshArchive_NoError
293#D-* Purpose: Archive rsh command, without error
294#D-*          used only in monitoring.job
295#D-* Examples:
296#D-
297function IGCM_sys_RshArchive_NoError {
298  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
299  /bin/ksh <<-EOF
300    ${@} 2> /dev/null
301EOF
302  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
303}
304
305#D-#==================================================
306#D-function IGCM_sys_MkdirArchive
307#D-* Purpose: Mkdir on Archive
308#D-* Examples:
309#D-
310function IGCM_sys_MkdirArchive {
311  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
312  if ( $DEBUG_sys ) ; then
313    echo "IGCM_sys_MkdirArchive :" $@
314  fi
315  #- creation de repertoire sur le serveur fichier
316  if [ ! -d ${1} ]; then
317    \mkdir -p $1
318    status=$?
319
320    if [ ${status} -gt 0 ] ; then
321      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
322      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
323    fi
324  fi
325  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
326}
327
328#D-#==================================================
329#D-function IGCM_sys_TestDirArchive
330#D-* Purpose: Test Directory that must exists on Archive
331#D-* Examples:
332#D-
333function IGCM_sys_TestDirArchive {
334  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
335  if ( $DEBUG_sys ) ; then
336    echo "IGCM_sys_TestDirArchive :" $@
337  fi
338  typeset ExistFlag
339  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
340  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
341  return ${ExistFlag}
342}
343
344#D-#==================================================
345#D-function IGCM_sys_IsFileArchived
346#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
347#D-* Examples:
348#D-
349function IGCM_sys_IsFileArchived {
350  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
351  if ( $DEBUG_sys ) ; then
352    echo "IGCM_sys_IsFileArchived :" $@
353  fi
354  typeset IsArchivedFlag
355  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
356  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
357
358  return ${IsArchivedFlag}
359}
360
361#D-#==================================================
362#D-function IGCM_sys_TestFileArchive
363#D-* Purpose: Test file that must NOT EXISTS on Archive
364#D-* Examples:
365#D-
366function IGCM_sys_TestFileArchive {
367  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
368  typeset ExistFlag
369  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
370  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
371
372  return ${ExistFlag}
373}
374
375#D-#==================================================
376#D-function IGCM_sys_CountFileArchive
377#D-* Purpose: Count files on Archive filesystem
378#D-* Examples:
379#D-
380function IGCM_sys_CountFileArchive {
381  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
382  ls ${@} 2>/dev/null | wc -l
383  if [ $? -gt 0 ] ; then
384    echo "IGCM_sys_CountFileArchive : erreur."
385  fi
386  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
387}
388
389#D-#==================================================
390#D-function IGCM_sys_Tree
391#D-* Purpose: Tree directories with files on ${ARCHIVE}
392#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
393#D-
394function IGCM_sys_Tree {
395  IGCM_debug_PushStack "IGCM_sys_Tree" $@
396  if ( $DEBUG_sys ) ; then
397    echo "IGCM_sys_Tree :" $@
398  fi
399
400  \ls -lR ${@}
401
402  IGCM_debug_PopStack "IGCM_sys_Tree"
403}
404
405#D-#==================================================
406#D-function IGCM_sys_Qsub
407#D-* Purpose: Qsub new job
408#D-* Examples:
409#D-
410function IGCM_sys_Qsub {
411  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
412  if ( $DEBUG_sys ) ; then
413    echo "IGCM_sys_Qsub :" $@
414  fi
415  typeset options status
416  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
417
418  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
419  status=$?
420
421  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
422  if [ ${status} -gt 0 ] ; then
423    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
424    IGCM_debug_Exit "IGCM_sys_Qsub"
425  else
426    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
427  fi
428  IGCM_debug_PopStack "IGCM_sys_Qsub"
429}
430
431#D-#==================================================
432#D-function IGCM_sys_QsubPost
433#D-* Purpose: Qsub new job on scalaire
434#D-* Examples:
435#D-
436function IGCM_sys_QsubPost {
437  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
438  if ( $DEBUG_sys ) ; then
439    echo "IGCM_sys_QsubPost :" $@
440  fi
441  typeset options status
442
443  # EASIER TO DO THIS INSTEAD OF DUPLICATING libIGCM_sys_curie.ksh
444  case $( hostname -s ) in
445  curie*)
446    options="-Q normal -A ${PROJECT} -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out";;
447  airain*)
448    options="-q ivybridge -A dsm -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
449  esac
450
451  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
452  status=$?
453
454  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
455  if [ ${status} -gt 0 ] ; then
456    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
457    IGCM_debug_Exit "IGCM_sys_QsubPost"
458  else
459    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
460  fi
461  IGCM_debug_PopStack "IGCM_sys_QsubPost"
462}
463
464#D-*************************
465#D- File transfer functions
466#D-*************************
467#D-
468
469#D-#==================================================
470#D-function IGCM_sys_RmRunDir
471#D-* Purpose: rm tmpdir (dummy function most of the time batch
472#D-                      scheduler will do the job)
473#D-* Examples:
474#D-
475function IGCM_sys_RmRunDir {
476  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
477  if ( $DEBUG_sys ) ; then
478    echo "IGCM_sys_RmRunDir :" $@
479  fi
480
481  typeset status
482
483  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
484  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
485  status=$?
486
487  if [ ${status} -gt 0 ] ; then
488    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
489    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
490    IGCM_debug_Exit "IGCM_sys_RmRunDir"
491  else
492    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
493  fi
494  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
495}
496
497#D-#==================================================
498#D-function IGCM_sys_Put_Dir
499#D-* Purpose: Copy a complete directory on $(ARCHIVE)
500#D-* Examples:
501#D-
502function IGCM_sys_Put_Dir {
503  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
504  if ( $DEBUG_sys ) ; then
505    echo "IGCM_sys_Put_Dir :" $@
506  fi
507  if [ $DRYRUN = 0 ]; then
508    if [ ! -d ${1} ] ; then
509      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
510      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
511      return
512    fi
513
514    typeset status
515
516    # Only if we use rsync
517    #IGCM_sys_TestDirArchive $( dirname $2 )
518    #
519    #USUAL WAY
520    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
521    status=$?
522
523    if [ ${status} -gt 0 ] ; then
524      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
525      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
526      IGCM_debug_Exit "IGCM_sys_Put_Dir"
527    else
528      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
529    fi
530  fi
531  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
532}
533
534#D-#==================================================
535#D-function IGCM_sys_Get_Dir
536#D-* Purpose: Copy a complete directory from ${ARCHIVE}
537#D-* Examples:
538#D-
539function IGCM_sys_Get_Dir {
540  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
541  if ( $DEBUG_sys ) ; then
542    echo "IGCM_sys_Get_Dir :" $@
543  fi
544  if [ $DRYRUN = 0 ]; then
545    typeset NB_ESSAI DELAI status i
546    # number of tentative
547    NB_ESSAI=3
548    # time delay between tentative
549    DELAI=2
550
551    #
552    # USUAL WAY
553    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
554    ccc_hsm get -r $1
555
556    i=0
557    while [ $i -lt $NB_ESSAI ] ; do
558      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
559      status=$?
560      if [ ${status} -gt 0 ] ; then
561        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
562        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
563        sleep $DELAI
564      else
565        break
566      fi
567      (( i = i + 1 ))
568    done
569
570    if [ ${status} -gt 0 ] ; then
571      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
572      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
573      IGCM_debug_Exit "IGCM_sys_Get_Dir"
574    else
575      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
576    fi
577  fi
578  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
579}
580
581#D-#==================================================
582#D-function IGCM_sys_Put_Rest
583#D-* Purpose: Put computied restarts on ${ARCHIVE}.
584#D-           File and target directory must exist.
585#D-* Examples:
586#D-
587function IGCM_sys_Put_Rest {
588  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
589  if ( $DEBUG_sys ) ; then
590    echo "IGCM_sys_Put_Rest :" $@
591  fi
592  if [ $DRYRUN = 0 ]; then
593    if [ ! -f ${1} ] ; then
594      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
595      IGCM_debug_Exit "IGCM_sys_Put_Rest"
596    fi
597
598    typeset status
599    #
600    # USUAL WAY
601    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
602    status=$?
603
604#       #RSYNC WITH NETWORK SSH CALL
605#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
606#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
607
608#       #RSYNC WITH NFS USE
609#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
610#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
611
612#       status=$?
613#       IGCM_sys_Rsync_out $status
614
615#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
616#       (( status=status+$? ))
617
618    if [ ${status} -gt 0 ] ; then
619      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
620      [ -f ${1} ] && ls -l ${1}
621      [ -f ${2} ] && ls -l ${2}
622      [ -f ${2}/${1} ] && ls -l ${2}/${1}
623      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
624      IGCM_debug_Exit "IGCM_sys_Put_Rest"
625    else
626
627      if [ X${JobType} = XRUN ] ; then
628        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
629        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
630      fi
631
632      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
633    fi
634  fi
635  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
636}
637
638#D-#==================================================
639#D-function IGCM_sys_Put_Out
640#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
641#D-* Examples:
642#D-
643function IGCM_sys_Put_Out {
644  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
645  if ( $DEBUG_sys ) ; then
646    echo "IGCM_sys_Put_Out :" $@
647  fi
648
649  typeset NB_ESSAI DELAI status i exist skip
650  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
651
652  # number of tentative
653  NB_ESSAI=3
654  # time delay between tentative
655  DELAI=2
656
657  if [ $DRYRUN = 0 ]; then
658    if [ ! -f ${1} ] ; then
659      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
660      IGCM_debug_PopStack "IGCM_sys_Put_Out"
661      return 1
662    fi
663    #
664    IGCM_sys_MkdirArchive $( dirname $2 )
665    #
666    exist=false
667    skip=false
668    if [ -f $2 ] ; then
669      IGCM_debug_Print 1 "$2 already exist"
670      ccc_hsm get $2
671      exist=true
672      if [ "X$( diff $1 $2 )" = X ] ; then
673        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
674        skip=true
675      else
676        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
677        skip=false
678      fi
679    fi
680    #
681    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
682      IGCM_sys_Chmod u+w $2
683    fi
684
685    if [ X${skip} = Xfalse ] ; then
686      i=0
687      while [ $i -lt $NB_ESSAI ] ; do
688        # Identify file system
689        fileDeviceNumberInHex=$( stat -c %d $1 )
690        status=$?
691        if [ ${status} -gt 0 ] ; then
692          IGCM_debug_Exit "IGCM_sys_Put_Out"
693        fi
694        # Identify file system
695        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
696        status=$?
697        if [ ${status} -gt 0 ] ; then
698          IGCM_debug_Exit "IGCM_sys_Put_Out"
699        fi
700
701        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
702          # They are not on the same device. USUAL WAY
703          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
704          status=$?
705        else
706          # They are on the same device. NOT SO USUAL WAY
707          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
708          status=$?
709        fi
710        if [ ${status} -gt 0 ]; then
711          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
712          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
713          [ -f ${1} ] && ls -l ${1}
714          [ -f ${2} ] && ls -l ${2}
715          [ -f ${2}/${1} ] && ls -l ${2}/${1}
716          sleep $DELAI
717        else
718          break
719        fi
720        (( i = i + 1 ))
721      done
722    fi
723
724#       #RSYNC WITH NETWORK SSH CALL
725#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
726#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
727
728#       #RSYNC WITH NFS USE
729#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
730#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
731
732#       status=$?
733#       IGCM_sys_Rsync_out $status
734
735#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
736#       (( status=status+$? ))
737
738    if [ ${status} -gt 0 ] ; then
739      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
740      [ -f ${1} ] && ls -l ${1}
741      [ -f ${2} ] && ls -l ${2}
742      [ -f ${2}/${1} ] && ls -l ${2}/${1}
743      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
744      IGCM_debug_Exit "IGCM_sys_Put_Out"
745    else
746
747      if [ X${JobType} = XRUN ] ; then
748        if [ X${3} = X ] ; then
749          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
750          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
751        fi
752      fi
753
754      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
755    fi
756  fi
757  IGCM_debug_PopStack "IGCM_sys_Put_Out"
758  return 0
759}
760
761#D-#==================================================
762#D-function IGCM_sys_Get
763#D-* Purpose: Get a file from ${ARCHIVE}
764#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
765#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
766function IGCM_sys_Get {
767  IGCM_debug_PushStack "IGCM_sys_Get" $@
768
769  typeset DEST dm_liste target file_work
770  typeset NB_ESSAI DELAI status i
771
772  if ( $DEBUG_sys ) ; then
773    echo "IGCM_sys_Get :" $@
774  fi
775
776  # number of tentative
777  NB_ESSAI=3
778  # time delay between tentative
779  DELAI=2
780
781  if [ $DRYRUN -le 2 ]; then
782    if [ X${1} = X'/l' ] ; then
783      eval set +A dm_liste \${${2}}
784    else
785      eval set +A dm_liste ${1}
786    fi
787    eval DEST=\${${#}}
788    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
789    status=$?
790    if [ ${status} -gt 0 ] ; then
791      echo "WARNING IGCM_sys_Get : error code ${status}"
792      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
793      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
794    fi
795
796    #if [ ${status} -gt 0 ] ; then
797    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
798    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
799    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
800    #    sleep 30
801    #    echo "We try another time"
802    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
803    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
804    #    status=$?
805    #    if [ ${status} -gt 0 ] ; then
806    #      echo "ERROR IGCM_sys_Get : again demigration error :"
807    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
808    #      IGCM_debug_Exit "IGCM_sys_Get"
809    #    fi
810    #  else
811    #    echo "ERROR IGCM_sys_Get : demigration error :"
812    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
813    #    IGCM_debug_Exit "IGCM_sys_Get"
814    #  fi
815    #fi
816
817    #   #RSYNC WITH NETWORK SSH CALL
818    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
819    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
820
821    #   #RSYNC WITH NFS USE
822    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
823    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
824
825    #   status=$?
826    #   IGCM_sys_Rsync_out $status
827
828    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
829    #   (( status=status+$? ))
830
831    #USUAL WAY
832    if [ X${1} = X'/l' ] ; then
833      for target in ${dm_liste[*]} ; do
834        local_file=$( basename ${target} )
835        # test if the target file is present before the loop
836        IGCM_sys_TestFileArchive ${target}
837        status=$?
838        if [ ${status} -gt 0 ] ; then
839          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
840          IGCM_debug_Exit "IGCM_sys_Get"
841        else
842          i=0
843          while [ $i -lt $NB_ESSAI ] ; do
844            #if [ X${DoLink} = Xtrue ] ; then
845            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
846            #  status=$?
847            #  else
848            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
849            #  status=$?
850            #fi
851            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
852            status=$?
853            if [ ${status} -gt 0 ]; then
854              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
855              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
856              sleep $DELAI
857            else
858              break
859            fi
860            (( i = i + 1 ))
861          done
862          if [ ${status} -gt 0 ] ; then
863            echo "IGCM_sys_Get : error"
864            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
865            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
866            IGCM_debug_Exit "IGCM_sys_Get"
867          else
868            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
869          fi
870        fi
871      done
872    else
873      i=0
874      while [ $i -lt $NB_ESSAI ] ; do
875        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
876        status=$?
877        if [ ${status} -gt 0 ]; then
878          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
879          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
880          sleep $DELAI
881        else
882          break
883        fi
884        (( i = i + 1 ))
885      done
886      if [ ${status} -gt 0 ] ; then
887        echo "IGCM_sys_Get : error"
888        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
889        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
890        IGCM_debug_Exit "IGCM_sys_Get"
891      else
892        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
893      fi
894    fi
895  fi
896  IGCM_debug_PopStack "IGCM_sys_Get"
897}
898
899#D-#==================================================
900#D-function IGCM_sys_GetDate_Monitoring
901#D-* Purpose: get the last year for which the monitoring has been computed
902#D-* Examples:
903#D-
904function IGCM_sys_GetDate_Monitoring {
905  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
906  if ( $DEBUG_sys ) ; then
907    echo "IGCM_sys_GetDate_Monitoring :" $@
908  fi
909
910  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
911
912  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
913}
914
915#D-#==================================================
916#D-function IGCM_sys_Dods_Rm
917#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
918#D-* Examples:
919#D-
920function IGCM_sys_Dods_Rm {
921  if ( $DEBUG_sys ) ; then
922    echo "IGCM_sys_Dods_Rm :" $@
923  fi
924  typeset status
925  if [ $DRYRUN = 0 ]; then
926
927#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
928#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
929#      echo "Nothing has been done."
930#      return
931#    fi
932
933    if [ "$#" -eq 1 ]; then
934      $(ccc_home -u igcmg)/Tools/curie/bin/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
935      ## /ccc/cont003/home/igcmg/igcmg/Tools/curie/bin/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
936      status=$?
937    else
938      $(ccc_home -u igcmg)/Tools/curie/bin/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
939      ## /ccc/cont003/home/igcmg/igcmg/Tools/curie/bin/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
940      status=$?
941    fi
942
943#    if [ ${status} -gt 0 ] ; then
944#      echo "IGCM_sys_Dods_Rm : error."
945#      cat out_dods_rm
946#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
947#    else
948#      rm out_dods_rm
949#    fi
950
951  fi
952  return $status
953}
954
955#D-#==================================================
956#D-function IGCM_sys_Dods_Cp
957#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
958#D-* Examples:
959#D-
960function IGCM_sys_Dods_Cp {
961  if ( $DEBUG_sys ) ; then
962    echo "IGCM_sys_Dods_Cp :" $@
963  fi
964  typeset status
965  if [ $DRYRUN = 0 ]; then
966
967#    if [ ! -d ${R_SAVE}/${1} ] ; then
968#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
969#      echo "Nothing has been done."
970#      return
971#    fi
972
973     $(ccc_home -u igcmg)/Tools/curie/bin/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
974    ## /ccc/cont003/home/igcmg/igcmg/Tools/curie/bin/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
975    status=$?
976
977#       if [ ${status} -gt 0 ] ; then
978#           echo "IGCM_sys_Dods_Cp : error."
979#           cat out_dods_cp
980#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
981#       else
982#           rm out_dods_cp
983#       fi
984
985  fi
986  return $status
987}
988
989#D-#==================================================
990#D-function IGCM_sys_Put_Dods
991#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
992#D-* Examples:
993#D-
994function IGCM_sys_Put_Dods {
995  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
996  if ( $DEBUG_sys ) ; then
997    echo "IGCM_sys_Put_Dods :" $@
998  fi
999  typeset status
1000  if [ $DRYRUN = 0 ]; then
1001    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1002      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1003      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1004      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1005      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1006      return
1007    fi
1008
1009    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1010      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1011      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1012      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1013      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1014      return
1015    fi
1016    #
1017    if [ -d ${R_SAVE}/${1} ] ; then
1018      cd ${R_SAVE}
1019    elif [ -d ${R_FIGR}/${1} ] ; then
1020      cd ${R_FIGR}
1021    fi
1022
1023    IGCM_sys_Dods_Rm ${1}
1024    IGCM_sys_Dods_Cp ${1}
1025    status=0
1026
1027    if [ ${status} -gt 0 ] ; then
1028      echo "IGCM_sys_Put_Dods : error."
1029      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1030    fi
1031  fi
1032  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1033}
1034
1035##############################################################
1036# REBUILD OPERATOR
1037
1038#D-#==================================================
1039#D-function IGCM_sys_sync
1040#D-* Purpose: flush buffer on disk
1041#D-* Examples:
1042#D-
1043function IGCM_sys_sync {
1044  IGCM_debug_PushStack "IGCM_sys_sync" $@
1045  if ( $DEBUG_sys ) ; then
1046    echo "IGCM_sys_sync :" $@
1047  fi
1048
1049  /bin/sync
1050
1051  IGCM_debug_PopStack "IGCM_sys_sync"
1052}
1053
1054############################################################
1055# Activate Running Environnment Variables
1056
1057#D-#==================================================
1058#D-function IGCM_sys_activ_variables
1059#D-* Purpose: set environement variables prior to execution
1060#D-* Examples:
1061#D-
1062function IGCM_sys_activ_variables {
1063  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1064  if ( $DEBUG_sys ) ; then
1065    echo "IGCM_sys_activ_variables"
1066  fi
1067
1068# --------------------------------------------------------------------
1069#D- MODULE specifications
1070# --------------------------------------------------------------------
1071  module purge > /dev/null 2>&1
1072  if [ X${compilerVersion} = XV17 ] ; then
1073      . $(ccc_home -u igcmg)/MachineEnvironment/curie/env_intel17.0.2_curie   > /dev/null 2>&1
1074      ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/curie/env_intel17.0.2_curie   > /dev/null 2>&1
1075  elif [ X${compilerVersion} = XV14 ] ; then
1076      . $(ccc_home -u igcmg)/MachineEnvironment/curie/env_intel14.0.3_curie   > /dev/null 2>&1
1077      ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/curie/env_intel14.0.3_curie   > /dev/null 2>&1
1078  else
1079    IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1080  fi
1081  . $(ccc_home -u igcmg)/MachineEnvironment/curie/env_netcdf4.3.3.1_curie > /dev/null 2>&1
1082  ##. /ccc/cont003/home/igcmg/igcmg/MachineEnvironment/curie/env_netcdf4.3.3.1_curie > /dev/null 2>&1
1083
1084  # Keep track of this
1085  [ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
1086  [ ! X${TaskType} = Xchecking ] && module list
1087
1088# --------------------------------------------------------------------
1089#D- MPI specifications
1090# --------------------------------------------------------------------
1091
1092# --------------------------------------------------------------------
1093#D- Other specifications
1094# --------------------------------------------------------------------
1095
1096  ulimit -s unlimited
1097
1098  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1099}
1100
1101############################################################
1102# Desactivate Running Environnment Variables
1103
1104#D-#==================================================
1105#D-function IGCM_sys_desactiv_variables
1106#D-* Purpose: unset environement variables after execution
1107#D-* Examples:
1108#D-
1109function IGCM_sys_desactiv_variables {
1110  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1111  if ( $DEBUG_sys ) ; then
1112    echo "IGCM_sys_desactiv_variables"
1113  fi
1114# --------------------------------------------------------------------
1115#D- MPI specifications
1116# --------------------------------------------------------------------
1117
1118# --------------------------------------------------------------------
1119#D- Other specifications
1120# --------------------------------------------------------------------
1121
1122  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1123}
1124
1125############################################################
1126# Update job headers to be used by the scheduler
1127
1128#D-#==================================================
1129#D-function IGCM_sys_updateHeaders
1130#D-* Purpose: Update job headers to be used by the scheduler
1131#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1132#D-
1133function IGCM_sys_updateHeaders {
1134  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1135  if ( $DEBUG_sys ) ; then
1136    echo "IGCM_sys_updateHeaders"
1137  fi
1138  typeset file
1139  file=$1
1140
1141  if [ ${executionType} -eq 1 ] ; then
1142    # MPMD + MPI : mpirun
1143    if [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ; then
1144        sed -e "/::openMPthreads::/d"                  \
1145            -e "s/::JobNumProcTot::/${coreNumber}/"    \
1146             ${file} > ${file}.tmp
1147    else
1148        # MPMD + MPI : ccc_mprun
1149        sed -e "/::openMPthreads::/d"                  \
1150            -e "s/::JobNumProcTot::/${coreNumber}/"    \
1151            -e "/#MSUB -x/d"                           \
1152            -e "/--cpu_bind=none/d"                    \
1153            ${file} > ${file}.tmp
1154    fi
1155  elif [ ${executionType} -eq 2 ] ; then
1156    # MPMD + MPI + OMP : mpirun/ccc_mprun/error
1157    if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1158      sed -e "/::openMPthreads::/d"                  \
1159          -e "s/::JobNumProcTot::/${coreNumber}/"    \
1160        ${file} > ${file}.tmp
1161    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1162      sed -e "/::openMPthreads::/d"                  \
1163          -e "s/::JobNumProcTot::/${coreNumber}/"    \
1164          -e "/--cpu_bind=none/d"                    \
1165        ${file} > ${file}.tmp
1166    else
1167     IGCM_debug_Print 1 "You have to set ExecutionModeOnCurie=ccc_mprun or mpirun in config.card"
1168     IGCM_debug_Exit "IGCM_sys_updateHeaders"
1169    fi
1170  elif [ ${executionType} -eq 3 ] ; then
1171    # SPMD + MPI/OMP
1172    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1173        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1174        -e "/#MSUB -x/d"                           \
1175        -e "/--cpu_bind=none/d"                    \
1176      ${file} > ${file}.tmp
1177
1178  elif [ ${executionType} -eq 4 ] ; then
1179    # SPMD + MPI only
1180    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1181        -e "/::openMPthreads::/d"                  \
1182        -e "/#MSUB -x/d"                           \
1183        -e "/--cpu_bind=none/d"                    \
1184      ${file} > ${file}.tmp
1185
1186  elif [ ${executionType} -eq 5 ] ; then
1187    # SPMD + OMP only
1188    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1189        -e "/::JobNumProcTot::/d"                  \
1190        -e "/#MSUB -x/d"                           \
1191        -e "/--cpu_bind=none/d"                    \
1192      ${file} > ${file}.tmp
1193
1194  elif [ ${executionType} -eq 6 ] ; then
1195    # SEQUENTIAL THEN
1196    sed -e "s/::JobNumProcTot::/1/"                \
1197        -e "/::openMPthreads::/d"                  \
1198        -e "/#MSUB -x/d"                           \
1199        -e "/--cpu_bind=none/d"                    \
1200      ${file} > ${file}.tmp
1201
1202  fi
1203
1204  IGCM_sys_Mv ${file}.tmp ${file}
1205
1206  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1207}
1208
1209############################################################
1210# Build MPI/OMP scripts run file (dummy function)
1211
1212#D-#==================================================
1213#D-function IGCM_sys_build_run_file
1214#D-* Purpose: build run file (deprecated)
1215#D-* Examples:
1216#D-
1217function IGCM_sys_build_run_file {
1218  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1219}
1220
1221############################################################
1222# Build MPI/OMP scripts
1223
1224#D-#==================================================
1225#D-function IGCM_sys_build_execution_scripts
1226#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1227#D-* Examples:
1228#D-
1229function IGCM_sys_build_execution_scripts
1230{
1231  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1232  if ( $DEBUG_sys ) ; then
1233    echo "IGCM_sys_build_execution_scripts " $@
1234  fi
1235
1236  EXECUTION=${HOST_MPIRUN_COMMAND}
1237
1238  if ( ${OK_PARA_MPMD} ) ; then
1239
1240    # MPMD mode
1241    # 1 MPI only
1242    # 2 MPI/OpenMP mpirun method
1243    # 3 MPI/OpenMP ccc_mprun method
1244
1245    if [ -f run_file ] ; then
1246      IGCM_sys_Rm -f run_file
1247    fi
1248    touch run_file
1249
1250    # case 1 : Only MPI (MPMD)
1251    if  ( ! ${OK_PARA_OMP} ) ; then
1252
1253        if ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then 
1254      # use of ccc_mprun (default mode)
1255      # Build run_file
1256
1257      # First loop on the components for the coupler ie oasis (only if oasis3)
1258      # the coupler ie oasis3 must be the first one
1259            for comp in ${config_ListOfComponents[*]} ; do
1260
1261                eval ExeNameIn=\${config_Executable_${comp}[0]}
1262                eval ExeNameOut=\${config_Executable_${comp}[1]}
1263
1264        # for CPL component only
1265                if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1266
1267                    eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1268                    eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1269                    echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1270                fi
1271            done
1272
1273      # Then second loop on the components
1274            for comp in ${config_ListOfComponents[*]} ; do
1275
1276                eval ExeNameIn=\${config_Executable_${comp}[0]}
1277                eval ExeNameOut=\${config_Executable_${comp}[1]}
1278
1279        # Only if we really have an executable for the component and not the coupler ie oasis:
1280                if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1281
1282                    eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1283                    eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1284                    echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1285                fi
1286            done
1287
1288            EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1289
1290            IGCM_sys_Chmod u+x run_file
1291            if ( $DEBUG_sys ) ; then
1292                echo "run_file contains : "
1293                cat run_file
1294            fi
1295
1296        elif [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ; then
1297            # use of mpirun
1298
1299                  # Use of mpirun instead of ccc_mprun
1300            EXECUTION="time mpirun"
1301
1302      #  Hosts treatment
1303            ${EXECUTION} hostname | sort | uniq > hosts.tmp
1304
1305            i=0
1306            rm -f hosts rankfile
1307            IGCM_debug_Print 1 "sys Curie, Hosts available :"
1308            for nodes in `cat hosts.tmp`
1309            do
1310                host[$i]=$nodes
1311                echo "${host[$i]}" >> hosts
1312                IGCM_debug_Print 1 ${host[$i]}
1313                i=$((i+1))
1314            done
1315            rm -f hosts.tmp
1316
1317            listnodes=${host[*]}
1318
1319            EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1320
1321      # Initialisation
1322            rank=0
1323            current_core=0
1324            init_exec=n
1325
1326      # Loop on the components
1327            for comp in ${config_ListOfComponents[*]} ; do
1328
1329                eval ExeNameIn=\${config_Executable_${comp}[0]}
1330                eval ExeNameOut=\${config_Executable_${comp}[1]}
1331
1332        # Not possible if oasis has an executable (i.e old version of oasis3)
1333                if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1334                    IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1335                    IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1336                    IGCM_debug_Verif_Exit
1337                fi
1338
1339        # Only if we really have an executable for the component :
1340                if [ "X${ExeNameOut}" != X\"\" ] ; then
1341
1342                    eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1343                    eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1344                    eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1345
1346                    echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1347                    echo ""  >> script_${ExeNameOut}.ksh
1348                    if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1349
1350            # Check if the number of threads is correct
1351                        case ${comp_proc_omp_loc} in
1352                            2|4|8|16)
1353                                IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1354                                ;;
1355                            *)
1356                                IGCM_debug_Exit "ERROR with OMP parameters !"
1357                                IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1358                                IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1359                                IGCM_debug_Verif_Exit
1360                                ;;
1361                        esac
1362                        echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1363                        echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1364                        echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1365                        echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1366                    fi
1367
1368          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1369                    echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1370                    echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1371                    echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1372                    IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1373
1374                    if [ ${init_exec} = y ] ; then
1375                        EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1376                    else
1377                        EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1378                        init_exec=y
1379                    fi
1380
1381          # Build rankfile : method used to assign cores and nodes for the MPI process
1382          # Ex :
1383          #rank 0=curie5296 slot=0,1,2,3
1384          #rank 1=curie5296 slot=4,5,6,7
1385          # Example of final command :
1386          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1387          # with script_lmdz.x.ksh :
1388          # #!/bin/ksh
1389          #export KMP_STACKSIZE=3g
1390          #export KMP_LIBRARY=turnaround
1391          #export MKL_SERIAL=YES
1392          #OMP_NUM_THREADS=4
1393          #./lmdz.x
1394                    if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1395                        (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1396                    else
1397                        (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1398                    fi
1399                    for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1400                        (( index_host = current_core / NB_CORE_PER_NODE ))
1401                        host_value=${host[${index_host}]}
1402                        (( slot =  current_core % NB_CORE_PER_NODE ))
1403                        virg=","
1404                        string_final=""
1405                        for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1406                            string=$index$virg
1407                            string_final=$string_final$string
1408                        done
1409                        string_final=$( echo $string_final | sed "s/.$//" )
1410                        echo "rank $rank=$host_value slot=$string_final" >> rankfile
1411                        (( rank = rank + 1 ))
1412                        (( current_core = current_core + offset_comp_proc_loc ))
1413                    done
1414                fi
1415            done
1416
1417        fi
1418
1419    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1420
1421    # 2 MPI/OpenMP mpirun method
1422
1423      # Use of mpirun instead of ccc_mprun
1424      EXECUTION="time mpirun"
1425
1426      #  Hosts treatment
1427      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1428
1429      i=0
1430      rm -f hosts rankfile
1431      IGCM_debug_Print 1 "sys Curie, Hosts available :"
1432      for nodes in `cat hosts.tmp`
1433      do
1434        host[$i]=$nodes
1435        echo "${host[$i]}" >> hosts
1436        IGCM_debug_Print 1 ${host[$i]}
1437        i=$((i+1))
1438      done
1439      rm -f hosts.tmp
1440
1441      listnodes=${host[*]}
1442
1443      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1444
1445      # Initialisation
1446      rank=0
1447      current_core=0
1448      init_exec=n
1449
1450      # Loop on the components
1451      for comp in ${config_ListOfComponents[*]} ; do
1452
1453        eval ExeNameIn=\${config_Executable_${comp}[0]}
1454        eval ExeNameOut=\${config_Executable_${comp}[1]}
1455
1456        # Not possible if oasis has an executable (i.e old version of oasis3)
1457        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1458          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1459          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1460          IGCM_debug_Verif_Exit
1461        fi
1462
1463        # Only if we really have an executable for the component :
1464        if [ "X${ExeNameOut}" != X\"\" ] ; then
1465
1466          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1467          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1468          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1469
1470          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1471          echo ""  >> script_${ExeNameOut}.ksh
1472          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1473
1474            # Check if the number of threads is correct
1475            case ${comp_proc_omp_loc} in
1476            2|4|8|16)
1477              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1478              ;;
1479            *)
1480              IGCM_debug_Exit "ERROR with OMP parameters !"
1481              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1482              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1483              IGCM_debug_Verif_Exit
1484              ;;
1485            esac
1486            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1487            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1488            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1489            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1490          fi
1491
1492          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1493          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1494          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1495          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1496          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1497
1498          if [ ${init_exec} = y ] ; then
1499            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1500          else
1501            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1502            init_exec=y
1503          fi
1504
1505          # Build rankfile : method used to assign cores and nodes for the MPI process
1506          # Ex :
1507          #rank 0=curie5296 slot=0,1,2,3
1508          #rank 1=curie5296 slot=4,5,6,7
1509          # Example of final command :
1510          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1511          # with script_lmdz.x.ksh :
1512          # #!/bin/ksh
1513          #export KMP_STACKSIZE=3g
1514          #export KMP_LIBRARY=turnaround
1515          #export MKL_SERIAL=YES
1516          #OMP_NUM_THREADS=4
1517          #./lmdz.x
1518          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1519              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1520          else
1521              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1522          fi
1523          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1524             (( index_host = current_core / NB_CORE_PER_NODE ))
1525             host_value=${host[${index_host}]}
1526             (( slot =  current_core % NB_CORE_PER_NODE ))
1527             virg=","
1528             string_final=""
1529             for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1530                 string=$index$virg
1531                 string_final=$string_final$string
1532             done
1533             string_final=$( echo $string_final | sed "s/.$//" )
1534             echo "rank $rank=$host_value slot=$string_final" >> rankfile
1535             (( rank = rank + 1 ))
1536             (( current_core = current_core + offset_comp_proc_loc ))
1537          done
1538        fi
1539      done
1540
1541    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1542
1543    # 3 MPI/OpenMP ccc_mprun method
1544
1545    # MPI-OpenMP (MPMD)
1546    # example of run_file
1547    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1548    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1549    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1550
1551      # Loop on the components
1552      for comp in ${config_ListOfComponents[*]} ; do
1553
1554        eval ExeNameIn=\${config_Executable_${comp}[0]}
1555        eval ExeNameOut=\${config_Executable_${comp}[1]}
1556
1557        # Not possible if oasis has an executable (i.e old version of oasis3)
1558        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1559          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1560          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1561          IGCM_debug_Verif_Exit
1562        fi
1563
1564        # Only if we really have an executable for the component :
1565        if [ "X${ExeNameOut}" != X\"\" ] ; then
1566
1567          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1568          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1569
1570          # Build script files
1571
1572          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1573          echo ""  >> script_${ExeNameOut}.ksh
1574          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1575
1576            # Check if the number of threads is correct
1577            case ${comp_proc_omp_loc} in
1578            2|4|8|16)
1579              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1580              ;;
1581            *)
1582              IGCM_debug_Exit "ERROR with OMP parameters !"
1583              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1584              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1585              IGCM_debug_Verif_Exit
1586              ;;
1587            esac
1588            ### only for LMDZ?
1589            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1590            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1591            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1592            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1593
1594          fi
1595
1596          # to have out/err per process on different files
1597          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1598          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1599
1600          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1601
1602          # Complete run_file
1603
1604          echo "${comp_proc_mpi_loc}-${comp_proc_omp_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1605
1606        fi
1607      done
1608
1609      ## variable added to stop after 60s instead of 600s by default.
1610      ## This is used when no error comes from executables and when something stopped an executable without notice.
1611      export SLURM_WAIT=60
1612
1613      ## module advised by TGCC (instead of 2 variables)
1614      module load feature/bridge/heterogenous_mpmd
1615
1616      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1617
1618      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1619
1620      IGCM_sys_Chmod u+x run_file
1621      if ( $DEBUG_sys ) ; then
1622        echo "run_file contains : "
1623        cat run_file
1624      fi
1625
1626    else
1627
1628      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1629
1630    fi # 1 2 or 3 if ${OK_PARA_MPMD}
1631
1632  else
1633  # Only one executable (SPMD mode).
1634
1635    for comp in ${config_ListOfComponents[*]} ; do
1636
1637      # Only if we really have an executable for the component :
1638      eval ExeNameOut=\${config_Executable_${comp}[1]}
1639      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1640
1641        # Build script files
1642
1643        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1644        echo ""  >> script_${ExeNameOut}.ksh
1645        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1646
1647        if ( ${OK_PARA_OMP} ) ; then
1648          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1649          echo ""  >> script_${ExeNameOut}.ksh
1650          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1651          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1652          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1653          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1654        fi
1655
1656        if  ( ${OK_PARA_MPI} ) ; then
1657          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1658          # Default : ccc_mprun used if nb_proc gt 1
1659          # to have out/err per process on different files
1660          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1661          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1662          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1663        else
1664          # Default : ccc_mprun is NOT used if nb_proc eq 1
1665          # to have out/err per process on different files
1666          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1667          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1668          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1669        fi
1670
1671        IGCM_debug_Print 1 "sys Curie : script_${ExeNameOut}.ksh contains"
1672        cat script_${ExeNameOut}.ksh
1673
1674      fi
1675
1676    done
1677
1678  fi # ${OK_PARA_MPMD}
1679
1680  IGCM_debug_Print 1 "sys Curie : execution command is "
1681  IGCM_debug_Print 1 "$EXECUTION"
1682
1683  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1684}
1685
1686#D-#==================================================
1687#D-function IGCM_sys_check_path
1688#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1689#D-* do not point to an important use directory. Stop immediately in that case.
1690#D-* Examples:
1691#D-
1692function IGCM_sys_check_path {
1693  IGCM_debug_PushStack "IGCM_sys_check_path"
1694  if ( $DEBUG_sys ) ; then
1695    echo "IGCM_sys_check_path"
1696  fi
1697
1698  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${WORKDIR} ] || [ X${RUN_DIR_PATH} = X${SCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1699    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1700    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1701    IGCM_debug_Exit "This will stop the job"
1702  fi
1703  IGCM_debug_PopStack "IGCM_sys_check_path"
1704}
1705
1706#D-#==================================================
1707#D-function IGCM_sys_check_quota
1708#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1709#D-* Examples:
1710#D-
1711function IGCM_sys_check_quota {
1712  IGCM_debug_PushStack "IGCM_sys_check_quota"
1713  if ( $DEBUG_sys ) ; then
1714    echo "IGCM_sys_check_quota"
1715  fi
1716  # Limit of quota (in %)
1717  limit_quota=90
1718
1719  # Check of the volume
1720  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1721  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1722
1723  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1724
1725    unit_avail=${volume_avail: -1}
1726    unit_quota=${volume_quota: -1}
1727
1728    if [ "${unit_quota}" = "*" ] ; then
1729      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1730      IGCM_debug_Print 1 "More than 100% of your quota is used"
1731      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1732      IGCM_debug_Print 1 "You must have more than 10% available to run"
1733      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1734      IGCM_debug_Verif_Exit
1735    fi
1736
1737    temp_avail=${volume_avail%%${volume_avail: -1}*}
1738    temp_quota=${volume_quota%%${volume_quota: -1}*}
1739
1740    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1741
1742    # Convertion
1743      if [ ${volume_avail: -1} = "T" ] ; then
1744        (( temp_avail = temp_avail * 1000000000000 ))
1745      elif [ ${volume_avail: -1} = "G" ] ; then
1746        (( temp_avail = temp_avail * 1000000000 ))
1747      elif [ ${volume_avail: -1} = "M" ] ; then
1748        (( temp_avail = temp_avail * 1000000 ))
1749      elif [ ${volume_avail: -1} = "k" ] ; then
1750        (( temp_avail = temp_avail * 1000 ))
1751      else
1752        (( temp_avail = volume_avail ))
1753      fi
1754      if [ ${volume_quota: -1} = "T" ] ; then
1755        (( temp_quota = temp_quota * 1000000000000 ))
1756      elif [ ${volume_quota: -1} = "G" ] ; then
1757        (( temp_quota = temp_quota * 1000000000 ))
1758      elif [ ${volume_quota: -1} = "M" ] ; then
1759        (( temp_quota = temp_quota * 1000000 ))
1760      elif [ ${volume_quota: -1} = "k" ] ; then
1761        (( temp_quota = temp_quota * 1000 ))
1762      else
1763        (( temp_quota = volume_quota ))
1764      fi
1765    fi
1766
1767    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1768#    echo "volume ratio is " $quota_volume
1769
1770    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1771      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1772      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1773      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1774      IGCM_debug_Print 1 "You must have more than 10% available to run"
1775      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1776      IGCM_debug_Verif_Exit
1777    fi
1778
1779  fi
1780
1781# Check of the number of inodes
1782
1783  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1784  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1785
1786  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1787
1788    unit_avail=${inode_avail: -1}
1789    unit_quota=${inode_quota: -1}
1790
1791    if [ "${unit_quota}" = "*" ] ; then
1792      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1793      IGCM_debug_Print 1 "More than 100% of your quota is used"
1794      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1795      IGCM_debug_Print 1 "You must have more than 10% available to run"
1796      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1797      IGCM_debug_Verif_Exit
1798    fi
1799
1800    temp_avail=${inode_avail%%${inode_avail: -1}*}
1801    temp_quota=${inode_quota%%${inode_quota: -1}*}
1802
1803    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1804
1805    # Convertion
1806      if [ ${inode_avail: -1} = "T" ] ; then
1807        (( temp_avail = temp_avail * 1000000000000 ))
1808      elif [ ${inode_avail: -1} = "G" ] ; then
1809        (( temp_avail = temp_avail * 1000000000 ))
1810      elif [ ${inode_avail: -1} = "M" ] ; then
1811        (( temp_avail = temp_avail * 1000000 ))
1812      elif [ ${inode_avail: -1} = "k" ] ; then
1813        (( temp_avail = temp_avail * 1000 ))
1814      else
1815        (( temp_avail = inode_avail ))
1816      fi
1817
1818      if [ ${inode_quota: -1} = "T" ] ; then
1819        (( temp_quota = temp_quota * 1000000000000 ))
1820      elif [ ${inode_quota: -1} = "G" ] ; then
1821        (( temp_quota = temp_quota * 1000000000 ))
1822      elif [ ${inode_quota: -1} = "M" ] ; then
1823        (( temp_quota = temp_quota * 1000000 ))
1824      elif [ ${inode_quota: -1} = "k" ] ; then
1825        (( temp_quota = temp_quota * 1000 ))
1826      else
1827        (( temp_quota = inode_quota ))
1828      fi
1829    fi
1830    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1831#    echo "inode ratio is " $quota_inode
1832
1833    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1834      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1835      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1836      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1837      IGCM_debug_Print 1 "You must have more than 10% available to run"
1838      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1839      IGCM_debug_Verif_Exit
1840    fi
1841  fi
1842  IGCM_debug_PopStack "IGCM_sys_check_quota"
1843}
1844
1845#D-#==================================================
1846#D-function IGCM_sys_projectAccounting
1847#D-* Purpose: store project accounting information in a file
1848#D-* Examples:
1849#D-
1850function IGCM_sys_projectAccounting {
1851  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1852  if ( $DEBUG_sys ) ; then
1853    echo "IGCM_sys_check_quota"
1854  fi
1855
1856  ssh curie91 /usr/bin/ccc_myproject > $1
1857
1858  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1859}
1860
1861#D-#==================================================
1862#D-function IGCM_sys_getJobSchedulerID
1863#D-* Purpose: Get the job ID during execution
1864#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1865#D-
1866function IGCM_sys_getJobSchedulerID {
1867  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1868  if ( $DEBUG_sys ) ; then
1869    echo "IGCM_sys_getJobSchedulerID"
1870  fi
1871
1872  eval ${1}=${BRIDGE_MSUB_JOBID}
1873
1874  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1875}
1876
1877#D-#==================================================
1878#D-function IGCM_sys_GetJobID
1879#D-* Purpose: Get the job ID from the JobName
1880#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1881#D-
1882function IGCM_sys_GetJobID {
1883  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1884  if ( $DEBUG_sys ) ; then
1885    echo "IGCM_sys_GetJobID"
1886  fi
1887
1888  # With -f option, the full job name is given in the last column
1889  ID=$( ccc_mstat -f -u $2 | \
1890        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1891
1892  eval ${3}=${ID}
1893  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1894}
1895
1896#D-#==================================================
1897#D-function IGCM_sys_CountJobInQueue
1898#D-* Purpose: Count number of users job
1899#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1900#D-
1901function IGCM_sys_CountJobInQueue {
1902  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1903  if ( $DEBUG_sys ) ; then
1904    echo "IGCM_sys_CountJobInQueue"
1905  fi
1906
1907  # With -f option, the full job name is given in the last column
1908  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1909
1910  eval ${2}=${NbRun}
1911
1912  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1913}
1914
1915#D-#==================================================
1916#D-function IGCM_sys_ListJobInQueue
1917#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1918#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1919#D-
1920function IGCM_sys_ListJobInQueue {
1921  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1922  if ( $DEBUG_sys ) ; then
1923    echo "IGCM_sys_ListJobInQueue"
1924  fi
1925
1926  # With -f option, the full job name is given in the last column
1927  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1928                                        '( $2  == User      && \
1929                                           $NF != /TS/      && \
1930                                           $NF !~ /PACK/    && \
1931                                           $NF !~ /REBUILD/ && \
1932                                           $NF !~ /pack/ )     \
1933                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1934
1935  eval set -A ${2} ${JobList[*]}
1936
1937  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1938}
1939
1940#D-#==================================================
1941#D-function IGCM_sys_atlas
1942#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1943#D-* Examples:
1944#D-
1945function IGCM_sys_atlas {
1946  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1947  if ( $DEBUG_sys ) ; then
1948    echo "IGCM_sys_atlas :" $@
1949  fi
1950
1951  typeset status
1952
1953  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1954  status=$?
1955  if [ ${status} -gt 0 ] ; then
1956    echo "IGCM_sys_atlas : error code ${status}"
1957    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1958    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1959    IGCM_debug_PopStack "IGCM_sys_atlas"
1960    return 1
1961  else
1962    IGCM_debug_PopStack "IGCM_sys_atlas"
1963    return 0
1964  fi
1965
1966  IGCM_debug_PopStack "IGCM_sys_atlas"
1967}
1968
1969#D-#==================================================
1970#D-function IGCM_sys_rebuild_nemo
1971#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1972#D-* Examples:
1973#D-
1974
1975function IGCM_sys_rebuild_nemo {
1976  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1977  if ( $DEBUG_sys ) ; then
1978    echo "IGCM_sys_rebuild_nemo :" $@
1979  fi
1980
1981  $( ccc_home -u igcmg)/Tools/curie/bin/rebuild_nemo ${1} ${2}
1982
1983  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1984}
1985
Note: See TracBrowser for help on using the repository browser.