source: trunk/libIGCM/libIGCM_sys/libIGCM_sys_irene-amd.ksh @ 1593

Last change on this file since 1593 was 1593, checked in by aclsce, 10 months ago

Fixed a bug in the computing of number of cores needed in case of irene-amd and dedicated node for XIOS server.
File size: 66.2 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Sebastien Denvil, Martial Mancip, Arnaud Caubel
5# Contact: Arnaud.Caubel__at__lsce.ipsl.fr
6# $Revision:: 1512                                     $ Revision of last commit
7# $Author:: acosce                                     $ Author of last commit
8# $Date:: 2020-01-14 18:52:20 +0100 (Tue, 14 Jan 2020) $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13
14#=========================================================
15# The documentation of this file can be automatically generated
16# if you use the prefix #D- for comments to be extracted.
17# Extract with command: cat lib* | grep "^#D-" | cut -c "4-"
18#=========================================================
19
20#D-#==================================================
21#D-LibIGCM_sys for Irene-amd
22#D-#==================================================
23#D-
24#D- This ksh library if a layer under some usefull
25#D-environment variables and shell commands.
26#D-All those definitions depend on host particularities.
27#D-It manages a stack mechanism and test validity of operations.
28#D-All function described bellow must be prefixed by IGCM_sys.
29
30#====================================================
31# libIGCM_sys PARAMETERS
32#====================================================
33
34#====================================================
35# set DEBUG_sys to true to output calls of function
36typeset -r DEBUG_sys=${DEBUG_sys:=true}
37
38#====================================================
39# Turn in dry run mode ? (sys_Put_Rest, sys_Put_Out, sys_Get)
40if ([ X${DRYRUN} != X ] && [ ${DRYRUN} = 4 ]) ; then
41  typeset -r DRYRUN=0
42  typeset -r DRYRUN_DEBUG=4
43else
44  typeset -r DRYRUN=${DRYRUN:=0}
45  typeset -r DRYRUN_DEBUG=0
46fi
47
48# YOU MUST COMPILE YOUR EXE FILES FOR DRYRUN MODE !
49# -------------------------------------------------------------------------------------
50# | DRYRUN=  |  Date computations, | sys_Get    |  Exe    | sys_Put_Out; sys_Put_Rest |
51# |          |  Cp/Exe/param/files |            |         |                           |
52# |          |  Chmod Qsub         |            |         |                           |
53# -------------------------------------------------------------------------------------
54# |    0     |       yes           |    yes     |  yes    |      yes                  |
55# -------------------------------------------------------------------------------------
56# |    1     |       yes           |    yes     |  yes    |      no                   |
57# -------------------------------------------------------------------------------------
58# |    2     |       yes           |    yes     |  no     |      no                   |
59# -------------------------------------------------------------------------------------
60# |    3     |       yes           |    no      |  no     |      no                   |
61# -------------------------------------------------------------------------------------
62# |    4     |       yes           |    yes     |  creation of RUN_DIR and Job_debug  |
63# -------------------------------------------------------------------------------------
64
65
66#=====================================================
67# Global Variables :
68#=====================================================
69# Language : "fr" or "en"
70typeset -r MYLANG="fr"
71
72#=====================================================
73# Host user names project maxCpuTime
74# $hostname ou hostname
75typeset HOST=${HOST:=$( hostname )}
76# $username ou whoami
77typeset LOGIN=${LOGIN:=$( whoami )}
78# $hostname of the MASTER job
79typeset MASTER=irene-amd
80# add default project on irene-amd
81typeset PROJECT=$(echo ${BRIDGE_MSUB_PROJECT:=::default_project::} | cut -d@ -f1 )
82# jobWarningDelay in seconds
83typeset jobWarningDelay=${BRIDGE_MSUB_MAXTIME}
84
85#D-
86#D-#==================================================
87#D-Program used in libIGCM
88#D-#==================================================
89
90# Submit command
91typeset SUBMIT=${SUBMIT:=ccc_msub}
92# rsync with path
93typeset -r RSYNC=/usr/bin/rsync
94# RSYNC_opt args to rsync
95typeset -r RSYNC_opt="-va"
96# ie storage filesystem
97typeset -r STOREHOST=${MASTER}
98typeset -r REMOTE_RSYNC=/usr/bin/rsync
99
100#====================================================
101# Set environment tools (ferret, nco, cdo, rebuild, ...)
102#====================================================
103typeset CCCOS=${CCCOS:=$( ccc_os )}
104
105if [ X${TaskType} = Xcomputing ] ; then
106  IGCM_debug_Print 1 "Modules will be loaded later in IGCM_sys_activ_variables."
107else
108  module purge > /dev/null 2>&1
109  if [ X${CCCOS} = X"Rhel_8__x86_64" ] ; then
110      . $( ccc_home -u igcmg)/MachineEnvironment/irene/env_atlas_irene > /dev/null 2>&1
111  else
112      . $( ccc_home -u igcmg)/MachineEnvironment/irene-amd_prev/env_irene-amd > /dev/null 2>&1
113      . $( ccc_home -u igcmg)/MachineEnvironment/irene-amd_prev/env_atlas_irene-amd > /dev/null 2>&1
114  fi
115
116  export PCMDI_MP=/ccc/work/cont003/igcmg/igcmg/PCMDI-MP
117  export UVCDAT_ANONYMOUS_LOG=FALSE
118fi
119
120# FYI
121[ ! X${TaskType} = Xchecking ] && IGCM_debug_Print 1 "List of loaded modules:"
122[ ! X${TaskType} = Xchecking ] && module list
123
124# For AddNoise and AddPertu3DOCE
125export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/bin
126
127# For rebuild
128export PATH=${PATH}:$(ccc_home -u igcmg)/Tools/irene/rebuild/modipsl/bin
129
130#====================================================
131# Host specific DIRECTORIES
132#====================================================
133
134# ============ CESIUM START ============ #
135
136#====================================================
137#- Mirror libIGCM from titane to cesium if needed
138#ROOTSYS=$( echo ${libIGCM} | gawk -F"/" '{print $3}' )
139#if [ ! ${ROOTSYS} = "home" ] ; then
140#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=true}
141#else
142#  typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
143#fi
144
145#====================================================
146#- libIGCM_POST
147#if ( ${MirrorlibIGCM} ) ; then
148#  PATHlibIGCM=$( echo ${libIGCM} | gawk -F"${LOGIN}/" '{print $2}' | sed -e "s&/libIGCM&&" )
149#  typeset -r libIGCM_POST=${HOME}/MIRROR/${PATHlibIGCM}/libIGCM
150#else
151#  typeset -r libIGCM_POST=${libIGCM}
152#fi
153
154# ============ CESIUM  END  ============ #
155
156#====================================================
157#- MirrorlibIGCM for frontend
158typeset -r MirrorlibIGCM=${MirrorlibIGCM:=false}
159
160#====================================================
161#- libIGCM_POST for frontend
162typeset -r libIGCM_POST=${libIGCM}
163
164#====================================================
165#- R_EXE   (==> BIN_DIR = ${MODIPSL}/bin )
166typeset -r R_EXE="${MODIPSL}/bin"
167
168#====================================================
169#- SUBMIT_DIR : submission dir
170typeset -x SUBMIT_DIR=${SUBMIT_DIR:=${BRIDGE_MSUB_PWD}}
171
172#====================================================
173#- IN
174typeset -r R_IN=${R_IN:=/ccc/work/cont003/igcmg/igcmg/IGCM}
175
176#====================================================
177#- RUN_DIR_PATH : Temporary working directory (=> TMP)
178typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
179
180#====================================================
181#- OUTCOMMAND_PATH : tmp place to store command lines standard error and outputs
182typeset -r OUTCOMMAND_PATH=/tmp
183
184#====================================================
185#- HOST_MPIRUN_COMMAND
186typeset -r HOST_MPIRUN_COMMAND=${HOST_MPIRUN_COMMAND:="/usr/bin/time ccc_mprun -E-K1 "}
187
188#====================================================
189#- Max number of arguments passed to nco operator or demigration command
190UNIX_MAX_LIMIT=360
191
192#====================================================
193#- set PackDefault to true on irene-amd
194PackDefault=true
195
196#====================================================
197#- Default number of MPI task for IPSL coupled model
198#- required for backward compatibility
199#-
200DEFAULT_NUM_PROC_OCE=5
201DEFAULT_NUM_PROC_CPL=1
202(( DEFAULT_NUM_PROC_ATM = BATCH_NUM_PROC_TOT - DEFAULT_NUM_PROC_OCE - DEFAULT_NUM_PROC_CPL ))
203
204#====================================================
205#- Number of computing cores per node
206#-
207NB_CORE_PER_NODE=128
208
209#D-#==================================================
210#D-function IGCM_sys_defineArchives
211#D-* Purpose:
212#D-* Load dfldatadir module : According to project used for submission (default) or set in config.card (optional)
213#D-* Define ARCHIVE : Dedicated to large files
214#D-* Define STORAGE : Dedicated to small/medium files
215#D-* Define R_OUT   : Output tree located on ARCHIVE
216#D-* Define R_FIG   : Output tree located on STORAGE hosting figures (monitoring and atlas, and/or small files)
217#D-* Define R_BUF   : Output tree located on CCCSCRATCHDIR hosting files waiting for rebuild or pack processes
218#D-* if SpaceName=TEST everything is stored on CCCSCRATCHDIR
219#D-* Examples:
220#D-
221function IGCM_sys_defineArchives {
222  IGCM_debug_PushStack "IGCM_sys_defineArchives"
223
224  # Load dfldatadir depending on the project used for submission (default) or set in config.card (optional)
225  if [ X${config_UserChoices_DataProject} = X ] || [ X${config_UserChoices_DataProject} = DEFAULT ]; then
226    # Default option: Change the dfldatadir according to the project used for submission
227    # The variable DataProject in section UserChoices in config.card is not set or is set to DEFAULT
228    if [ X"$( echo ${PROJECT} | grep cmip6 )" != "X" ] ; then
229      module switch dfldatadir dfldatadir/gencmip6
230      IGCM_debug_Print 1 "Load dfldatadir for forced gencmip6 project"
231    else
232      module switch dfldatadir dfldatadir/${PROJECT}
233      IGCM_debug_Print 1 "Load dfldatadir for project ${PROJECT}"
234    fi
235  else
236    # Use the project set by the variable DataProject in section UserChoices in config.card
237    module switch dfldatadir dfldatadir/${config_UserChoices_DataProject}
238    IGCM_debug_Print 1 "Load dfldatadir for DataProject red in config.card : ${config_UserChoices_DataProject}"
239  fi
240
241 #====================================================
242 #- RUN_DIR_PATH : Temporary working directory (=> TMP)
243 typeset -r RUN_DIR_PATH=${RUN_DIR_PATH:=${CCCSCRATCHDIR}/RUN_DIR/${BRIDGE_MSUB_JOBID}_${$}}
244
245  if [ ! X${config_UserChoices_ARCHIVE} = X ]; then
246    #====================================================
247    #- ARCHIVE (dedicated to large files)
248    ARCHIVE=${config_UserChoices_ARCHIVE}
249  else
250    #====================================================
251    #- ARCHIVE (dedicated to large files)
252    ARCHIVE=${CCCSTOREDIR}
253  fi
254
255  if [ ! X${config_UserChoices_STORAGE} = X ]; then
256    #====================================================
257    #- STORAGE (dedicated to small/medium files)
258    STORAGE=${config_UserChoices_STORAGE}
259  else
260    #====================================================
261    #- STORAGE (dedicated to small/medium files)
262    STORAGE=${CCCWORKDIR}
263  fi
264
265  if [ X${config_UserChoices_SpaceName} = XTEST ]; then
266    #====================================================
267    #- R_OUT
268    R_OUT=${CCCSCRATCHDIR}/IGCM_OUT
269
270    #====================================================
271    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
272    R_FIG=${CCCSCRATCHDIR}/IGCM_OUT
273
274    IGCM_debug_Print 1 "SpaceName=TEST ==> OVERRULE destination path directories"
275
276  else
277    #====================================================
278    #- R_OUT
279    R_OUT=${ARCHIVE}/IGCM_OUT
280
281    #====================================================
282    #- R_FIG (hosting figures : monitoring and atlas, and/or small files)
283    R_FIG=${STORAGE}/IGCM_OUT
284  fi
285
286  #====================================================
287  #- CMIP6 (hosting CMIP6 files produced by XIOS2 and configured by dr2xml)
288  CMIP6_BUF=${STORAGE}/IGCM_OUT
289
290  #====================================================
291  #- R_BUF (ONLY FOR double copy an scratch)
292  R_BUF=${CCCSCRATCHDIR}/IGCM_OUT
293
294  IGCM_debug_Print 1 "R_OUT has been defined = ${R_OUT}"
295  IGCM_debug_Print 1 "R_BUF has been defined = ${R_BUF}"
296  IGCM_debug_Print 1 "R_FIG has been defined = ${R_FIG}"
297
298  IGCM_debug_PopStack "IGCM_sys_defineArchives"
299}
300
301#D-#==================================================
302#D-function IGCM_sys_RshArchive
303#D-* Purpose: Archive rsh command
304#D-* Examples:
305#D-
306function IGCM_sys_RshArchive {
307  IGCM_debug_PushStack "IGCM_sys_RshArchive" $@
308  /bin/ksh <<-EOF
309    ${@}
310EOF
311  status=$?
312  if [ ${status} -gt 0 ] ; then
313    IGCM_debug_Print 2 "IGCM_sys_RshArchive : command failed error code ${status}"
314    IGCM_debug_Exit "IGCM_sys_RshArchive"
315  fi
316  IGCM_debug_PopStack "IGCM_sys_RshArchive"
317}
318
319#D-#==================================================
320#D-function IGCM_sys_RshArchive_NoError
321#D-* Purpose: Archive rsh command, without error
322#D-*          used only in monitoring.job
323#D-* Examples:
324#D-
325function IGCM_sys_RshArchive_NoError {
326  IGCM_debug_PushStack "IGCM_sys_RshArchive_NoError" $@
327  /bin/ksh <<-EOF
328    ${@} 2> /dev/null
329EOF
330  IGCM_debug_PopStack "IGCM_sys_RshArchive_NoError"
331}
332
333#D-#==================================================
334#D-function IGCM_sys_MkdirArchive
335#D-* Purpose: Mkdir on Archive
336#D-* Examples:
337#D-
338function IGCM_sys_MkdirArchive {
339  IGCM_debug_PushStack "IGCM_sys_MkdirArchive" $@
340  if ( $DEBUG_sys ) ; then
341    echo "IGCM_sys_MkdirArchive :" $@
342  fi
343  #- creation de repertoire sur le serveur fichier
344  if [ ! -d ${1} ]; then
345    \mkdir -p $1
346    status=$?
347
348    if [ ${status} -gt 0 ] ; then
349      IGCM_debug_Print 2 "IGCM_sys_MkdirArchive : mkdir failed error code ${status}"
350      IGCM_debug_Exit "IGCM_sys_MkdirArchive"
351    fi
352  fi
353  IGCM_debug_PopStack "IGCM_sys_MkdirArchive"
354}
355
356#D-#==================================================
357#D-function IGCM_sys_TestDirArchive
358#D-* Purpose: Test Directory that must exists on Archive
359#D-* Examples:
360#D-
361function IGCM_sys_TestDirArchive {
362  IGCM_debug_PushStack "IGCM_sys_TestDirArchive" $@
363  if ( $DEBUG_sys ) ; then
364    echo "IGCM_sys_TestDirArchive :" $@
365  fi
366  typeset ExistFlag
367  ExistFlag=$( [ -d $1 ] && echo 0 || echo 1 )
368  IGCM_debug_PopStack "IGCM_sys_TestDirArchive"
369  return ${ExistFlag}
370}
371
372#D-#==================================================
373#D-function IGCM_sys_IsFileArchived
374#D-* Purpose: Test file that must NOT EXISTS on Archive based on filename only
375#D-* Examples:
376#D-
377function IGCM_sys_IsFileArchived {
378  IGCM_debug_PushStack "IGCM_sys_IsFileArchived" $@
379  if ( $DEBUG_sys ) ; then
380    echo "IGCM_sys_IsFileArchived :" $@
381  fi
382  typeset IsArchivedFlag
383  IsArchivedFlag=$( [ X$( echo $@ | grep ^\/ccc\/store ) != X ] && echo 0 || echo 1 )
384  IGCM_debug_PopStack "IGCM_sys_IsFileArchived"
385
386  return ${IsArchivedFlag}
387}
388
389#D-#==================================================
390#D-function IGCM_sys_TestFileArchive
391#D-* Purpose: Test file that must NOT EXISTS on Archive
392#D-* Examples:
393#D-
394function IGCM_sys_TestFileArchive {
395  IGCM_debug_PushStack "IGCM_sys_TestFileArchive" $@
396  typeset ExistFlag
397  ExistFlag=$( [ -f $1 ] && echo 0 || echo 1 )
398  IGCM_debug_PopStack "IGCM_sys_TestFileArchive"
399
400  return ${ExistFlag}
401}
402
403#D-#==================================================
404#D-function IGCM_sys_CountFileArchive
405#D-* Purpose: Count files on Archive filesystem
406#D-* Examples:
407#D-
408function IGCM_sys_CountFileArchive {
409  IGCM_debug_PushStack "IGCM_sys_CountFileArchive" $@
410  ls ${@} 2>/dev/null | wc -l
411  if [ $? -gt 0 ] ; then
412    echo "IGCM_sys_CountFileArchive : erreur."
413  fi
414  IGCM_debug_PopStack "IGCM_sys_CountFileArchive"
415}
416
417#D-#==================================================
418#D-function IGCM_sys_Tree
419#D-* Purpose: Tree directories with files on ${ARCHIVE}
420#D-* Examples: IGCM_sys_Tree ${R_IN} ${R_OUT}
421#D-
422function IGCM_sys_Tree {
423  IGCM_debug_PushStack "IGCM_sys_Tree" $@
424  if ( $DEBUG_sys ) ; then
425    echo "IGCM_sys_Tree :" $@
426  fi
427
428  \ls -lR ${@}
429
430  IGCM_debug_PopStack "IGCM_sys_Tree"
431}
432
433#D-#==================================================
434#D-function IGCM_sys_Qsub
435#D-* Purpose: Qsub new job
436#D-* Examples:
437#D-
438function IGCM_sys_Qsub {
439  IGCM_debug_PushStack "IGCM_sys_Qsub" $@
440  if ( $DEBUG_sys ) ; then
441    echo "IGCM_sys_Qsub :" $@
442  fi
443  typeset options status
444  options="-o ${SUBMIT_DIR}/${Script_Output} -e ${SUBMIT_DIR}/${Script_Output}"
445
446  /usr/bin/ccc_msub ${options} $1 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
447  status=$?
448
449  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
450  if [ ${status} -gt 0 ] ; then
451    IGCM_debug_Print 2 "IGCM_sys_Qsub ${options} $1 : error code ${status}"
452    IGCM_debug_Exit "IGCM_sys_Qsub"
453  else
454    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
455  fi
456  IGCM_debug_PopStack "IGCM_sys_Qsub"
457}
458
459#D-#==================================================
460#D-function IGCM_sys_QsubPost
461#D-* Purpose: Qsub new job on scalaire
462#D-* Examples:
463#D-
464function IGCM_sys_QsubPost {
465  IGCM_debug_PushStack "IGCM_sys_QsubPost" $@
466  if ( $DEBUG_sys ) ; then
467    echo "IGCM_sys_QsubPost :" $@
468  fi
469  typeset options status
470  options="-Q normal -o ${POST_DIR}/${Script_Post_Output}.out -e ${POST_DIR}/${Script_Post_Output}.out"
471
472  /usr/bin/ccc_msub ${options} ${libIGCM_POST}/$1.job > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
473  status=$?
474
475  cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
476  if [ ${status} -gt 0 ] ; then
477    IGCM_debug_Print 2 "IGCM_sys_QsubPost ${options} ${libIGCM_POST}/$1.job : error code ${status}"
478    IGCM_debug_Exit "IGCM_sys_QsubPost"
479  else
480    JobID=$( gawk {'print $4'} ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )
481  fi
482  IGCM_debug_PopStack "IGCM_sys_QsubPost"
483}
484
485#D-*************************
486#D- File transfer functions
487#D-*************************
488#D-
489
490#D-#==================================================
491#D-function IGCM_sys_RmRunDir
492#D-* Purpose: rm tmpdir (dummy function most of the time batch
493#D-                      scheduler will do the job)
494#D-* Examples:
495#D-
496function IGCM_sys_RmRunDir {
497  IGCM_debug_PushStack "IGCM_sys_RmRunDir" $@
498  if ( $DEBUG_sys ) ; then
499    echo "IGCM_sys_RmRunDir :" $@
500  fi
501
502  typeset status
503
504  echo rm $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
505  \rm $@ >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
506  status=$?
507
508  if [ ${status} -gt 0 ] ; then
509    IGCM_debug_Print 1 "IGCM_sys_RmRunDir : rm error code is ${status}."
510    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
511    IGCM_debug_Exit "IGCM_sys_RmRunDir"
512  else
513    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
514  fi
515  IGCM_debug_PopStack "IGCM_sys_RmRunDir"
516}
517
518#D-#==================================================
519#D-function IGCM_sys_Put_Dir
520#D-* Purpose: Copy a complete directory on $(ARCHIVE)
521#D-* Examples:
522#D-
523function IGCM_sys_Put_Dir {
524  IGCM_debug_PushStack "IGCM_sys_Put_Dir" $@
525  if ( $DEBUG_sys ) ; then
526    echo "IGCM_sys_Put_Dir :" $@
527  fi
528  if [ $DRYRUN = 0 ]; then
529    if [ ! -d ${1} ] ; then
530      echo "WARNING : IGCM_sys_Put_Dir ${1} DOES NOT EXIST ."
531      IGCM_debug_PopStack "IGCM_sys_Put_Dir"
532      return
533    fi
534
535    typeset status
536
537    # Only if we use rsync
538    #IGCM_sys_TestDirArchive $( dirname $2 )
539    #
540    #USUAL WAY
541    \cp -r $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
542    status=$?
543
544    if [ ${status} -gt 0 ] ; then
545      IGCM_debug_Print 2 "IGCM_sys_Put_Dir : cp failed error code ${status}"
546      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
547      IGCM_debug_Exit "IGCM_sys_Put_Dir"
548    else
549      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
550    fi
551  fi
552  IGCM_debug_PopStack "IGCM_sys_Put_Dir"
553}
554
555#D-#==================================================
556#D-function IGCM_sys_Get_Dir
557#D-* Purpose: Copy a complete directory from ${ARCHIVE}
558#D-* Examples:
559#D-
560function IGCM_sys_Get_Dir {
561  IGCM_debug_PushStack "IGCM_sys_Get_Dir" $@
562  if ( $DEBUG_sys ) ; then
563    echo "IGCM_sys_Get_Dir :" $@
564  fi
565  if [ $DRYRUN = 0 ]; then
566    typeset NB_ESSAI DELAI status i
567    # number of tentative
568    NB_ESSAI=3
569    # time delay between tentative
570    DELAI=2
571
572    #
573    # USUAL WAY
574    # add 'ccc_hsm get' (to demigrate all offline files) to reduce time of this command :
575    ccc_hsm get -r $1
576
577    i=0
578    while [ $i -lt $NB_ESSAI ] ; do
579      \cp -ur $1 $2 >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
580      status=$?
581      if [ ${status} -gt 0 ] ; then
582        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status} ${i}/${NB_ESSAI}"
583        IGCM_debug_Print 2 "IGCM_sys_Get_Dir : sleep ${DELAI} seconds and try again."
584        sleep $DELAI
585      else
586        break
587      fi
588      (( i = i + 1 ))
589    done
590
591    if [ ${status} -gt 0 ] ; then
592      IGCM_debug_Print 2 "IGCM_sys_Get_Dir : cp failed error code ${status}"
593      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
594      IGCM_debug_Exit "IGCM_sys_Get_Dir"
595    else
596      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
597    fi
598  fi
599  IGCM_debug_PopStack "IGCM_sys_Get_Dir"
600}
601
602#D-#==================================================
603#D-function IGCM_sys_Put_Rest
604#D-* Purpose: Put computied restarts on ${ARCHIVE}.
605#D-           File and target directory must exist.
606#D-* Examples:
607#D-
608function IGCM_sys_Put_Rest {
609  IGCM_debug_PushStack "IGCM_sys_Put_Rest" $@
610  if ( $DEBUG_sys ) ; then
611    echo "IGCM_sys_Put_Rest :" $@
612  fi
613  if [ $DRYRUN = 0 ]; then
614    if [ ! -f ${1} ] ; then
615      echo "ERROR : IGCM_sys_Put_Rest ${1} DOES NOT EXIST ."
616      IGCM_debug_Exit "IGCM_sys_Put_Rest"
617    fi
618
619    typeset status
620    #
621    # USUAL WAY
622    \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
623    status=$?
624
625#       #RSYNC WITH NETWORK SSH CALL
626#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
627#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
628
629#       #RSYNC WITH NFS USE
630#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
631#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
632
633#       status=$?
634#       IGCM_sys_Rsync_out $status
635
636#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
637#       (( status=status+$? ))
638
639    if [ ${status} -gt 0 ] ; then
640      IGCM_debug_Print 2 "IGCM_sys_Put_Rest : cp failed error code ${status}"
641      [ -f ${1} ] && ls -l ${1}
642      [ -f ${2} ] && ls -l ${2}
643      [ -f ${2}/${1} ] && ls -l ${2}/${1}
644      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
645      IGCM_debug_Exit "IGCM_sys_Put_Rest"
646    else
647
648      if [ X${JobType} = XRUN ] ; then
649        [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
650        [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
651      fi
652
653      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
654    fi
655  fi
656  IGCM_debug_PopStack "IGCM_sys_Put_Rest"
657}
658
659#D-#==================================================
660#D-function IGCM_sys_Put_Out
661#D-* Purpose: Copy a file on ${ARCHIVE} after having chmod it in readonly
662#D-* Examples:
663#D-
664function IGCM_sys_Put_Out {
665  IGCM_debug_PushStack "IGCM_sys_Put_Out" $@
666  if ( $DEBUG_sys ) ; then
667    echo "IGCM_sys_Put_Out :" $@
668  fi
669
670  typeset NB_ESSAI DELAI status i exist skip
671  typeset fileDeviceNumberInHex directoryDeviceNumberInHex
672
673  # number of tentative
674  NB_ESSAI=3
675  # time delay between tentative
676  DELAI=2
677
678  if [ $DRYRUN = 0 ]; then
679    if [ ! -f ${1} ] ; then
680      echo "WARNING : IGCM_sys_Put_Out ${1} DOES NOT EXIST ."
681      IGCM_debug_PopStack "IGCM_sys_Put_Out"
682      return 1
683    fi
684    #
685    IGCM_sys_MkdirArchive $( dirname $2 )
686    #
687    exist=false
688    skip=false
689    if [ -f $2 ] ; then
690      IGCM_debug_Print 1 "$2 already exist"
691      ccc_hsm get $2
692      exist=true
693      if [ "X$( diff $1 $2 )" = X ] ; then
694        IGCM_debug_Print 2 "$1 and $2 are the same file, we skip the copy"
695        skip=true
696      else
697        IGCM_debug_Print 2 "$1 and $2 are not the same file, we force the copy"
698        skip=false
699      fi
700    fi
701    #
702    if ( [ X${exist} = Xtrue ] && [ X${skip} = Xfalse ] ) ; then
703      IGCM_sys_Chmod u+w $2
704    fi
705
706    if [ X${skip} = Xfalse ] ; then
707      i=0
708      while [ $i -lt $NB_ESSAI ] ; do
709        # Identify file system
710        fileDeviceNumberInHex=$( stat -c %d $1 )
711        status=$?
712        if [ ${status} -gt 0 ] ; then
713          IGCM_debug_Exit "IGCM_sys_Put_Out"
714        fi
715        # Identify file system
716        directoryDeviceNumberInHex=$( stat -c %d $( dirname $2 ) )
717        status=$?
718        if [ ${status} -gt 0 ] ; then
719          IGCM_debug_Exit "IGCM_sys_Put_Out"
720        fi
721
722        if ( [ ${fileDeviceNumberInHex} -ne ${directoryDeviceNumberInHex} ] || [ X$3 = XNOMOVE ] ) ; then
723          # They are not on the same device. USUAL WAY
724          \cp $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
725          status=$?
726        else
727          # They are on the same device. NOT SO USUAL WAY
728          \mv $1 $2 > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
729          status=$?
730        fi
731        if [ ${status} -gt 0 ]; then
732          IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status} ${i}/${NB_ESSAI}"
733          IGCM_debug_Print 2 "IGCM_sys_Put_Out : sleep ${DELAI} seconds and try again."
734          [ -f ${1} ] && ls -l ${1}
735          [ -f ${2} ] && ls -l ${2}
736          [ -f ${2}/${1} ] && ls -l ${2}/${1}
737          sleep $DELAI
738        else
739          break
740        fi
741        (( i = i + 1 ))
742      done
743    fi
744
745#       #RSYNC WITH NETWORK SSH CALL
746#       echo ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
747#       ${RSYNC} ${RSYNC_opt} -e ssh ${RUN_DIR}/$1 ${STOREHOST}:${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
748
749#       #RSYNC WITH NFS USE
750#       echo ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
751#       ${RSYNC} ${RSYNC_opt} ${RUN_DIR}/$1 ${2} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
752
753#       status=$?
754#       IGCM_sys_Rsync_out $status
755
756#       ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
757#       (( status=status+$? ))
758
759    if [ ${status} -gt 0 ] ; then
760      IGCM_debug_Print 2 "IGCM_sys_Put_Out : cp failed error code ${status}"
761      [ -f ${1} ] && ls -l ${1}
762      [ -f ${2} ] && ls -l ${2}
763      [ -f ${2}/${1} ] && ls -l ${2}/${1}
764      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
765      IGCM_debug_Exit "IGCM_sys_Put_Out"
766    else
767
768      if [ X${JobType} = XRUN ] ; then
769        if [ X${3} = X ] ; then
770          [ -f ${2} ] && IGCM_sys_Chmod 444 ${2}
771          [ -f ${2}/${1} ] && IGCM_sys_Chmod 444 ${2}/${1}
772        fi
773      fi
774
775      \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
776    fi
777  fi
778  IGCM_debug_PopStack "IGCM_sys_Put_Out"
779  return 0
780}
781
782#D-#==================================================
783#D-function IGCM_sys_Get
784#D-* Purpose: Get a file from ${ARCHIVE}
785#D-* Examples: IGCM_sys_Get myfile /destpath/myfile_with_PREFIX
786#D-            IGCM_sys_Get /l Array_contain_myfiles /destpath/
787function IGCM_sys_Get {
788  IGCM_debug_PushStack "IGCM_sys_Get" $@
789
790  typeset DEST dm_liste target file_work
791  typeset NB_ESSAI DELAI status i
792
793  if ( $DEBUG_sys ) ; then
794    echo "IGCM_sys_Get :" $@
795  fi
796
797  # number of tentative
798  NB_ESSAI=3
799  # time delay between tentative
800  DELAI=2
801
802  if [ $DRYRUN -le 2 ]; then
803    if [ X${1} = X'/l' ] ; then
804      eval set +A dm_liste \${${2}}
805    else
806      eval set +A dm_liste ${1}
807    fi
808    eval DEST=\${${#}}
809    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
810    status=$?
811    if [ ${status} -gt 0 ] ; then
812      echo "WARNING IGCM_sys_Get : error code ${status}"
813      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
814      echo "WARNING IGCM_sys_Get : will stop later if the cp fails."
815    fi
816
817    #if [ ${status} -gt 0 ] ; then
818    #  if [ ! "X$( grep "Lost dmusrcmd connection" ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ )" = "X" ] ; then
819    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
820    #    echo "WARNING IGCM_sys_Get : Lost dmusrcmd connection : "
821    #    sleep 30
822    #    echo "We try another time"
823    ##    dmget ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
824    #    ccc_hsm get ${dm_liste[*]} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
825    #    status=$?
826    #    if [ ${status} -gt 0 ] ; then
827    #      echo "ERROR IGCM_sys_Get : again demigration error :"
828    #      cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
829    #      IGCM_debug_Exit "IGCM_sys_Get"
830    #    fi
831    #  else
832    #    echo "ERROR IGCM_sys_Get : demigration error :"
833    #    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
834    #    IGCM_debug_Exit "IGCM_sys_Get"
835    #  fi
836    #fi
837
838    #   #RSYNC WITH NETWORK SSH CALL
839    #   echo ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
840    #   ${RSYNC} ${RSYNC_opt} -e ssh ${STOREHOST}:"${dm_liste}" ${STOREHOST}:${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
841
842    #   #RSYNC WITH NFS USE
843    #   echo ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
844    #   ${RSYNC} ${RSYNC_opt} ${dm_liste} ${RUN_DIR}/${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
845
846    #   status=$?
847    #   IGCM_sys_Rsync_out $status
848
849    #   ${libIGCM}/libIGCM_sys/IGCM_analyse_rsync_out.awk ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
850    #   (( status=status+$? ))
851
852    #USUAL WAY
853    if [ X${1} = X'/l' ] ; then
854      for target in ${dm_liste[*]} ; do
855        local_file=$( basename ${target} )
856        # test if the target file is present before the loop
857        IGCM_sys_TestFileArchive ${target}
858        status=$?
859        if [ ${status} -gt 0 ] ; then
860          echo "IGCM_sys_Get, ERROR : regular file ${target} DOES NOT EXIST ."
861          IGCM_debug_Exit "IGCM_sys_Get"
862        else
863          i=0
864          while [ $i -lt $NB_ESSAI ] ; do
865            #if [ X${DoLink} = Xtrue ] ; then
866            #  \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
867            #  status=$?
868            #  else
869            #  \cp ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
870            #  status=$?
871            #fi
872            \ln -s ${target} ${DEST}/${local_file} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
873            status=$?
874            if [ ${status} -gt 0 ]; then
875              IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
876              IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
877              sleep $DELAI
878            else
879              break
880            fi
881            (( i = i + 1 ))
882          done
883          if [ ${status} -gt 0 ] ; then
884            echo "IGCM_sys_Get : error"
885            cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
886            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
887            IGCM_debug_Exit "IGCM_sys_Get"
888          else
889            \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
890          fi
891        fi
892      done
893    else
894      i=0
895      while [ $i -lt $NB_ESSAI ] ; do
896        \cp ${dm_liste} ${DEST} >> ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
897        status=$?
898        if [ ${status} -gt 0 ]; then
899          IGCM_debug_Print 2 "IGCM_sys_Get : cp failed error code ${status} ${i}/${NB_ESSAI}"
900          IGCM_debug_Print 2 "IGCM_sys_Get : sleep ${DELAI} seconds and try again."
901          sleep $DELAI
902        else
903          break
904        fi
905        (( i = i + 1 ))
906      done
907      if [ ${status} -gt 0 ] ; then
908        echo "IGCM_sys_Get : error"
909        cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
910        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
911        IGCM_debug_Exit "IGCM_sys_Get"
912      else
913        \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
914      fi
915    fi
916  fi
917  IGCM_debug_PopStack "IGCM_sys_Get"
918}
919
920#D-#==================================================
921#D-function IGCM_sys_GetDate_Monitoring
922#D-* Purpose: get the last year for which the monitoring has been computed
923#D-* Examples:
924#D-
925function IGCM_sys_GetDate_Monitoring {
926  IGCM_debug_PushStack "IGCM_sys_GetDate_Monitoring" $@
927  if ( $DEBUG_sys ) ; then
928    echo "IGCM_sys_GetDate_Monitoring :" $@
929  fi
930
931  eval ${2}=$( cdo showyear ${1} 2> /dev/null | gawk '{ print $NF }' )
932
933  IGCM_debug_PopStack "IGCM_sys_GetDate_Monitoring"
934}
935
936#D-#==================================================
937#D-function IGCM_sys_Dods_Rm
938#D-* Purpose: DO NOTHING ! Put ${ARCHIVE} files on DODS internet protocole.
939#D-* Examples:
940#D-
941function IGCM_sys_Dods_Rm {
942  if ( $DEBUG_sys ) ; then
943    echo "IGCM_sys_Dods_Rm :" $@
944  fi
945  typeset status
946  if [ $DRYRUN = 0 ]; then
947
948#    if [ ! -d /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} ] ; then
949#      echo "WARNING : IGCM_sys_Dods_Rm /dmnfs/cont003/dods/public/${LOGIN}/${R_DODS}/${1} DOES NOT EXIST ."
950#      echo "Nothing has been done."
951#      return
952#    fi
953
954    if [ "$#" -eq 1 ]; then
955      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS}/${1} # > out_dods_rm 2>&1
956      status=$?
957    else
958      $(ccc_home -u igcmg)/Tools/irene/thredds_rm public/${LOGIN}/${R_DODS} # > out_dods_rm 2>&1
959      status=$?
960    fi
961
962#    if [ ${status} -gt 0 ] ; then
963#      echo "IGCM_sys_Dods_Rm : error."
964#      cat out_dods_rm
965#      IGCM_debug_Exit "IGCM_sys_Dods_Rm"
966#    else
967#      rm out_dods_rm
968#    fi
969
970  fi
971  return $status
972}
973
974#D-#==================================================
975#D-function IGCM_sys_Dods_Cp
976#D-* Purpose: Copy $(ARCHIVE) files on DODS internet protocole.
977#D-* Examples:
978#D-
979function IGCM_sys_Dods_Cp {
980  if ( $DEBUG_sys ) ; then
981    echo "IGCM_sys_Dods_Cp :" $@
982  fi
983  typeset status
984  if [ $DRYRUN = 0 ]; then
985
986#    if [ ! -d ${R_SAVE}/${1} ] ; then
987#      echo "WARNING : IGCM_sys_Dods_Cp ${R_SAVE}/${1} DOES NOT EXIST ."
988#      echo "Nothing has been done."
989#      return
990#    fi
991
992    $(ccc_home -u igcmg)/Tools/irene/thredds_cp ${1} public/${LOGIN}/${R_DODS} # > out_dods_cp 2>&1
993    status=$?
994
995#       if [ ${status} -gt 0 ] ; then
996#           echo "IGCM_sys_Dods_Cp : error."
997#           cat out_dods_cp
998#           IGCM_debug_Exit "IGCM_sys_Dods_Cp"
999#       else
1000#           rm out_dods_cp
1001#       fi
1002
1003  fi
1004  return $status
1005}
1006
1007#D-#==================================================
1008#D-function IGCM_sys_Put_Dods
1009#D-* Purpose: Put ${ARCHIVE} files on DODS internet protocole.
1010#D-* Examples:
1011#D-
1012function IGCM_sys_Put_Dods {
1013  IGCM_debug_PushStack "IGCM_sys_Put_Dods" $@
1014  if ( $DEBUG_sys ) ; then
1015    echo "IGCM_sys_Put_Dods :" $@
1016  fi
1017  typeset status
1018  if [ $DRYRUN = 0 ]; then
1019    if ( [ ! -d ${R_SAVE}/${1} ] && [ ! -d ${R_FIGR}/${1} ] ) ; then
1020      echo "WARNING IGCM_sys_Put_Dods : None of the following directories exist. Exactly one should."
1021      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} DOES NOT EXIST."
1022      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} DOES NOT EXIST."
1023      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1024      return
1025    fi
1026
1027    if ( [ -d ${R_SAVE}/${1} ] && [ -d ${R_FIGR}/${1} ] ) ; then
1028      echo "WARNING IGCM_sys_Put_Dods : Both of the following directories exist. Exactly one should."
1029      echo "WARNING IGCM_sys_Put_Dods : ${R_SAVE}/${1} EXISTS."
1030      echo "WARNING IGCM_sys_Put_Dods : ${R_FIGR}/${1} EXISTS."
1031      IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1032      return
1033    fi
1034    #
1035    if [ -d ${R_SAVE}/${1} ] ; then
1036      cd ${R_SAVE}
1037    elif [ -d ${R_FIGR}/${1} ] ; then
1038      cd ${R_FIGR}
1039    fi
1040
1041    IGCM_sys_Dods_Rm ${1}
1042    IGCM_sys_Dods_Cp ${1}
1043    status=0
1044
1045    if [ ${status} -gt 0 ] ; then
1046      echo "IGCM_sys_Put_Dods : error."
1047      IGCM_debug_Exit "IGCM_sys_Put_Dods"
1048    fi
1049  fi
1050  IGCM_debug_PopStack "IGCM_sys_Put_Dods"
1051}
1052
1053##############################################################
1054# REBUILD OPERATOR
1055
1056#D-#==================================================
1057#D-function IGCM_sys_sync
1058#D-* Purpose: flush buffer on disk
1059#D-* Examples:
1060#D-
1061function IGCM_sys_sync {
1062  IGCM_debug_PushStack "IGCM_sys_sync" $@
1063  if ( $DEBUG_sys ) ; then
1064    echo "IGCM_sys_sync :" $@
1065  fi
1066
1067  /bin/sync
1068
1069  IGCM_debug_PopStack "IGCM_sys_sync"
1070}
1071
1072############################################################
1073# Activate Running Environnment Variables
1074
1075#D-#==================================================
1076#D-function IGCM_sys_activ_variables
1077#D-* Purpose: set environement variables prior to execution
1078#D-* Examples:
1079#D-
1080function IGCM_sys_activ_variables {
1081  IGCM_debug_PushStack "IGCM_sys_activ_variables"
1082  if ( $DEBUG_sys ) ; then
1083    echo "IGCM_sys_activ_variables"
1084  fi
1085
1086# --------------------------------------------------------------------
1087#D- MODULE specifications
1088# --------------------------------------------------------------------
1089  # Source the file EnvFile containing all module load needed to run the model.
1090  # EnvFile can be specified in config.card. If this is the case and if the file exists,
1091  # it will be used. Otherwise a default file will be used.
1092  if [ ! X${config_UserChoices_EnvFile} = X ] && [ -f ${config_UserChoices_EnvFile} ] ; then
1093      # EnvFile is set in config.card and the file exists
1094      IGCM_debug_Print 1 "EnvFile set in config.card will be used"
1095      EnvFile=${config_UserChoices_EnvFile}
1096  else
1097      IGCM_debug_Print 1 "IGCM_sys_active_variables : Default modules will be used"
1098      module purge > /dev/null 2>&1
1099      if [ X${compilerVersion} = XV19 ] ; then
1100          IGCM_debug_Print 1 "IGCM_sys_active_variables : Using compiler version V19"
1101          EnvFile=$( ccc_home -u igcmg)/MachineEnvironment/irene-amd/env_irene-amd > /dev/null 2>&1
1102      else
1103          IGCM_debug_Exit "This compiler version is not yet supported : ${compilerVersion}"
1104      fi
1105  fi
1106  IGCM_debug_Print 1 "IGCM_sys_active_variables : Following EnvFile will be used :" 
1107  IGCM_debug_Print 1 ${EnvFile}
1108  . ${EnvFile}
1109
1110  IGCM_debug_Print 1 "IGCM_sys_active_variables : Now loaded modules for Irene-amd. "
1111  module list
1112
1113# --------------------------------------------------------------------
1114#D- MPI specifications
1115# --------------------------------------------------------------------
1116
1117# --------------------------------------------------------------------
1118#D- Other specifications
1119# --------------------------------------------------------------------
1120
1121  ulimit -s unlimited
1122
1123  IGCM_debug_PopStack "IGCM_sys_activ_variables"
1124}
1125
1126############################################################
1127# Desactivate Running Environnment Variables
1128
1129#D-#==================================================
1130#D-function IGCM_sys_desactiv_variables
1131#D-* Purpose: unset environement variables after execution
1132#D-* Examples:
1133#D-
1134function IGCM_sys_desactiv_variables {
1135  IGCM_debug_PushStack "IGCM_sys_desactiv_variables"
1136  if ( $DEBUG_sys ) ; then
1137    echo "IGCM_sys_desactiv_variables"
1138  fi
1139# --------------------------------------------------------------------
1140#D- MPI specifications
1141# --------------------------------------------------------------------
1142
1143# --------------------------------------------------------------------
1144#D- Other specifications
1145# --------------------------------------------------------------------
1146
1147  IGCM_debug_PopStack "IGCM_sys_desactiv_variables"
1148}
1149
1150############################################################
1151# Update job headers to be used by the scheduler
1152
1153#D-#==================================================
1154#D-function IGCM_sys_updateHeaders
1155#D-* Purpose: Update job headers to be used by the scheduler
1156#D-* Examples: IGCM_sys_updateHeaders /path/to/Job_MYEXP
1157#D-
1158function IGCM_sys_updateHeaders {
1159  IGCM_debug_PushStack "IGCM_sys_updateHeaders"
1160  if ( $DEBUG_sys ) ; then
1161    echo "IGCM_sys_updateHeaders"
1162  fi
1163  typeset file
1164  file=$1
1165
1166   # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1167   # Compute the number of core to add to force the use of a new node for xios servers
1168
1169   if [ "X${IOS_OK_DEDICATED}" == "X" ] ; then
1170       (( IOS_OK_DEDICATED = 0 ))
1171   fi
1172
1173
1174   if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1175      (( current_core_noxios = coreNumber - IOS_PROC_MPI * IOS_PROC_OMP * IOS_PROC_DEP ))
1176      (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1177      (( second_comp_proc_mpi_loc = IOS_PROC_MPI - 1 ))
1178      (( coreNumber_final = current_core_noxios - 1 + first_comp_proc_dep_loc * 2 + second_comp_proc_mpi_loc * IOS_PROC_OMP * IOS_PROC_DEP ))
1179   else
1180      (( coreNumber_final = coreNumber ))
1181   fi
1182
1183  if [ ${executionType} -eq 1 ] ; then
1184    # MPMD + MPI
1185    sed -e "/::openMPthreads::/d"                  \
1186        -e "s/::JobNumProcTot::/${coreNumber_final}/"    \
1187      ${file} > ${file}.tmp
1188
1189  elif [ ${executionType} -eq 2 ] ; then
1190    # MPMD + MPI + OMP :ccc_mprun
1191    sed -e "/::openMPthreads::/d"                  \
1192        -e "s/::JobNumProcTot::/${coreNumber_final}/"    \
1193      ${file} > ${file}.tmp
1194
1195  elif [ ${executionType} -eq 3 ] ; then
1196    # SPMD + MPI/OMP
1197    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1198        -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1199        -e "/#MSUB -x/d"                           \
1200      ${file} > ${file}.tmp
1201
1202  elif [ ${executionType} -eq 4 ] ; then
1203    # SPMD + MPI only
1204    sed -e "s/::JobNumProcTot::/${mpiTasks}/"      \
1205        -e "/::openMPthreads::/d"                  \
1206        -e "/#MSUB -x/d"                           \
1207      ${file} > ${file}.tmp
1208
1209  elif [ ${executionType} -eq 5 ] ; then
1210    # SPMD + OMP only
1211    sed -e "s/::openMPthreads::/${openMPthreads}/" \
1212        -e "/::JobNumProcTot::/d"                  \
1213        -e "/#MSUB -x/d"                           \
1214      ${file} > ${file}.tmp
1215
1216  elif [ ${executionType} -eq 6 ] ; then
1217    # SEQUENTIAL THEN
1218    sed -e "s/::JobNumProcTot::/1/"                \
1219        -e "/::openMPthreads::/d"                  \
1220        -e "/#MSUB -x/d"                           \
1221      ${file} > ${file}.tmp
1222
1223  fi
1224
1225  IGCM_sys_Mv ${file}.tmp ${file}
1226
1227  IGCM_debug_PopStack "IGCM_sys_updateHeaders"
1228}
1229
1230############################################################
1231# Build MPI/OMP scripts run file (dummy function)
1232
1233#D-#==================================================
1234#D-function IGCM_sys_build_run_file
1235#D-* Purpose: build run file (deprecated)
1236#D-* Examples:
1237#D-
1238function IGCM_sys_build_run_file {
1239  IGCM_debug_Print 3 " dummy function : IGCM_sys_build_run_file "
1240}
1241
1242############################################################
1243# Build MPI/OMP scripts
1244
1245#D-#==================================================
1246#D-function IGCM_sys_build_execution_scripts
1247#D-* Purpose: build execution scripts to be launch by ${HOST_MPIRUN_COMMAND}
1248#D-* Examples:
1249#D-
1250function IGCM_sys_build_execution_scripts
1251{
1252  IGCM_debug_PushStack "IGCM_sys_build_execution_scripts" $@
1253  if ( $DEBUG_sys ) ; then
1254    echo "IGCM_sys_build_execution_scripts " $@
1255  fi
1256
1257  EXECUTION=${HOST_MPIRUN_COMMAND}
1258
1259  if ( ${OK_PARA_MPMD} ) ; then
1260
1261    # MPMD mode
1262    # 1 MPI only
1263    # 2 MPI/OpenMP mpirun method
1264    # 3 MPI/OpenMP ccc_mprun method
1265
1266    if [ -f run_file ] ; then
1267      IGCM_sys_Rm -f run_file
1268    fi
1269    touch run_file
1270
1271    # case 1 : Only MPI (MPMD)
1272    if  ( ! ${OK_PARA_OMP} ) ; then
1273
1274      # Build run_file
1275
1276      # First loop on the components for the coupler ie oasis (only if oasis3)
1277      # the coupler ie oasis3 must be the first one
1278      for comp in ${config_ListOfComponents[*]} ; do
1279
1280        eval ExeNameIn=\${config_Executable_${comp}[0]}
1281        eval ExeNameOut=\${config_Executable_${comp}[1]}
1282
1283        # for CPL component only
1284        if [ "X${comp}" = "XCPL" ] && [ "X${ExeNameOut}" != X\"\" ] ; then
1285
1286          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1287          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1288          echo "${comp_proc_mpi_loc} ./${ExeNameOut}" >> run_file
1289        fi
1290      done
1291
1292      # Then second loop on the components
1293      for comp in ${config_ListOfComponents[*]} ; do
1294
1295        eval ExeNameIn=\${config_Executable_${comp}[0]}
1296        eval ExeNameOut=\${config_Executable_${comp}[1]}
1297
1298        # Only if we really have an executable for the component and not the coupler ie oasis:
1299        if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${comp}" != "XCPL" ] ) ; then
1300
1301          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1302          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1303          eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1304
1305          # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1306          # Compute the number of core to add to force the use of a new node for xios servers
1307         if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XIOS" ] ) ; then
1308            if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1309              (( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_dep_loc ))
1310              (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1311              (( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1312              if [ ${comp_proc_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1313                 echo "1-${first_comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1314                 if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1315                    echo "${second_comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1316                 fi
1317              else
1318                 echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1319              fi
1320           else
1321              echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1322           fi
1323         else
1324          echo "${comp_proc_mpi_loc}-${comp_proc_dep_loc} ./${ExeNameOut}" >>run_file
1325         fi
1326
1327
1328        fi
1329      done
1330
1331      ## module advised by TGCC (instead of 2 variables)
1332      module load feature/bridge/heterogenous_mpmd
1333
1334      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1335
1336      IGCM_sys_Chmod u+x run_file
1337      if ( $DEBUG_sys ) ; then
1338        echo "run_file contains : "
1339        cat run_file
1340      fi
1341
1342    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xmpirun" ] ) ; then
1343
1344    # 2 MPI/OpenMP mpirun method
1345
1346      # Use of mpirun instead of ccc_mprun
1347      EXECUTION="time mpirun"
1348
1349      #  Hosts treatment
1350      ${EXECUTION} hostname | sort | uniq > hosts.tmp
1351
1352      i=0
1353      rm -f hosts rankfile
1354      IGCM_debug_Print 1 "sys Irene-amd, Hosts available :"
1355      for nodes in `cat hosts.tmp`
1356      do
1357        host[$i]=$nodes
1358        echo "${host[$i]}" >> hosts
1359        IGCM_debug_Print 1 ${host[$i]}
1360        i=$((i+1))
1361      done
1362      rm -f hosts.tmp
1363
1364      listnodes=${host[*]}
1365
1366      EXECUTION="${EXECUTION} -hostfile hosts -rankfile rankfile"
1367
1368      # Initialisation
1369      rank=0
1370      current_core=0
1371      init_exec=n
1372
1373      # Loop on the components
1374      for comp in ${config_ListOfComponents[*]} ; do
1375
1376        eval ExeNameIn=\${config_Executable_${comp}[0]}
1377        eval ExeNameOut=\${config_Executable_${comp}[1]}
1378
1379        # Not possible if oasis has an executable (i.e old version of oasis3)
1380        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1381          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1382          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1383          IGCM_debug_Verif_Exit
1384        fi
1385
1386        # Only if we really have an executable for the component :
1387        if [ "X${ExeNameOut}" != X\"\" ] ; then
1388
1389          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1390          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1391          eval comp_proc_nod_loc=\${${comp}_PROC_NOD}
1392
1393          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1394          echo ""  >> script_${ExeNameOut}.ksh
1395          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1396
1397            # Check if the number of threads is correct
1398            case ${comp_proc_omp_loc} in
1399            2|4|8|16)
1400              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1401              ;;
1402            *)
1403              IGCM_debug_Exit "ERROR with OMP parameters !"
1404              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1405              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1406              IGCM_debug_Verif_Exit
1407              ;;
1408            esac
1409            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1410            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1411            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1412            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1413          fi
1414
1415          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1416          echo "(( MYMPIRANK = OMPI_COMM_WORLD_RANK )) " >> script_${ExeNameOut}.ksh
1417          echo "MYMPIRANK=\$(printf '%3.3d\n' \${MYMPIRANK})" >> script_${ExeNameOut}.ksh
1418          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${MYMPIRANK} 2>out_${ExeNameOut}.err.\${MYMPIRANK}" >> script_${ExeNameOut}.ksh
1419          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1420
1421          if [ ${init_exec} = y ] ; then
1422            EXECUTION="${EXECUTION} : -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1423          else
1424            EXECUTION="${EXECUTION} -np ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1425            init_exec=y
1426          fi
1427
1428          # Build rankfile : method used to assign cores and nodes for the MPI process
1429          # Ex :
1430          #rank 0=curie5296 slot=0,1,2,3
1431          #rank 1=curie5296 slot=4,5,6,7
1432          # Example of final command :
1433          # mpirun -hostfile hosts -rankfile rankfile -np 27 ./script_lmdz.x.ksh : -np 5 ./script_opa.xx.ksh
1434          # with script_lmdz.x.ksh :
1435          # #!/bin/ksh
1436          #export KMP_STACKSIZE=3g
1437          #export KMP_LIBRARY=turnaround
1438          #export MKL_SERIAL=YES
1439          #OMP_NUM_THREADS=4
1440          #./lmdz.x
1441          if [ ${comp_proc_nod_loc} -gt 1 ] ; then
1442              (( offset_comp_proc_loc =  NB_CORE_PER_NODE / (comp_proc_mpi_loc / comp_proc_nod_loc) ))
1443          else
1444              (( offset_comp_proc_loc =  comp_proc_omp_loc ))
1445          fi
1446          for nb_proc_mpi in `seq 0 $(($comp_proc_mpi_loc-1))`; do
1447             (( index_host = current_core / NB_CORE_PER_NODE ))
1448             host_value=${host[${index_host}]}
1449             (( slot =  current_core % NB_CORE_PER_NODE ))
1450             virg=","
1451             string_final=""
1452             for index in `seq $slot $(($slot+$comp_proc_omp_loc-1))`; do
1453                 string=$index$virg
1454                 string_final=$string_final$string
1455             done
1456             string_final=$( echo $string_final | sed "s/.$//" )
1457             echo "rank $rank=$host_value slot=$string_final" >> rankfile
1458             (( rank = rank + 1 ))
1459             (( current_core = current_core + offset_comp_proc_loc ))
1460          done
1461        fi
1462      done
1463
1464    elif ( [ "X${config_UserChoices_ExecutionModeOnCurie}" = "X" ] || [ "X${config_UserChoices_ExecutionModeOnCurie}" = "Xccc_mprun" ] ) ; then
1465
1466    # 3 MPI/OpenMP ccc_mprun method
1467
1468    # MPI-OpenMP (MPMD)
1469    # example of run_file
1470    # 71-8 env OMP_NUM_THREADS=8 ./script_lmdz.x.ksh
1471    # 480-1 env OMP_NUM_THREADS=1 ./script_opa.xx.ksh
1472    # 1-1 env OMP_NUM_THREADS=1 ./script_xios.x.ksh
1473
1474      # Loop on the components
1475      for comp in ${config_ListOfComponents[*]} ; do
1476
1477        eval ExeNameIn=\${config_Executable_${comp}[0]}
1478        eval ExeNameOut=\${config_Executable_${comp}[1]}
1479
1480        # Not possible if oasis has an executable (i.e old version of oasis3)
1481        if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XCPL" ] ) ; then
1482          IGCM_debug_Exit "ERROR MPMD with hybrid MPI-OpenMP is not available with oasis3 version"
1483          IGCM_debug_Print 2 "Only available with oasis3-MCT version coupler"
1484          IGCM_debug_Verif_Exit
1485        fi
1486
1487        # Only if we really have an executable for the component :
1488        if [ "X${ExeNameOut}" != X\"\" ] ; then
1489
1490          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1491          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1492          eval comp_proc_dep_loc=\${${comp}_PROC_DEP}
1493
1494          # Compute number of cores per process
1495          (( comp_proc_omp_dep_loc = comp_proc_omp_loc * comp_proc_dep_loc )) 
1496
1497          # Build script files
1498
1499          echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1500          echo ""  >> script_${ExeNameOut}.ksh
1501          if [ ${comp_proc_omp_loc} -gt 1 ] ; then
1502
1503            # Check if the number of threads is correct
1504            case ${comp_proc_omp_loc} in
1505            2|4|8|16)
1506              IGCM_debug_Print 1 "You run ${ExeNameOut} on ${comp_proc_omp_loc} OMP threads"
1507              ;;
1508            *)
1509              IGCM_debug_Exit "ERROR with OMP parameters !"
1510              IGCM_debug_Print 2 "${comp_proc_omp_loc} is not possible as number of OMP threads"
1511              IGCM_debug_Print 2 "Only 2,4,8,16 as number of OMP threads are possible "
1512              IGCM_debug_Verif_Exit
1513              ;;
1514            esac
1515            ### only for LMDZ?
1516            echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1517            echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1518            echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1519            echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1520
1521          fi
1522
1523          # to have out/err per process on different files
1524          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}" >> script_${ExeNameOut}.ksh
1525          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1526
1527          IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1528
1529          # Complete run_file
1530
1531          # In case of use of DEDICATED option XIOS servers, do not share xios nodes with xios clients
1532          # Compute the number of core to add to force the use of a new node for xios servers
1533         if ( [ "X${ExeNameOut}" != X\"\" ] &&  [ "X${comp}" = "XIOS" ] ) ; then
1534            if [ ${IOS_OK_DEDICATED} -eq 1 ] ; then
1535              (( current_core_noxios = coreNumber - comp_proc_mpi_loc * comp_proc_omp_dep_loc )) 
1536              (( first_comp_proc_dep_loc = 1 + NB_CORE_PER_NODE - current_core_noxios % NB_CORE_PER_NODE ))
1537              (( second_comp_proc_mpi_loc = comp_proc_mpi_loc - 1 ))
1538              if [ ${comp_proc_omp_dep_loc} -lt ${first_comp_proc_dep_loc} ] ; then
1539                 echo "1-${first_comp_proc_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1540                  if [ ${comp_proc_mpi_loc} -gt 1 ] ; then
1541                    echo "${second_comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1542                  fi
1543              else
1544                 echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1545              fi
1546           else
1547              echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=1 ./script_${ExeNameOut}.ksh " >>run_file
1548           fi
1549         else
1550          echo "${comp_proc_mpi_loc}-${comp_proc_omp_dep_loc} env OMP_NUM_THREADS=${comp_proc_omp_loc} ./script_${ExeNameOut}.ksh " >>run_file
1551         fi
1552
1553        fi
1554      done
1555
1556      ## variable added to stop after 60s instead of 600s by default.
1557      ## This is used when no error comes from executables and when something stopped an executable without notice.
1558      export SLURM_WAIT=60
1559
1560      ## module advised by TGCC (instead of 2 variables)
1561      module load feature/bridge/heterogenous_mpmd
1562
1563      # set EXECUTION for ccc_mprun case (similar to #1 : MPI only MPMD method)
1564
1565      EXECUTION="${HOST_MPIRUN_COMMAND} -f ./run_file"
1566
1567      IGCM_sys_Chmod u+x run_file
1568      if ( $DEBUG_sys ) ; then
1569        echo "run_file contains : "
1570        cat run_file
1571      fi
1572
1573    else
1574
1575      IGCM_debug_Print 1 "sys Curie :  choose mpirun or ccc_mprun in config.card for ExecutionModeOnCurie"
1576
1577    fi # 1 2 or 3 if ${OK_PARA_MPMD}
1578
1579  else
1580  # Only one executable (SPMD mode).
1581
1582    for comp in ${config_ListOfComponents[*]} ; do
1583
1584      # Only if we really have an executable for the component :
1585      eval ExeNameOut=\${config_Executable_${comp}[1]}
1586      if ( [ "X${ExeNameOut}" != X\"\" ] && [ "X${ExeNameOut}" != "Xinca.dat" ] ) ; then
1587
1588        # Build script files
1589
1590        echo "#!/bin/ksh" > script_${ExeNameOut}.ksh
1591        echo ""  >> script_${ExeNameOut}.ksh
1592        IGCM_sys_Chmod u+x script_${ExeNameOut}.ksh
1593
1594        if ( ${OK_PARA_OMP} ) ; then
1595          eval comp_proc_omp_loc=\${${comp}_PROC_OMP}
1596          echo ""  >> script_${ExeNameOut}.ksh
1597          echo "export KMP_STACKSIZE=3g"  >> script_${ExeNameOut}.ksh
1598          echo "export KMP_LIBRARY=turnaround"  >> script_${ExeNameOut}.ksh
1599          echo "export MKL_SERIAL=YES"  >> script_${ExeNameOut}.ksh
1600          echo "OMP_NUM_THREADS=${comp_proc_omp_loc}" >> script_${ExeNameOut}.ksh
1601        fi
1602
1603        if  ( ${OK_PARA_MPI} ) ; then
1604          eval comp_proc_mpi_loc=\${${comp}_PROC_MPI}
1605          # Default : ccc_mprun used if nb_proc gt 1
1606          # to have out/err per process on different files
1607          echo "./${ExeNameOut} > out_${ExeNameOut}.out.\${SLURM_PROCID} 2>out_${ExeNameOut}.err.\${SLURM_PROCID}"  >> script_${ExeNameOut}.ksh
1608          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1609          EXECUTION="${HOST_MPIRUN_COMMAND} -n ${comp_proc_mpi_loc} ./script_${ExeNameOut}.ksh"
1610        else
1611          # Default : ccc_mprun is NOT used if nb_proc eq 1
1612          # to have out/err per process on different files
1613          echo "./${ExeNameOut} > out_${ExeNameOut}.out 2>out_${ExeNameOut}.err" >> script_${ExeNameOut}.ksh
1614          #echo "./${ExeNameOut}" >> script_${ExeNameOut}.ksh
1615          EXECUTION="/usr/bin/time ./script_${ExeNameOut}.ksh"
1616        fi
1617
1618        IGCM_debug_Print 1 "sys Irene-amd : script_${ExeNameOut}.ksh contains"
1619        cat script_${ExeNameOut}.ksh
1620
1621      fi
1622
1623    done
1624
1625  fi # ${OK_PARA_MPMD}
1626
1627  IGCM_debug_Print 1 "sys Irene-amd : execution command is "
1628  IGCM_debug_Print 1 "$EXECUTION"
1629
1630  IGCM_debug_PopStack "IGCM_sys_build_execution_scripts"
1631}
1632
1633#D-#==================================================
1634#D-function IGCM_sys_check_path
1635#D-* Purpose: check that RUN_DIR_PATH that will be removed on some machine
1636#D-* do not point to an important use directory. Stop immediately in that case.
1637#D-* Examples:
1638#D-
1639function IGCM_sys_check_path {
1640  IGCM_debug_PushStack "IGCM_sys_check_path"
1641  if ( $DEBUG_sys ) ; then
1642    echo "IGCM_sys_check_path"
1643  fi
1644
1645  if ( [ X${RUN_DIR_PATH} = X${HOME} ] || [ X${RUN_DIR_PATH} = X${CCCWORKDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSCRATCHDIR} ] || [ X${RUN_DIR_PATH} = X${CCCSTOREDIR} ] ) ; then
1646    IGCM_debug_Print 1 "Variable RUN_DIR_PATH is pointing to an important directory : ${RUN_DIR_PATH}"
1647    IGCM_debug_Print 1 "Please check the RUN_DIR_PATH definition in your Job : Job_${config_UserChoices_JobName}"
1648    IGCM_debug_Exit "This will stop the job"
1649  fi
1650  IGCM_debug_PopStack "IGCM_sys_check_path"
1651}
1652
1653#D-#==================================================
1654#D-function IGCM_sys_check_quota
1655#D-* Purpose: check user quota. Stop the simulation if quota above 90%
1656#D-* Examples:
1657#D-
1658function IGCM_sys_check_quota {
1659  IGCM_debug_PushStack "IGCM_sys_check_quota"
1660  if ( $DEBUG_sys ) ; then
1661    echo "IGCM_sys_check_quota"
1662  fi
1663  # Limit of quota (in %)
1664  limit_quota=90
1665
1666  # Check of the volume
1667  volume_quota=$(ccc_quota | grep ' scratch' | gawk '{print $2}')
1668  volume_avail=$(ccc_quota | grep ' scratch' | gawk '{print $3}')
1669
1670  if ( [ ! X${volume_quota} = X ] && [ ! ${volume_quota} = "-" ] ) ; then
1671
1672    unit_avail=${volume_avail: -1}
1673    unit_quota=${volume_quota: -1}
1674
1675    if [ "${unit_quota}" = "*" ] ; then
1676      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1677      IGCM_debug_Print 1 "More than 100% of your quota is used"
1678      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1679      IGCM_debug_Print 1 "You must have more than 10% available to run"
1680      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1681      IGCM_debug_Verif_Exit
1682    fi
1683
1684    temp_avail=${volume_avail%%${volume_avail: -1}*}
1685    temp_quota=${volume_quota%%${volume_quota: -1}*}
1686
1687    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1688
1689    # Convertion
1690      if [ ${volume_avail: -1} = "T" ] ; then
1691        (( temp_avail = temp_avail * 1000000000000 ))
1692      elif [ ${volume_avail: -1} = "G" ] ; then
1693        (( temp_avail = temp_avail * 1000000000 ))
1694      elif [ ${volume_avail: -1} = "M" ] ; then
1695        (( temp_avail = temp_avail * 1000000 ))
1696      elif [ ${volume_avail: -1} = "k" ] ; then
1697        (( temp_avail = temp_avail * 1000 ))
1698      else
1699        (( temp_avail = volume_avail ))
1700      fi
1701      if [ ${volume_quota: -1} = "T" ] ; then
1702        (( temp_quota = temp_quota * 1000000000000 ))
1703      elif [ ${volume_quota: -1} = "G" ] ; then
1704        (( temp_quota = temp_quota * 1000000000 ))
1705      elif [ ${volume_quota: -1} = "M" ] ; then
1706        (( temp_quota = temp_quota * 1000000 ))
1707      elif [ ${volume_quota: -1} = "k" ] ; then
1708        (( temp_quota = temp_quota * 1000 ))
1709      else
1710        (( temp_quota = volume_quota ))
1711      fi
1712    fi
1713
1714    quota_volume=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1715#    echo "volume ratio is " $quota_volume
1716
1717    if [ ${quota_volume} -ge ${limit_quota} ] ; then
1718      IGCM_debug_Print 1 "Please, check your quota of volume on scratch"
1719      IGCM_debug_Print 1 "${quota_volume}% of your quota is used"
1720      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1721      IGCM_debug_Print 1 "You must have more than 10% available to run"
1722      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1723      IGCM_debug_Verif_Exit
1724    fi
1725
1726  fi
1727
1728# Check of the number of inodes
1729
1730  inode_quota=$(ccc_quota | grep ' scratch' | gawk '{print $6}')
1731  inode_avail=$(ccc_quota | grep ' scratch' | gawk '{print $7}')
1732
1733  if ( [ ! X${inode_quota} = X ] && [ ! ${inode_quota} = "-" ] ) ; then
1734
1735    unit_avail=${inode_avail: -1}
1736    unit_quota=${inode_quota: -1}
1737
1738    if [ "${unit_quota}" = "*" ] ; then
1739      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1740      IGCM_debug_Print 1 "More than 100% of your quota is used"
1741      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1742      IGCM_debug_Print 1 "You must have more than 10% available to run"
1743      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1744      IGCM_debug_Verif_Exit
1745    fi
1746
1747    temp_avail=${inode_avail%%${inode_avail: -1}*}
1748    temp_quota=${inode_quota%%${inode_quota: -1}*}
1749
1750    if [ ! ${unit_avail} = ${unit_quota} ] ; then
1751
1752    # Convertion
1753      if [ ${inode_avail: -1} = "T" ] ; then
1754        (( temp_avail = temp_avail * 1000000000000 ))
1755      elif [ ${inode_avail: -1} = "G" ] ; then
1756        (( temp_avail = temp_avail * 1000000000 ))
1757      elif [ ${inode_avail: -1} = "M" ] ; then
1758        (( temp_avail = temp_avail * 1000000 ))
1759      elif [ ${inode_avail: -1} = "k" ] ; then
1760        (( temp_avail = temp_avail * 1000 ))
1761      else
1762        (( temp_avail = inode_avail ))
1763      fi
1764
1765      if [ ${inode_quota: -1} = "T" ] ; then
1766        (( temp_quota = temp_quota * 1000000000000 ))
1767      elif [ ${inode_quota: -1} = "G" ] ; then
1768        (( temp_quota = temp_quota * 1000000000 ))
1769      elif [ ${inode_quota: -1} = "M" ] ; then
1770        (( temp_quota = temp_quota * 1000000 ))
1771      elif [ ${inode_quota: -1} = "k" ] ; then
1772        (( temp_quota = temp_quota * 1000 ))
1773      else
1774        (( temp_quota = inode_quota ))
1775      fi
1776    fi
1777    quota_inode=$(echo "scale=2 ; $temp_quota/$temp_avail*100" | bc)
1778#    echo "inode ratio is " $quota_inode
1779
1780    if [ ${quota_inode} -ge ${limit_quota} ] ; then
1781      IGCM_debug_Print 1 "Please, check your quota of inode on scratch"
1782      IGCM_debug_Print 1 "${quota_inode}% of your quota is used"
1783      IGCM_debug_Print 1 "Use the ccc_quota command to check"
1784      IGCM_debug_Print 1 "You must have more than 10% available to run"
1785      IGCM_debug_Exit "Not enough space to run ! STOP HERE"
1786      IGCM_debug_Verif_Exit
1787    fi
1788  fi
1789  IGCM_debug_PopStack "IGCM_sys_check_quota"
1790}
1791
1792#D-#==================================================
1793#D-function IGCM_sys_projectAccounting
1794#D-* Purpose: store project accounting information in a file
1795#D-* Examples:
1796#D-
1797function IGCM_sys_projectAccounting {
1798  IGCM_debug_PushStack "IGCM_sys_projectAccounting"
1799  if ( $DEBUG_sys ) ; then
1800    echo "IGCM_sys_check_quota"
1801  fi
1802
1803  ssh irene194 /usr/bin/ccc_myproject > $1
1804
1805  IGCM_debug_PopStack "IGCM_sys_projectAccounting"
1806}
1807
1808#D-#==================================================
1809#D-function IGCM_sys_getJobSchedulerID
1810#D-* Purpose: Get the job ID during execution
1811#D-* Examples: IGCM_sys_getJobSchedulerID jobSchedulerID
1812#D-
1813function IGCM_sys_getJobSchedulerID {
1814  IGCM_debug_PushStack "IGCM_sys_getJobSchedulerID"
1815  if ( $DEBUG_sys ) ; then
1816    echo "IGCM_sys_getJobSchedulerID"
1817  fi
1818  eval ${1}=${BRIDGE_MSUB_JOBID}
1819
1820  IGCM_debug_PopStack "IGCM_sys_getJobSchedulerID"
1821}
1822
1823#D-#==================================================
1824#D-function IGCM_sys_GetJobID
1825#D-* Purpose: Get the job ID from the JobName
1826#D-* Examples: IGCM_sys_GetJobID ${JobName} ${TargetUsr} JobID
1827#D-
1828function IGCM_sys_GetJobID {
1829  IGCM_debug_PushStack "IGCM_sys_GetJobID"
1830  if ( $DEBUG_sys ) ; then
1831    echo "IGCM_sys_GetJobID"
1832  fi
1833
1834  # With -f option, the full job name is given in the last column
1835  ID=$( ccc_mstat -f -u $2 | \
1836        gawk -v JobName=$1 '( $NF ~ JobName ) { print $1 }' )
1837
1838  eval ${3}=${ID}
1839  IGCM_debug_PopStack "IGCM_sys_GetJobID"
1840}
1841
1842#D-#==================================================
1843#D-function IGCM_sys_CountJobInQueue
1844#D-* Purpose: Count number of users job
1845#D-* Examples: IGCM_sys_CountJobInQueue ${JobName} NbRun
1846#D-
1847function IGCM_sys_CountJobInQueue {
1848  IGCM_debug_PushStack "IGCM_sys_CountJobInQueue"
1849  if ( $DEBUG_sys ) ; then
1850    echo "IGCM_sys_CountJobInQueue"
1851  fi
1852
1853  # With -f option, the full job name is given in the last column
1854  NbRun=$( ccc_mstat -f | gawk -v JobName=$1 'BEGIN { x=0 } ( $NF ~ JobName ) { x=x+1 } END { print x }' )
1855
1856  eval ${2}=${NbRun}
1857
1858  IGCM_debug_PopStack "IGCM_sys_CountJobInQueue"
1859}
1860
1861#D-#==================================================
1862#D-function IGCM_sys_ListJobInQueue
1863#D-* Purpose: Produce a list of users computing jobs (excluding post-processing)
1864#D-* Examples: IGCM_sys_ListJobInQueue ${User} JobNameList
1865#D-
1866function IGCM_sys_ListJobInQueue {
1867  IGCM_debug_PushStack "IGCM_sys_ListJobInQueue"
1868  if ( $DEBUG_sys ) ; then
1869    echo "IGCM_sys_ListJobInQueue"
1870  fi
1871
1872  # With -f option, the full job name is given in the last column
1873  set -A JobList $( ccc_mstat -f | gawk -v User=$1             \
1874                                        '( $2  == User      && \
1875                                           $NF != /TS/      && \
1876                                           $NF !~ /PACK/    && \
1877                                           $NF !~ /REBUILD/ && \
1878                                           $NF !~ /pack/ )     \
1879                                         { print $NF }' | sed -e "s/\(.*\)\.[0-9]*/\1/" )
1880
1881  eval set -A ${2} ${JobList[*]}
1882
1883  IGCM_debug_PopStack "IGCM_sys_ListJobInQueue"
1884}
1885
1886#D-#==================================================
1887#D-function IGCM_sys_atlas
1888#D-* Purpose: encapsulate atlas call so as to manage error code and curie specificity
1889#D-* Examples:
1890#D-
1891function IGCM_sys_atlas {
1892  IGCM_debug_PushStack "IGCM_sys_atlas" $@
1893  if ( $DEBUG_sys ) ; then
1894    echo "IGCM_sys_atlas :" $@
1895  fi
1896
1897  typeset status
1898
1899  \ccc_mprun atlas $@ > ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$ 2>&1
1900  status=$?
1901  if [ ${status} -gt 0 ] ; then
1902    echo "IGCM_sys_atlas : error code ${status}"
1903    cat ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1904    \rm ${OUTCOMMAND_PATH}/out_command_${LOGIN}.$$
1905    IGCM_debug_PopStack "IGCM_sys_atlas"
1906    return 1
1907  else
1908    IGCM_debug_PopStack "IGCM_sys_atlas"
1909    return 0
1910  fi
1911
1912  IGCM_debug_PopStack "IGCM_sys_atlas"
1913}
1914
1915#D-#==================================================
1916#D-function IGCM_sys_rebuild_nemo
1917#D-* Purpose: rebuild nemo parallel files with nemo specific rebuild on curie
1918#D-* Examples:
1919#D-
1920
1921function IGCM_sys_rebuild_nemo {
1922  IGCM_debug_PushStack "IGCM_sys_rebuild_nemo" $@
1923  if ( $DEBUG_sys ) ; then
1924    echo "IGCM_sys_rebuild_nemo :" $@
1925  fi
1926
1927  $( ccc_home -u igcmg)/Tools/irene/bin/rebuild_nemo ${1} ${2}
1928
1929  IGCM_debug_PopStack "IGCM_sys_rebuild_nemo"
1930}
1931
Note: See TracBrowser for help on using the repository browser.