source: trunk/libIGCM/ins_job @ 1516

Last change on this file since 1516 was 1490, checked in by jgipsl, 5 years ago

Added first version of sys_jeanzay. Still lots to do. Only case done for now is one executable mpi mode. Only create_ts done among post-treatments and only from main job (not done via TimeSeriesChecker?). Change of data project is not working (follwoing is not working when launched from job: eval $(idrenv -d ${PROJECT}) ).
  • Property svn:executable set to *
  • Property svn:keywords set to Revision Author Date
File size: 17.1 KB
Line 
1#!/bin/ksh
2
3#**************************************************************
4# Author: Jacques Belier
5# Contact:
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13#---------------------------------------------------------------------
14#- Installation of jobs according to environment
15#---------------------------------------------------------------------
16function ins_job_Usage
17{
18print - "
19ins_job installs the jobs in the directories
20which contain a file config.card
21
22ins_job must be launched on the host
23on which the job will be submitted
24
25Usage :
26  ${b_n} [-h] [-v] [-e]
27  or on irene/TGCC :
28  ${b_n} [-h] [-v] [-e] [-p project] [-q type_of_node] [-c number of cores]
29  or on ada /IDRIS
30  ${b_n} [-h] [-v] [-e] [-m MPI environment]
31Options :
32  -h                  : help
33  -v                  : verbose mode
34  -e                  : turn on ensemble mode (hindcast/forecast or date restart)
35  -f                  : ins_job is force to create jobs even if they already exist
36on irene only :
37  -p project          : add default project on irene
38  -q type_of_node     : add default type of nodes for postprocessing on irene skylake/xlarge
39  -c number of cores  : add default number of cores for postprocessing on irene 1-112
40on ada only :
41  - m MPI environment : add default MPI environment (Intel MPI or IBM MPI)
42"
43}
44function ins_job_Warning
45{
46   [[ ${x_v} = 'verbose' ]] && print - "\n############### WARNING ###############";
47   [[ ${x_v} = 'verbose' ]] && print - "File ${n_f} already exists\nin directory ${j}";
48   [[ ${x_v} = 'verbose' ]] && print - "You must delete this file to update !";
49}
50
51function ins_job_Check_JobName
52{
53  verif=${JobName##[a-zA-Z]*(?([.\-])[a-zA-Z0-9])}
54
55  if [ ${#verif} -ne 0 ] ; then
56    echo "################ ERROR ################"
57    echo "${JobName} is invalid."
58    echo "- JobName can only contain alphanumeric characters, \".\" and \"-\""
59    echo "- JobName must start with a letter"
60
61    ((NbErr=NbErr+1))
62
63    Status=1
64  else
65    Status=0
66  fi
67
68  return ${Status}
69}
70
71#-
72#     dirname     and      basename
73#-
74d_n=$(dirname ${0}); b_n=$(basename ${0});
75#-
76# Retrieving and validation of the options
77#-
78x_v='silencious';
79x_e=false;
80x_f=false;
81x_p=false;
82x_q=false;
83x_c=false;
84x_m=false;
85while getopts :hvefc:p:m:q:s: V ; do
86  case $V in
87  (h)  ins_job_Usage; exit 0;;
88  (v)  x_v='verbose';;
89  (e)  x_e=true;;
90  (f)  x_f=true;;
91  (p)  x_p=true
92       ProjectID=${OPTARG} ;;
93  (q)  x_q=true
94       ProjectNode=${OPTARG} ;;
95  (c)  x_c=true
96       ProjectCore=${OPTARG} ;;
97  (m)  x_m=true
98       MPIEnvironment=${OPTARG} ;;
99  (:)  echo ${b_n}" : -"${OPTARG}" option : missing value" 1>&2;
100       exit 2;;
101  (\?) echo ${b_n}" : -"${OPTARG}" option : not supported" 1>&2;
102       exit 2;;
103  esac
104done
105[ ${x_v} = 'silencious' ] && export DEBUG_sys=false
106shift $(($OPTIND-1));
107#-
108# Define working files
109#-
110F_MOD=$(cd ${d_n}'/..';/bin/pwd;)
111# [[ ${F_MOD##*/} != 'modipsl' ]] && \
112#  { print - "directory 'modipsl' unreachable"; exit 3; }
113W_W=${d_n}'/../libIGCM'
114[[ ! -d ${W_W} ]] && { print - "${W_W} unreachable"; exit 3; }
115libIGCM=$(cd ${W_W};/bin/pwd;)
116F_JOB=${libIGCM}'/AA_job';
117[[ ! -f ${F_JOB} ]] && { print - "${F_JOB} unreachable"; exit 3; }
118F_RCI=${libIGCM}'/run.card.init';
119[[ ! -f ${F_RCI} ]] && { print - "${F_RCI} unreachable"; exit 3; }
120#-
121# Accessing to functions (without stack)
122#-
123# No verbosity (0, 1, 2, 3)
124Verbosity=0
125# No de debug
126DEBUG_debug=false
127# Dont move libIGCM
128MirrorlibIGCM=false
129# Behave like computing job
130TaskType=computing
131# Source libIGCM
132. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
133. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
134. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
135. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
136. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
137if [ $x_e = 'true' ] ; then
138  . ${libIGCM}/libIGCM_ensemble/libIGCM_ensemble.ksh
139fi
140
141#-
142[[ ${x_v} = 'verbose' ]] && \
143 {
144  print - "";
145  print - '--- Host        : '${SYSTEM};
146  print - '--- modipsl     : '${F_MOD};
147  print - '--- libIGCM     : '${libIGCM};
148  print - '--- basic job   : '${F_JOB};
149  print - '--- basic card  : '${F_RCI};
150 }
151#-
152[[ ${x_v} = 'verbose' ]] && print - "\nInstallation of jobs for '${SYSTEM}'";
153#-
154
155NbErr=0
156
157#-
158# Define Project parameters to set up jobs header for Irene (TGCC)
159# on Irene define ProjectID and ProjectCore : option or answer
160# on Irene ProjectNode default projectnode set to xlarge since 17/10/2018
161#-
162
163
164if [ X"${SYSTEM}" == "Xirene" ] ; then
165
166  #- set ProjectID if required
167  if ( ! ${x_p} ) ; then
168    print - "Wait for the next question ..."
169    #- default ProjectID
170    ProjectID=$( ccc_myproject | grep -i irene |grep -i skylake | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | sort -u | grep -v gencmip6 | head -n 1 )
171    answer=""
172    print - "Hit Enter or give project ID (default is ${ProjectID}), possible projects are $( echo $( ccc_myproject | grep -i irene | grep -i skylake | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | grep -v gencmip6 | sort -u ) ) or other xxxcmip6 : $(for i in $(groups) ; do echo $i|grep -v gencmip6|grep .cmip6 1>/dev/null 2>&1 && echo -n $i " " ; done ; echo ) "
173    read answer
174
175    if [ "X${answer}" != "X" ] ; then
176      ProjectID=${answer}
177    fi
178
179  fi # if ( ! ${x_p} )
180
181  echo  ProjectID is ${ProjectID} at Irene
182
183
184  # set ProjectNode if required
185  if ( ! ${x_q} ) ; then 
186     #- default ProjectNode
187     ProjectNode="xlarge"
188     #- is xlarge possible for ${ProjectID} ?
189     echo $(ccc_myproject | grep -i irene | grep -i xlarge | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | grep -v gencmip6 | sort -u) $(for i in $(groups) ; do echo $i|grep -v gencmip6|grep .cmip6 1>/dev/null 2>&1 && echo -n $i " " ; done ) | grep ${ProjectID} >/dev/null 2>&1  || ProjectNode="skylake"
190
191     if [ "X${ProjectNode}" == "Xxlarge" ] ; then
192       answerOK=false
193 
194       while ( ! ${answerOK} ) ; do
195         answer="" 
196         print - "Hit Enter or give TYPE OF NODE required for post-processing (default is \"${ProjectNode}\"), possible types of nodes are \"skylake\" or \"xlarge\" : " 
197         read answer
198         [ "X${answer}" == "X" ] || [ "X${answer}" == "Xskylake" ] || [ "X${answer}" == "Xxlarge" ] && answerOK=true
199       done
200 
201       if [ "X${answer}" != "X" ] ; then
202         ProjectNode=${answer} 
203       fi
204
205     else
206       print - "PostProcessing will be done on ${ProjectNode}"
207     fi
208 
209  fi # if ( ! ${x_q} )
210  echo ProjectNode for post-processing is ${ProjectNode} at Irene
211
212  #- ProjectNode is known (option or answer) set ProjectCoreMax
213  [ "${ProjectNode}" = "xlarge" ] && ProjectCoreMax="112" || ProjectCoreMax="48" 
214  #- ProjectCoreMax is 48 for standard and 112 for xlarge
215
216  if ( ! ${x_c} ) ; then
217    #- ProjectNode is known (option or answer), set ProjectCore default
218    [ "${ProjectNode}" = "xlarge" ] && ProjectCore="8" || ProjectCore="4" 
219
220    # let check minimum/maximum value 1/${ProjectCoreMax}
221
222    answerOK=false
223
224    while ( ! ${answerOK} ) ; do
225      answer=""
226      print - "possible numbers of cores are \"1\" to \"${ProjectCoreMax}\" for ${ProjectNode} : "
227      print - "Hit Enter or give NUMBER OF CORES required for post-processing (default is \"${ProjectCore}\")"
228      read answer
229      [ "X${answer}" == "X" ] || [ ${answer} -ge 1 -a ${answer} -le ${ProjectCoreMax} ] && answerOK=true
230    done
231
232    if [ "X${answer}" != "X" ] ; then
233      ProjectCore=${answer}
234    fi
235
236  fi # if ( ! ${x_c} )
237
238  echo ProjectCore for post-processing is ${ProjectCore}
239  #- ProjectCore is set (option or answer)
240
241elif [ X"${SYSTEM}" == "Xjeanzay" ] ; then
242
243  #- set ProjectID if required
244  if ( ! ${x_p} ) ; then
245    print - "Wait for the next question ..."
246    #- default ProjectID
247    ProjectID=$( echo $IDRPROJ )
248    answer=""
249    print - "Hit Enter or give project ID (default is ${ProjectID}), possible projects are: $( groups ; echo ) "
250    read answer
251
252    if [ "X${answer}" != "X" ] ; then
253      ProjectID=${answer}
254    fi
255
256  fi # if ( ! ${x_p} )
257
258  ProjectID=${ProjectID}
259  echo  ProjectID is ${ProjectID} at Jean-Zay
260
261elif [ X"${SYSTEM}" == "Xada" ] ; then
262  if ( ! ${x_m} ) ; then
263    MPIEnvironment=IBM
264    # Intel MPI Environment.
265    answerOK=false     
266    while ( ! ${answerOK} ) ; do
267      answer=""
268      print - "Hit Enter or give MPI Environement (default is ${MPIEnvironment}), possible MPI environments are IBM (MPI IBM) and Intel (MPI Intel) :"
269      read answer
270      [ "X${answer}" == "X" ] || [ "X${answer}" == "XIBM" ] || [ "X${answer}" == "XIntel" ] && answerOK=true
271    done
272
273    if [ "X${answer}" != "X" ] ; then
274      MPIEnvironment=${answer}
275    fi
276  fi # if ( ! ${x_pm} )
277  echo MPIEnvironment is ${MPIEnvironment}
278
279elif [ X"${SYSTEM}" == "Xlxiv8" ] || [ X"${SYSTEM}" == "Xifort_CICLAD" ] ; then
280  # obelix, ciclad, climserv
281  echo ""
282  echo "You need to check and maybe adapt headers in the main job especially the line: "
283  echo "  #PBS -l nodes=x:ppn=y "
284  echo "where x is the number of nodes, y the number of cores per node and x*y is the total number of cores for the job. "
285  echo "y must not be bigger than the maximum numer of cores per node on the machine (often 8 or 16)."
286fi # if [ X"${SYSTEM}" == "Xirene" ]
287
288#-
289# Define the pattern string to substitute
290#-
291W_P='#-Q- '; W_W=${W_P}${SYSTEM};
292#-
293# Extract list of 'config.card' files
294# and create jobs with AA_job
295#-
296F_CFG='config.card';
297F_CFG_ENS='ensemble.card';
298SUBMIT_DIR_ENS=$( pwd )
299for i in $( pwd )/config.card
300do
301  if [ ! -f $i ] ; then
302    echo ""
303    echo "################## WARNING ##################"
304    echo "No config.card available in current directory"
305    echo ""
306    continue
307  fi
308
309
310  j=$(cd ${i%/*};/bin/pwd;)
311  n_f=${F_RCI##*/};
312
313  if [ ! X$( echo ${j} | grep EXPERIMENTS ) = X ] ; then
314    # Do not treat config.card if it is in sub-directory of EXPERIMENTS
315    # Continue to next config.card
316    continue
317  else
318    [[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
319  fi
320
321  # Find out if new structure and set .resol filename
322  if [ -d ${j}/EXPERIMENTS ] && [ -d ${j}/GENERAL ] ; then
323    # New Structure
324    [[ ${x_v} = 'verbose' ]] && echo "This is new configuration structure"
325    new_struct=yes
326    resolfile=$j/.resol
327  else
328    # Old Structure
329    new_struct=no
330    resolfile=$j/../.resol
331  fi
332
333  # Get all variables declared in section UserChoices in config.card
334  IGCM_card_DefineArrayFromSection ${j}'/'${F_CFG} UserChoices
335  # Set default values
336  config_UserChoices_ExpType=""
337  RESOL_ATM_3D=this_is_a_test_string
338  RESOL=this_is_another_test_string
339  typeset option
340  for option in ${config_UserChoices[*]} ; do
341    IGCM_card_DefineVariableFromOption ${j}'/'${F_CFG} UserChoices ${option}
342  done
343
344  # Find the JobName : JobName might contain the variable RESOL_ATM_3D that will be replaced by what is in .resol file
345  if [ ! X$( echo ${config_UserChoices_JobName} | grep ${RESOL_ATM_3D} ) = X ] ; then
346    TRUERESOL=$( tail -1 $resolfile | awk "-F=" '{print $2}' )
347    echo TRUERESOL = $TRUERESOL
348    JobName=$( echo ${config_UserChoices_JobName} | sed -e "s/${RESOL_ATM_3D}/${TRUERESOL}/" )
349    IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
350  elif [ ! X$( echo ${config_UserChoices_JobName} | grep ${RESOL} ) = X ] ; then
351    TRUERESOL=$( head -1 $resolfile  )
352    JobName=$( echo ${config_UserChoices_JobName} | sed -e "s/${RESOL}/${TRUERESOL}/" )
353    IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
354  else
355    JobName=${config_UserChoices_JobName}
356  fi
357
358  # Check JobName validity : only alphanumerical characters, "-" and "." are authorized
359  ins_job_Check_JobName
360  RetCode=$?
361  [[ $RetCode -gt 0 ]] && continue
362
363  [[ ${x_v} = 'verbose' ]] && echo "JobName=${JobName}"
364
365  # Add specific treatment for new type of directory structure
366  if [ ${new_struct} == yes ] ; then
367
368    if [ "X${config_UserChoices_ExpType}" = X ] ; then
369      echo "\nERROR in ${j}/config.card"
370      echo "ins_job stops here"
371      echo "=> The variable ExpType must be added in config.card in section UserChoices"
372      echo "=> ExpType gives the directory for the .card configuration files for the wanted experiement. For exemple ExpType=IPSLCM5/historical"
373      exit 4
374    else
375      [[ ${x_v} = 'verbose' ]] && echo "ExpType= ${config_UserChoices_ExpType}"
376    fi
377
378    if [ -d ${j}/${JobName} ] ; then
379      echo "Directory ${j}/${JobName} exists already. It will not be overwritten."
380      echo "Remove the existing directory or change JobName before relaunching ins_job."
381      #continue
382      exit 
383    fi
384    echo "=> Submit directory ${JobName} will be created with cards from EXPERIMENTS/${config_UserChoices_ExpType}"
385    cp -r ${j}/EXPERIMENTS/${config_UserChoices_ExpType} ${j}/${JobName}
386    cp -r ${j}/GENERAL/* ${j}/${JobName}/.
387    cp -f ${j}/${F_CFG}  ${j}/${JobName}/.
388    if [ -f ${F_CFG_ENS} ] ; then
389      cp -f ${j}/${F_CFG_ENS}  ${j}/${JobName}/.
390      SUBMIT_DIR_ENS=${j}/${JobName}
391    fi
392    rm -f ${j}/${F_CFG}
393    # rm -f ${j}/${F_CFG_ENS}
394    rm -f ${j}/${F_CFG}.bak
395    j=${j}/${JobName}
396    [[ ${x_v} = 'verbose' ]] && echo new j=$j
397  fi
398  # end specific treatment for new type directory structure
399
400  [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; } || \
401   {
402    [[ ${x_v} = 'verbose' ]] && print - "\nCopying file ${F_RCI}\nin directory ${j}";
403    \cp ${F_RCI} ${j};
404   }
405
406  #==================================
407  # Read ListOfComponents section:
408  #echo
409  #IGCM_debug_Print 1 "DefineArrayFromSection : ListOfComponents"
410
411  IGCM_card_DefineArrayFromSection  ${j}'/'${F_CFG} ListOfComponents
412  for comp in ${config_ListOfComponents[*]} ; do
413    IGCM_card_DefineArrayFromOption  ${j}'/'${F_CFG} ListOfComponents ${comp}
414  done
415  #IGCM_debug_Print 3 ${config_ListOfComponents[*]}
416
417  #==================================
418  # Read Executable section:
419  IGCM_card_DefineArrayFromSection ${j}'/'${F_CFG} Executable
420
421  # Define the execution context (MPMD, SPMD, MPI/OMP ...)
422  IGCM_config_ConfigureExecution ${j}'/'${F_CFG}
423
424  # coreNumber    : TOTAL NUMBER OF CORES
425  # mpiTasks      : TOTAL NUMBER OF MPI TASKS
426  # openMPthreads : NUMBER OF OpenMP THREADS
427
428  # File name for Job
429  n_f='Job_'${JobName};
430  [[ ${x_f} = 'false' ]] && [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; continue; }
431  [[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
432  sed -e "/^${W_W} */ s///" \
433      -e "/^${W_P}/d"       \
434      -e "s%::modipsl::%${F_MOD}%" \
435      -e "s/::Jobname::/${JobName}/" \
436      -e "s/::default_project::/${ProjectID}/" \
437      ${F_JOB} > ${j}'/'${n_f}
438  chmod u+x ${j}'/'${n_f}
439
440  # update Headers so that ressources description are accurate (MPMD/SPMD/...)
441  IGCM_sys_updateHeaders ${j}'/'${n_f}
442done
443
444#-
445# Extract list of AA_* files in libIGCM
446# and create jobs (for all except AA_job)
447#-
448for i in $(find ${libIGCM} -maxdepth 1 -name "AA_*" -print)
449do
450  i_f=${i##*/};
451  [[ ${i_f} = 'AA_job' ]] && { continue; }
452  j=${i%/*}; n_f=${i_f#AA_}'.job';
453  [[ ${x_f} = 'false' ]] && [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; continue; }
454  [[ ${x_v} = 'verbose' ]] && print - "\nIn directory ${j}\n${i_f} -> ${n_f}"
455  sed -e "/^${W_W} */ s///" \
456      -e "s%::modipsl::%${F_MOD}%" \
457      -e "/^${W_P}/d"       \
458      -e "s/::default_node::/${ProjectNode}/" \
459      -e "s/::default_core::/${ProjectCore}/" \
460      -e "s/::default_project::/${ProjectID}/" \
461      -e "s/::default_post_project::/${ProjectID}/" \
462      ${i} > ${j}'/'${n_f}
463  chmod u+x ${j}'/'${n_f}
464done
465#-
466# set default_project in libIGCM_sys_irene and libIGCM_sys_jeanzay.
467#-
468if [ X"${SYSTEM}" == "Xirene" ] ; then
469  i=${libIGCM}/libIGCM_sys/libIGCM_sys_irene.ksh
470  sed -i -e "s/::default_project::/${ProjectID}/" ${i}
471elif [ X"${SYSTEM}" == "Xjeanzay" ] ; then
472  i=${libIGCM}/libIGCM_sys/libIGCM_sys_jeanzay.ksh
473  sed -i -e "s/::default_project::/${ProjectID}/" ${i}
474fi
475#-
476# Limited to hindcast/forecast and date restart Ensemble for the time being
477if [ ${x_e} = 'true' ] ; then
478    if [ ! -f ${F_CFG_ENS} ] ; then
479       echo ""
480       echo "################## WARNING ##################"
481       echo "No ensemble.card available in current directory"
482       echo ""
483       exit
484      #  continue
485   fi
486
487  #.. Read input data from ensemble.card ..
488  SUBMIT_DIR=${SUBMIT_DIR_ENS}
489  RUN_DIR="${CCCWORKDIR}/ENSEMBLE_TMP"
490  #
491  # Copy initial things around and define variables (hindcast/forecast case)
492  IGCM_sys_Cd ${SUBMIT_DIR}
493  IGCM_ensemble_Init
494
495  if [[ ${ensemble_Ens_DATE_active} = 'y' ]] ; then
496    IGCM_sys_Cd ${SUBMIT_DIR}
497    IGCM_ensemble_DateInit
498    # As it says
499    IGCM_sys_Cd ${SUBMIT_DIR}
500    IGCM_ensemble_DateNonPeriodicStarts
501  fi
502
503  if [[ ${ensemble_Ens_PERTURB_active} = 'y' ]] ; then
504    IGCM_sys_Cd ${SUBMIT_DIR}
505    IGCM_ensemble_CastInit
506    # As it says
507    IGCM_sys_Cd ${SUBMIT_DIR}
508    IGCM_ensemble_CastPeriodicStarts
509    # As it says
510    #IGCM_sys_Cd ${SUBMIT_DIR}
511    #IGCM_ensemble_CastMemberList
512  fi
513  # Done
514  IGCM_sys_Cd ${SUBMIT_DIR}
515  # Clean
516  IGCM_sys_Rm -rf ${RUN_DIR}
517 
518fi
519#-
520[[ ${x_v} = 'verbose' ]] && print - "";
521#-
522# That's all folks
523#-
524
525if [ ${NbErr} -ne 0 ] ; then
526  echo "################ ERROR ################"
527  echo "${NbErr} invalid JobName(s) found, check the log"
528fi
529
530
531exit 0;
Note: See TracBrowser for help on using the repository browser.