source: trunk/libIGCM/ins_job @ 1269

Last change on this file since 1269 was 1259, checked in by sdipsl, 9 years ago
  • be restrictive when considering config.card to work with
  • Property svn:executable set to *
  • Property svn:keywords set to Revision Author Date
File size: 11.9 KB
RevLine 
[1164]1#!/bin/ksh
2
3#**************************************************************
4# Author: Jacques Belier
5# Contact:
6# $Revision::                                          $ Revision of last commit
7# $Author::                                            $ Author of last commit
8# $Date::                                              $ Date of last commit
9# IPSL (2006)
10#  This software is governed by the CeCILL licence see libIGCM/libIGCM_CeCILL.LIC
11#
12#**************************************************************
13#---------------------------------------------------------------------
14#- Installation of jobs according to environment
15#---------------------------------------------------------------------
16function ins_job_Usage
17{
18print - "
19ins_job installs the jobs in the directories
20which contain a file config.card
21
22ins_job must be launched on the host
23on which the job will be submitted
24
25Usage :
26  ${b_n} [-h] [-v] [-e] [-p project]
27
28Options :
29  -h          : help
30  -v          : verbose mode
31  -e          : turn on ensemble mode (hindcast/forecast or date restart)
32  -p project  : add default project on curie
33"
34}
35function ins_job_Warning
36{
37   [[ ${x_v} = 'verbose' ]] && print - "\n############### WARNING ###############";
38   [[ ${x_v} = 'verbose' ]] && print - "File ${n_f} already exists\nin directory ${j}";
39   [[ ${x_v} = 'verbose' ]] && print - "You must delete this file to update !";
40}
41
42function ins_job_Check_JobName
43{
44  verif=${JobName##[a-zA-Z]*(?([.\-])[a-zA-Z0-9])}
45
46  if [ ${#verif} -ne 0 ] ; then
47    echo "################ ERROR ################"
48    echo "${JobName} is invalid."
49    echo "- JobName can only contain alphanumeric characters, \".\" and \"-\""
50    echo "- JobName must start with a letter"
51
52    ((NbErr=NbErr+1))
53
54    Status=1
55  else
56    Status=0
57  fi
58
59  return ${Status}
60}
61
62#-
63#     dirname     and      basename
64#-
65d_n=$(dirname ${0}); b_n=$(basename ${0});
66#-
67# Retrieving and validation of the options
68#-
69x_v='silencious';
70x_e=false;
71x_p=false;
72while getopts :hvep: V ; do
73  case $V in
74  (h)  ins_job_Usage; exit 0;;
75  (v)  x_v='verbose';;
76  (e)  x_e=true;;
77  (p)  x_p=true
78       ProjectID=${OPTARG} ;;
79  (:)  echo ${b_n}" : -"${OPTARG}" option : missing value" 1>&2;
80       exit 2;;
81  (\?) echo ${b_n}" : -"${OPTARG}" option : not supported" 1>&2;
82       exit 2;;
83  esac
84done
85[ ${x_v} = 'silencious' ] && export DEBUG_sys=false
86shift $(($OPTIND-1));
87#-
88# Define working files
89#-
90F_MOD=$(cd ${d_n}'/..';/bin/pwd;)
91# [[ ${F_MOD##*/} != 'modipsl' ]] && \
92#  { print - "directory 'modipsl' unreachable"; exit 3; }
93W_W=${d_n}'/../libIGCM'
94[[ ! -d ${W_W} ]] && { print - "${W_W} unreachable"; exit 3; }
95libIGCM=$(cd ${W_W};/bin/pwd;)
96F_JOB=${libIGCM}'/AA_job';
97[[ ! -f ${F_JOB} ]] && { print - "${F_JOB} unreachable"; exit 3; }
98F_RCI=${libIGCM}'/run.card.init';
99[[ ! -f ${F_RCI} ]] && { print - "${F_RCI} unreachable"; exit 3; }
100#-
101# Accessing to functions (without stack)
102#-
[1249]103# No verbosity (0, 1, 2, 3)
104Verbosity=0
105# No de debug
[1164]106DEBUG_debug=false
[1249]107# Dont move libIGCM
[1164]108MirrorlibIGCM=false
[1249]109# Behave like computing job
[1164]110TaskType=computing
[1249]111# Source libIGCM
[1164]112. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
113. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
114. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
115. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
[1242]116. ${libIGCM}/libIGCM_config/libIGCM_config.ksh
[1164]117if [ $x_e = 'true' ] ; then
118  . ${libIGCM}/libIGCM_ensemble/libIGCM_ensemble.ksh
119fi
120
[1172]121#-
122[[ ${x_v} = 'verbose' ]] && \
123 {
124  print - "";
125  print - '--- Host        : '${SYSTEM};
126  print - '--- modipsl     : '${F_MOD};
127  print - '--- libIGCM     : '${libIGCM};
128  print - '--- basic job   : '${F_JOB};
129  print - '--- basic card  : '${F_RCI};
130 }
131#-
132[[ ${x_v} = 'verbose' ]] && print - "\nInstallation of jobs for '${SYSTEM}'";
133#-
134
[1164]135NbErr=0
136
137#-
138# Define Project ID to set up jobs header for Curie (TGCC)
139#-
140if ( ! ${x_p} ) ; then
[1172]141  if [ X"${SYSTEM}" == "Xcurie" ] ; then
[1164]142    ProjectID=$( ccc_myproject | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | head -n 1 )
143    ProjectID=${ProjectID:="gen2211"}
144    answer=""
145    print - "Hit Enter or give project ID (default is ${ProjectID}), possible projects are $( echo $( ccc_myproject | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | sort -u ) ) :"
146    read answer
147
148    if [ "X${answer}" != "X" ] ; then
149      ProjectID=${answer}
150    fi
151    [ "${ProjectID}" = "gencmip6" ] && ProjectNode="standard" || ProjectNode="xlarge"
152  fi
153else
[1172]154  if [ X"${SYSTEM}" != "Xcurie" ] ; then
155    print - "\nOption -p invalid with this host (${SYSTEM}): it will be ignored" 1>&2;
[1164]156  fi
157fi
158
159#-
160# Define the pattern string to substitute
161#-
[1172]162W_P='#-Q- '; W_W=${W_P}${SYSTEM};
[1164]163#-
164# Extract list of 'config.card' files
165# and create jobs with AA_job
166#-
167F_CFG='config.card';
168F_CFG_ENS='ensemble.card';
169SUBMIT_DIR_ENS=$( pwd )
[1259]170for i in $(find ${d_n}/../config -name ${F_CFG} -print)
[1164]171do
172  j=$(cd ${i%/*};/bin/pwd;)
173  n_f=${F_RCI##*/};
174
175  if [ ! X$( echo ${j} | grep EXPERIMENTS ) = X ] ; then
176    # Do not treat config.card if it is in sub-directory of EXPERIMENTS
177    # Continue to next config.card
178    continue
179  else
180    [[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
181  fi
182
183  # Find out if new structure and set .resol filename
184  if [ -d ${j}/EXPERIMENTS ] && [ -d ${j}/GENERAL ] ; then
185    # New Structure
186    [[ ${x_v} = 'verbose' ]] && echo "This is new configuration structure"
187    new_struct=yes
188    resolfile=$j/.resol
189  else
190    # Old Structure
191    new_struct=no
192    resolfile=$j/../.resol
193  fi
194
195  # Get all variables declared in section UserChoices in config.card
196  IGCM_card_DefineArrayFromSection ${j}'/'${F_CFG} UserChoices
197  # Set default values
198  config_UserChoices_ExpType=""
199  RESOL_ATM_3D=this_is_a_test_string
200  RESOL=this_is_another_test_string
201  typeset option
202  for option in ${config_UserChoices[*]} ; do
203    IGCM_card_DefineVariableFromOption ${j}'/'${F_CFG} UserChoices ${option}
204  done
205
206  # Find the JobName : JobName might contain the variable RESOL_ATM_3D that will be replaced by what is in .resol file
207  if [ ! X$( echo ${config_UserChoices_JobName} | grep ${RESOL_ATM_3D} ) = X ] ; then
208    TRUERESOL=$( tail -1 $resolfile | awk "-F=" '{print $2}' )
209    echo TRUERESOL = $TRUERESOL
210    JobName=$( echo ${config_UserChoices_JobName} | sed -e "s/${RESOL_ATM_3D}/${TRUERESOL}/" )
211    IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
212  elif [ ! X$( echo ${config_UserChoices_JobName} | grep ${RESOL} ) = X ] ; then
213    TRUERESOL=$( head -1 $resolfile  )
214    JobName=$( echo ${config_UserChoices_JobName} | sed -e "s/${RESOL}/${TRUERESOL}/" )
215    IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
216  else
217    JobName=${config_UserChoices_JobName}
218  fi
219
220  # Check JobName validity : only alphanumerical characters, "-" and "." are authorized
221  ins_job_Check_JobName
222  RetCode=$?
223  [[ $RetCode -gt 0 ]] && continue
224
225  [[ ${x_v} = 'verbose' ]] && echo "JobName=${JobName}"
226
227  # Add specific treatment for new type of directory structure
228  if [ ${new_struct} == yes ] ; then
229
230    if [ "X${config_UserChoices_ExpType}" = X ] ; then
231      echo "\nERROR in ${j}/config.card"
232      echo "ins_job stops here"
233      echo "=> The variable ExpType must be added in config.card in section UserChoices"
[1238]234      echo "=> ExpType gives the directory for the .card configuration files for the wanted experiement. For exemple ExpType=IPSLCM5/historical"
[1164]235      exit 4
236    else
237      [[ ${x_v} = 'verbose' ]] && echo "ExpType= ${config_UserChoices_ExpType}"
238    fi
239
240    if [ -d ${j}/${JobName} ] ; then
241      echo "Directory ${j}/${JobName} exist already, continue next config.card"
242      continue
243    fi
244    echo "=> Submit directory ${JobName} will be created with cards from EXPERIMENTS/${config_UserChoices_ExpType}"
245    cp -r ${j}/EXPERIMENTS/${config_UserChoices_ExpType} ${j}/${JobName}
246    cp -r ${j}/GENERAL/* ${j}/${JobName}/.
247    cp -f ${j}/${F_CFG}  ${j}/${JobName}/.
248    if [ -f ${F_CFG_ENS} ] ; then
249      cp -f ${j}/${F_CFG_ENS}  ${j}/${JobName}/.
250      SUBMIT_DIR_ENS=${j}/${JobName}
251    fi
252    rm -f ${j}/${F_CFG}
253    rm -f ${j}/${F_CFG_ENS}
254    rm -f ${j}/${F_CFG}.bak
255    j=${j}/${JobName}
256    [[ ${x_v} = 'verbose' ]] && echo new j=$j
257  fi
258  # end specific treatment for new type directory structure
259
260  [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; } || \
261   {
262    [[ ${x_v} = 'verbose' ]] && print - "\nCopying file ${F_RCI}\nin directory ${j}";
263    \cp ${F_RCI} ${j};
264   }
[1242]265
[1238]266  #==================================
[1242]267  # Read ListOfComponents section:
268  #echo
269  #IGCM_debug_Print 1 "DefineArrayFromSection : ListOfComponents"
270
271  IGCM_card_DefineArrayFromSection  ${j}'/'${F_CFG} ListOfComponents
272  for comp in ${config_ListOfComponents[*]} ; do
273    IGCM_card_DefineArrayFromOption  ${j}'/'${F_CFG} ListOfComponents ${comp}
274  done
275  #IGCM_debug_Print 3 ${config_ListOfComponents[*]}
276
277  #==================================
[1238]278  # Read Executable section:
[1242]279  IGCM_card_DefineArrayFromSection ${j}'/'${F_CFG} Executable
[1164]280
[1238]281  # Define the execution context (MPMD, SPMD, MPI/OMP ...)
[1242]282  IGCM_config_ConfigureExecution ${j}'/'${F_CFG}
[1238]283
284  # coreNumber    : TOTAL NUMBER OF CORES
285  # mpiTasks      : TOTAL NUMBER OF MPI TASKS
286  # openMPthreads : NUMBER OF OpenMP THREADS
287
[1164]288  # File name for Job
289  n_f='Job_'${JobName};
290  [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; continue; }
291  [[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
292  sed -e "/^${W_W} */ s///" \
293      -e "/^${W_P}/d"       \
294      -e "s%::modipsl::%${F_MOD}%" \
295      -e "s/::Jobname::/${JobName}/" \
296      -e "s/::default_project::/${ProjectID}/" \
297      ${F_JOB} > ${j}'/'${n_f}
298  chmod u+x ${j}'/'${n_f}
[1238]299
300  # update Headers so that ressources description are accurate (MPMD/SPMD/...)
301  IGCM_sys_updateHeaders ${j}'/'${n_f}
[1164]302done
[1238]303
[1164]304#-
305# Extract list of AA_* files in libIGCM
306# and create jobs (for all except AA_job)
307#-
308for i in $(find ${libIGCM} -maxdepth 1 -name "AA_*" -print)
309do
310  i_f=${i##*/};
311  [[ ${i_f} = 'AA_job' ]] && { continue; }
312  j=${i%/*}; n_f=${i_f#AA_}'.job';
313  [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; continue; }
314  [[ ${x_v} = 'verbose' ]] && print - "\nIn directory ${j}\n${i_f} -> ${n_f}"
315  sed -e "/^${W_W} */ s///" \
316      -e "s%::modipsl::%${F_MOD}%" \
317      -e "/^${W_P}/d"       \
318      -e "s/::default_node::/${ProjectNode}/" \
319      -e "s/::default_project::/${ProjectID}/" \
320      ${i} > ${j}'/'${n_f}
321  chmod u+x ${j}'/'${n_f}
322done
323#-
324# set default_project in libIGCM_sys_curie.ksh too.
325#-
[1172]326if [ X"${SYSTEM}" == "Xcurie" ] ; then
[1164]327  i=${libIGCM}/libIGCM_sys/libIGCM_sys_curie.ksh
328  sed -i -e "s/::default_project::/${ProjectID}/" ${i}
329fi
330#-
331# Limited to hindcast/forecast and date restart Ensemble for the time being
332if [ ${x_e} = 'true' ] ; then
333  #.. Read input data from ensemble.card ..
334  SUBMIT_DIR=${SUBMIT_DIR_ENS}
335  RUN_DIR="${WORKDIR}/ENSEMBLE"
336  #
337  # Copy initial things around and define variables (hindcast/forecast case)
338  IGCM_sys_Cd ${SUBMIT_DIR}
339  IGCM_ensemble_Init
340
341  if [[ ${ensemble_Ens_PARAMETRIC_active} = 'y' ]] ; then
342    echo "WARNING: Parametric Ensemble is not implemented yet..."
343  fi
344
345  if [[ ${ensemble_Ens_DATE_active} = 'y' ]] ; then
346    IGCM_sys_Cd ${SUBMIT_DIR}
347    IGCM_ensemble_DateInit
348    # As it says
349    IGCM_sys_Cd ${SUBMIT_DIR}
350    IGCM_ensemble_DatePeriodicStarts
351    # As it says
352    IGCM_sys_Cd ${SUBMIT_DIR}
353    IGCM_ensemble_DateNonPeriodicStarts
354    # Clean
355    IGCM_sys_Rm -rf ${RUN_DIR}
356  fi
357
358  if [[ ${ensemble_Ens_PERTURB_active} = 'y' ]] ; then
359    IGCM_sys_Cd ${SUBMIT_DIR}
360    IGCM_ensemble_CastInit
361    # As it says
362    IGCM_sys_Cd ${SUBMIT_DIR}
363    IGCM_ensemble_CastPeriodicStarts
364    # As it says
365    IGCM_sys_Cd ${SUBMIT_DIR}
366    IGCM_ensemble_CastNonPeriodicStarts
367    # As it says
368    IGCM_sys_Cd ${SUBMIT_DIR}
369    IGCM_ensemble_CastMemberList
370    # Done
371    IGCM_sys_Cp ${RUN_DIR}/CreatedDir.txt ${SUBMIT_DIR}
372    IGCM_sys_Cd ${SUBMIT_DIR}
373    # Clean
374    IGCM_sys_Rm -rf ${RUN_DIR}
375  fi
376fi
377#-
378[[ ${x_v} = 'verbose' ]] && print - "";
379#-
380# That's all folks
381#-
382
383if [ ${NbErr} -ne 0 ] ; then
384  echo "################ ERROR ################"
385  echo "${NbErr} invalid JobName(s) found, check the log"
386fi
387
388
389exit 0;
Note: See TracBrowser for help on using the repository browser.