source: modipsl/trunk/util/ins_job @ 2304

Last change on this file since 2304 was 2304, checked in by sdipsl, 7 years ago

Add IGCM_ensemble_CastMemberList call when needed

  • Property svn:executable set to *
  • Property svn:keywords set to Id
File size: 10.9 KB
Line 
1#!/bin/ksh
2# $Id$
3#---------------------------------------------------------------------
4#- Installation of jobs according to an environment
5#---------------------------------------------------------------------
6function ins_job_Usage
7{
8print - "
9ins_job installs the jobs in the directories
10which contain a file config.card
11
12ins_job must be launched on the host
13on which the job will be submitted
14
15Usage :
16  ${b_n} [-h] [-v] [-e] [-p project]
17
18Options :
19  -h          : help
20  -v          : verbose mode
21  -e          : turn on ensemble mode (hindcast/forecast or date restart)
22  -p project  : add default project on curie
23"
24}
25function ins_job_Warning
26{
27   [[ ${x_v} = 'verbose' ]] && print - "\n############### WARNING ###############";
28   [[ ${x_v} = 'verbose' ]] && print - "File ${n_f} already exists\nin directory ${j}";
29   [[ ${x_v} = 'verbose' ]] && print - "You must delete this file to update !";
30}
31
32function ins_job_Check_JobName
33{
34  verif=${JobName##[a-zA-Z]*(?([.\-])[a-zA-Z0-9])}
35
36  if [ ${#verif} -ne 0 ] ; then
37    echo "################ ERROR ################"
38    echo "${JobName} is invalid."
39    echo "- JobName can only contain alphanumeric characters, \".\" and \"-\""
40    echo "- JobName must start with a letter"
41
42    ((NbErr=NbErr+1))
43
44    Status=1
45  else
46    Status=0
47  fi
48
49  return ${Status}
50}
51
52#-
53#     dirname     and      basename
54#-
55d_n=$(dirname ${0}); b_n=$(basename ${0});
56#-
57# Retrieving and validation of the options
58#-
59x_v='silencious';
60x_e=false;
61x_p=false;
62while getopts :hvep: V ; do
63  case $V in
64  (h)  ins_job_Usage; exit 0;;
65  (v)  x_v='verbose';;
66  (e)  x_e=true;;
67  (p)  x_p=true
68       ProjectID=${OPTARG} ;;
69  (:)  echo ${b_n}" : -"${OPTARG}" option : missing value" 1>&2;
70       exit 2;;
71  (\?) echo ${b_n}" : -"${OPTARG}" option : not supported" 1>&2;
72       exit 2;;
73  esac
74done
75[ ${x_v} = 'silencious' ] && export DEBUG_sys=false
76shift $(($OPTIND-1));
77#-
78# Define working files
79#-
80F_MOD=$(cd ${d_n}'/..';/bin/pwd;)
81# [[ ${F_MOD##*/} != 'modipsl' ]] && \
82#  { print - "directory 'modipsl' unreachable"; exit 3; }
83W_W=${d_n}'/../libIGCM'
84[[ ! -d ${W_W} ]] && { print - "${W_W} unreachable"; exit 3; }
85libIGCM=$(cd ${W_W};/bin/pwd;)
86F_JOB=${libIGCM}'/AA_job';
87[[ ! -f ${F_JOB} ]] && { print - "${F_JOB} unreachable"; exit 3; }
88F_RCI=${libIGCM}'/run.card.init';
89[[ ! -f ${F_RCI} ]] && { print - "${F_RCI} unreachable"; exit 3; }
90#-
91# Host Name
92#-
93x_t=$(${d_n}/w_i_h) 2>/dev/null;
94{ [[ ${?} != 0 ]] && \
95   { print - "\nw_i_h or uname unreachable\n" 1>&2; exit 1; }; } || \
96  [[ ${x_t} = "Unknown" ]] && \
97   { print - "\nLocal_Host not supported\n" 1>&2; exit 1; };
98#-
99W_W=$(sed -n -e "s/^#-Q-  *\([^ ]*\).*$/\1/p" ${F_JOB} | \
100       sort -u | sed -e "/${x_t}/!d");
101[[ '\?'"${W_W}" != '\?'${x_t} ]] && \
102  {
103   print - "\nHost  "${x_t}"  not supported" 1>&2;
104   print - "'default' will be used" 1>&2;
105   x_t='default'
106  }
107#-
108[[ ${x_v} = 'verbose' ]] && \
109 {
110  print - "";
111  print - '--- Host        : '${x_t};
112  print - '--- modipsl     : '${F_MOD};
113  print - '--- libIGCM     : '${libIGCM};
114  print - '--- basic job   : '${F_JOB};
115  print - '--- basic card  : '${F_RCI};
116 }
117#-
118[[ ${x_v} = 'verbose' ]] && print - "\nInstallation of jobs for '${x_t}'";
119#-
120# Accessing to functions (without stack)
121#-
122DEBUG_debug=false
123MirrorlibIGCM=false
124TaskType=computing
125. ${libIGCM}/libIGCM_debug/libIGCM_debug.ksh
126. ${libIGCM}/libIGCM_card/libIGCM_card.ksh
127. ${libIGCM}/libIGCM_date/libIGCM_date.ksh
128. ${libIGCM}/libIGCM_sys/libIGCM_sys.ksh
129if [ $x_e = 'true' ] ; then
130  . ${libIGCM}/libIGCM_ensemble/libIGCM_ensemble.ksh
131fi
132
133NbErr=0
134
135#-
136# Define Project ID to set up jobs header for Curie (TGCC)
137#-
138if ( ! ${x_p} ) ; then
139  if [ X"${x_t}" == "Xcurie" ] ; then
140    ProjectID=$( ccc_myproject | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | head -n 1 )
141    ProjectID=${ProjectID:="gen2211"}
142    answer=""
143    print - "Hit Enter or give project ID (default is ${ProjectID}), possible projects are $( echo $( ccc_myproject | gawk '{ if ( $3 ~ /^project$/ && $4 !~ /^tgcc/ ) { print $4 } }' | sort -u ) ) :"
144    read answer
145
146    if [ "X${answer}" != "X" ] ; then
147      ProjectID=${answer}
148    fi
149  fi
150else
151  if [ X"${x_t}" != "Xcurie" ] ; then
152    print - "\nOption -p invalid with this host (${x_t}): it will be ignored" 1>&2;
153  fi
154fi
155
156#-
157# Define the pattern string to substitute
158#-
159W_P='#-Q- '; W_W=${W_P}${x_t};
160#-
161# Extract list of 'config.card' files
162# and create jobs with AA_job
163#-
164F_CFG='config.card';
165F_CFG_ENS='ensemble.card';
166SUBMIT_DIR_ENS=$( pwd )
167for i in $(find ${d_n}/.. -name ${F_CFG} -print)
168do
169  j=$(cd ${i%/*};/bin/pwd;)
170  n_f=${F_RCI##*/};
171
172  if [ ! X$( echo ${j} | grep EXPERIMENTS ) = X ] ; then
173    # Do not treat config.card if it is in sub-directory of EXPERIMENTS
174    # Continue to next config.card
175    continue
176  else
177    [[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
178  fi
179
180  # Find out if new structure and set .resol filename
181  if [ -d ${j}/EXPERIMENTS ] && [ -d ${j}/GENERAL ] ; then
182    # New Structure
183    [[ ${x_v} = 'verbose' ]] && echo "This is new configuration structure"
184    new_struct=yes
185    resolfile=$j/.resol
186  else
187    # Old Structure
188    new_struct=no
189    resolfile=$j/../.resol
190  fi
191
192  # Get all variables declared in section UserChoices in config.card
193  IGCM_card_DefineArrayFromSection ${j}'/'${F_CFG} UserChoices
194  # Set default values
195  config_UserChoices_JobClass=mono
196  config_UserChoices_JobNumProcTot=1
197  config_UserChoices_ExpType=""
198  RESOL_ATM_3D=this_is_a_test_string
199  RESOL=this_is_another_test_string
200  typeset option
201  for option in ${config_UserChoices[*]} ; do
202    IGCM_card_DefineVariableFromOption ${j}'/'${F_CFG} UserChoices ${option}
203  done
204
205  # Find the JobName : JobName might contain the variable RESOL_ATM_3D that will be replaced by what is in .resol file
206  if [ ! X$( echo ${config_UserChoices_JobName} | grep ${RESOL_ATM_3D} ) = X ] ; then
207    TRUERESOL=$( tail -1 $resolfile | awk "-F=" '{print $2}' )
208    echo TRUERESOL = $TRUERESOL
209    JobName=$( echo ${config_UserChoices_JobName} | sed -e "s/${RESOL_ATM_3D}/${TRUERESOL}/" )
210    IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
211  elif [ ! X$( echo ${config_UserChoices_JobName} | grep ${RESOL} ) = X ] ; then
212    TRUERESOL=$( head -1 $resolfile  )
213    JobName=$( echo ${config_UserChoices_JobName} | sed -e "s/${RESOL}/${TRUERESOL}/" )
214    IGCM_card_WriteOption ${j}'/'${F_CFG} UserChoices JobName ${JobName}
215  else
216    JobName=${config_UserChoices_JobName}
217  fi
218
219  # Check JobName validity : only alphanumerical characters, "-" and "." are authorized
220  ins_job_Check_JobName
221  RetCode=$?
222  [[ $RetCode -gt 0 ]] && continue
223
224  [[ ${x_v} = 'verbose' ]] && echo "JobName=${JobName}"
225
226  # Add specific treatment for new type of directory structure
227  if [ ${new_struct} == yes ] ; then
228
229    if [ "X${config_UserChoices_ExpType}" = X ] ; then
230      echo "\nERROR in ${j}/config.card"
231      echo "ins_job stops here"
232      echo "=> The variable ExpType must be added in config.card in section UserChoices"
233      echo "=> ExpType gives the directory for the .card configurtaion files for the wanted experiement. For exemple ExpType=IPSLCM5/historical"
234      exit 4
235    else
236      [[ ${x_v} = 'verbose' ]] && echo "ExpType= ${config_UserChoices_ExpType}"
237    fi
238
239    if [ -d ${j}/${JobName} ] ; then
240      echo "Directory ${j}/${JobName} exist already, continue next config.card"
241      continue
242    fi
243    echo "=> Submit directory ${JobName} will be created with cards from EXPERIMENTS/${config_UserChoices_ExpType}"
244    cp -r ${j}/EXPERIMENTS/${config_UserChoices_ExpType} ${j}/${JobName}
245    cp -r ${j}/GENERAL/* ${j}/${JobName}/.
246    cp -f ${j}/${F_CFG}  ${j}/${JobName}/.
247    if [ -f ${F_CFG_ENS} ] ; then
248      cp -f ${j}/${F_CFG_ENS}  ${j}/${JobName}/.
249      SUBMIT_DIR_ENS=${j}/${JobName}
250    fi
251    rm -f ${j}/${F_CFG}
252    rm -f ${j}/${F_CFG_ENS}
253    rm -f ${j}/${F_CFG}.bak
254    j=${j}/${JobName}
255    [[ ${x_v} = 'verbose' ]] && echo new j=$j
256  fi
257  # end specific treatment for new type directory structure
258
259  [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; } || \
260   {
261    [[ ${x_v} = 'verbose' ]] && print - "\nCopying file ${F_RCI}\nin directory ${j}";
262    \cp ${F_RCI} ${j};
263   }
264
265  # File name for Job
266  n_f='Job_'${JobName};
267  [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; continue; }
268  [[ ${x_v} = 'verbose' ]] && print - "\nWorking with file ${F_CFG}\nin directory ${j}\nfor ${n_f}";
269  sed -e "/^${W_W} */ s///" \
270      -e "/^${W_P}/d"       \
271      -e "s%::modipsl::%${F_MOD}%" \
272      -e "s/::Jobname::/${JobName}/" \
273      -e "s/::JobClass::/${config_UserChoices_JobClass}/" \
274      -e "s/::JobNumProcTot::/${config_UserChoices_JobNumProcTot}/" \
275      -e "s/::default_project::/${ProjectID}/" \
276      ${F_JOB} > ${j}'/'${n_f}
277  chmod u+x ${j}'/'${n_f}
278done
279#-
280# Extract list of AA_* files in libIGCM
281# and create jobs (for all except AA_job)
282#-
283for i in $(find ${libIGCM} -maxdepth 1 -name "AA_*" -print)
284do
285  i_f=${i##*/};
286  [[ ${i_f} = 'AA_job' ]] && { continue; }
287  j=${i%/*}; n_f=${i_f#AA_}'.job';
288  [[ -f ${j}'/'${n_f} ]] && { ins_job_Warning; continue; }
289  [[ ${x_v} = 'verbose' ]] && print - "\nIn directory ${j}\n${i_f} -> ${n_f}"
290  sed -e "/^${W_W} */ s///" \
291      -e "s%::modipsl::%${F_MOD}%" \
292      -e "/^${W_P}/d"       \
293      -e "s/::default_project::/${ProjectID}/" \
294      ${i} > ${j}'/'${n_f}
295  chmod u+x ${j}'/'${n_f}
296done
297#-
298# set default_project in libIGCM_sys_curie.ksh too.
299#-
300if [ X"${x_t}" == "Xcurie" ] ; then
301  i=${libIGCM}/libIGCM_sys/libIGCM_sys_curie.ksh
302  sed -i -e "s/::default_project::/${ProjectID}/" ${i}
303fi
304#-
305# Limited to hindcast/forecast and date restart Ensemble for the time being
306if [ ${x_e} = 'true' ] ; then
307  #.. Read input data from ensemble.card ..
308  SUBMIT_DIR=${SUBMIT_DIR_ENS}
309  RUN_DIR="${WORKDIR}/ENSEMBLE"
310  #
311  # Copy initial things around and define variables (hindcast/forecast case)
312  IGCM_sys_Cd ${SUBMIT_DIR}
313  IGCM_ensemble_Init
314
315  if [[ ${ensemble_Ens_PARAMETRIC_active} = 'y' ]] ; then
316    echo "WARNING: Parametric Ensemble is not implemented yet..."
317  fi
318
319  if [[ ${ensemble_Ens_DATE_active} = 'y' ]] ; then
320    IGCM_sys_Cd ${SUBMIT_DIR}
321    IGCM_ensemble_DateInit
322    # As it says
323    IGCM_sys_Cd ${SUBMIT_DIR}
324    IGCM_ensemble_DatePeriodicStarts
325    # As it says
326    IGCM_sys_Cd ${SUBMIT_DIR}
327    IGCM_ensemble_DateNonPeriodicStarts
328    # Clean
329    IGCM_sys_Rm -rf ${RUN_DIR}
330  fi
331
332  if [[ ${ensemble_Ens_PERTURB_active} = 'y' ]] ; then
333    IGCM_sys_Cd ${SUBMIT_DIR}
334    IGCM_ensemble_CastInit
335    # As it says
336    IGCM_sys_Cd ${SUBMIT_DIR}
337    IGCM_ensemble_CastPeriodicStarts
338    # As it says
339    IGCM_sys_Cd ${SUBMIT_DIR}
340    IGCM_ensemble_CastNonPeriodicStarts
341    # As it says
342    IGCM_sys_Cd ${SUBMIT_DIR}
343    IGCM_ensemble_CastMemberList
344    # Done
345    IGCM_sys_Cp ${RUN_DIR}/CreatedDir.txt ${SUBMIT_DIR}
346    IGCM_sys_Cd ${SUBMIT_DIR}
347    # Clean
348    IGCM_sys_Rm -rf ${RUN_DIR}
349  fi
350fi
351#-
352[[ ${x_v} = 'verbose' ]] && print - "";
353#-
354# That's all folks
355#-
356
357if [ ${NbErr} -ne 0 ] ; then
358  echo "################ ERROR ################"
359  echo "${NbErr} invalid JobName(s) found, check the log"
360fi
361
362
363exit 0;
Note: See TracBrowser for help on using the repository browser.