| 1 | #!/bin/ksh |
|---|
| 2 | # $Id$ |
|---|
| 3 | ###################################################### |
|---|
| 4 | # Original : C. Talandier for NEMO team |
|---|
| 5 | # Contact : nemo_st@locean-ipsl.upmc.fr |
|---|
| 6 | # |
|---|
| 7 | # INPUT ARGS: 4 + 1 optional |
|---|
| 8 | # - CONF : configuration name (ORCA2_LIM, GYRE ..) |
|---|
| 9 | # - MTIME : timing/no timing (timing or notiming) |
|---|
| 10 | # - JOBTOL: jobs to launch: all, long, 1_short or gtime |
|---|
| 11 | # - OWNEXP: name of the experience performed |
|---|
| 12 | # - REFTAG: (optional): reference tag name to which current |
|---|
| 13 | # version will be compare to |
|---|
| 14 | # |
|---|
| 15 | # WORK: This script aims to (for each standard configuration) : |
|---|
| 16 | # |
|---|
| 17 | # I. Take into account target's specificities |
|---|
| 18 | # Variables below must be filled for a new target |
|---|
| 19 | # - W_XX = name of target |
|---|
| 20 | # - LAUN = name of jobs launcher command |
|---|
| 21 | # - LLJOBS = name of jobs listing command |
|---|
| 22 | # - LSUB = name of running command |
|---|
| 23 | # - LPERF = name of timing command for the whole run |
|---|
| 24 | # - LJTIM = required time for a one year run (/proc) |
|---|
| 25 | # - LJTIMJ = required time for a one year run (job) |
|---|
| 26 | # - CMDGET = name of specific command to retrieve files |
|---|
| 27 | # - SYMBOL = identifier associated to a platform which |
|---|
| 28 | # is used to name all outputs files |
|---|
| 29 | # Ex: For Brodie, this symbol could be "B_" so |
|---|
| 30 | # file names looks like B_solver.stat ... |
|---|
| 31 | # |
|---|
| 32 | # II. Build jobs |
|---|
| 33 | # - based on specific header depending on target, it must |
|---|
| 34 | # be provided for a new target following samples |
|---|
| 35 | # jhd_[target_name]_mon for mono-processors runs |
|---|
| 36 | # jhd_[target_name]_mpi for multi-processors MPI runs |
|---|
| 37 | # jhd_[target_name]_omp for multi-processors OPen-MP runs |
|---|
| 38 | # |
|---|
| 39 | # III. Launch jobs |
|---|
| 40 | # to test both reproductibility and restartability |
|---|
| 41 | # for each configuration NAME_CONF |
|---|
| 42 | # & for each run type [mon, mpi, omp] |
|---|
| 43 | # 3 runs will be launched: |
|---|
| 44 | # - LONG |
|---|
| 45 | # - 1_SHORT |
|---|
| 46 | # - 2_SHORT |
|---|
| 47 | # |
|---|
| 48 | # IV. Launch cron job |
|---|
| 49 | # Since no jobs (for a given configuration) are in |
|---|
| 50 | # the batch queue, this cron will launch the assessment.ksh |
|---|
| 51 | # script |
|---|
| 52 | # |
|---|
| 53 | ########################################################## |
|---|
| 54 | #set -xv |
|---|
| 55 | ########################################################## |
|---|
| 56 | ##### Begin Users modifications |
|---|
| 57 | ########################################################## |
|---|
| 58 | # IODIR : directory where to get ORCA2_LIM_nemo_v3.1.tar |
|---|
| 59 | # PRC : total number of processors which will be used |
|---|
| 60 | ########################################################## |
|---|
| 61 | IODIR=/u/rech/eee/reee512/NEMO/ |
|---|
| 62 | PRC=8 |
|---|
| 63 | TARGET=sx8brodie |
|---|
| 64 | ########################################################## |
|---|
| 65 | ##### End Users modifications |
|---|
| 66 | ########################################################## |
|---|
| 67 | |
|---|
| 68 | CONF=$1 |
|---|
| 69 | MTIME=$2 |
|---|
| 70 | JOBTOL=$3 |
|---|
| 71 | OWNEXP=$4 |
|---|
| 72 | REFTAG=$5 |
|---|
| 73 | RUN=$(basename `pwd`) |
|---|
| 74 | |
|---|
| 75 | # Specific case of Zahir: |
|---|
| 76 | if [ ${RUN} = 'mon' -a ${TARGET} = 'aix6' ] ; then TARGET=ax6_mono ; fi |
|---|
| 77 | |
|---|
| 78 | # --------------------------------------------------------------- |
|---|
| 79 | # I. Take into account target's specificities |
|---|
| 80 | # Select appropriate target lines in Job_${CONF}.ksh |
|---|
| 81 | # --------------------------------------------------------------- |
|---|
| 82 | if [ ${TARGET} = 'sx8brodie' ]; then |
|---|
| 83 | W_XX='#-T- sx8brodie' |
|---|
| 84 | LAUN='qsub ' |
|---|
| 85 | LLJOBS=qstat |
|---|
| 86 | LSUB="mpirun -np ${PRC}" |
|---|
| 87 | LPERF= |
|---|
| 88 | LJTIM='01:20:00' |
|---|
| 89 | LJTIMJ='01:40:00' |
|---|
| 90 | CMDGET=mfget |
|---|
| 91 | SYMBOL=B_ |
|---|
| 92 | elif [ ${TARGET} = 'aix6' ]; then |
|---|
| 93 | W_XX='#-T- aix6' |
|---|
| 94 | LAUN='llsubmit ' |
|---|
| 95 | LLJOBS=Qstat |
|---|
| 96 | LSUB= |
|---|
| 97 | LPERF="poe hpmcount -o perfs_mpi.txt" |
|---|
| 98 | LJTIM='01:10:00' |
|---|
| 99 | LJTIMJ= |
|---|
| 100 | CMDGET=mfget |
|---|
| 101 | SYMBOL=V_ |
|---|
| 102 | elif [ ${TARGET} = 'ax6_mono' ]; then |
|---|
| 103 | W_XX='#-T- ax6_mono' |
|---|
| 104 | LAUN='llsubmit ' |
|---|
| 105 | LLJOBS=Qstat |
|---|
| 106 | LSUB= |
|---|
| 107 | LPERF="hpmcount -o perfs_mon.txt" |
|---|
| 108 | LJTIM='05:10:00' |
|---|
| 109 | LJTIMJ= |
|---|
| 110 | CMDGET=mfget |
|---|
| 111 | SYMBOL=V_ |
|---|
| 112 | elif [ ${TARGET} = 'osxxlf' ]; then |
|---|
| 113 | W_XX='#-T- osxxlf' |
|---|
| 114 | LAUN='./' |
|---|
| 115 | LLJOBS= |
|---|
| 116 | LSUB="mpirun -np ${PRC}" |
|---|
| 117 | LPERF= |
|---|
| 118 | LJTIM= |
|---|
| 119 | LJTIMJ= |
|---|
| 120 | CMDGET=cp |
|---|
| 121 | SYMBOL=D_ |
|---|
| 122 | else |
|---|
| 123 | echo ' #######################################' |
|---|
| 124 | echo ' You must give a target name :' |
|---|
| 125 | echo ' sx8brodie, aix6, ax6_mono or add one' |
|---|
| 126 | echo ' Have a look in lance_batch.ksh script' |
|---|
| 127 | echo ' #######################################' |
|---|
| 128 | stop |
|---|
| 129 | fi |
|---|
| 130 | |
|---|
| 131 | sed -e "s%^$W_XX *%%" job_${CONF}.ksh | grep -v '^#-T- ' > job_${CONF}.$$ |
|---|
| 132 | |
|---|
| 133 | # -------------------------------------------------------------- |
|---|
| 134 | # II. Build jobs |
|---|
| 135 | # Adapt header (batch jobs) and concatanate it with core job |
|---|
| 136 | # -------------------------------------------------------------- |
|---|
| 137 | if [ ${CONF} == 'ORCA2_LIM' ] ; then |
|---|
| 138 | CONFS=ORCA2 |
|---|
| 139 | elif [ ${CONF} == 'GYRE' ] ; then |
|---|
| 140 | CONFS=GYREO |
|---|
| 141 | elif [ ${CONF} == 'GYRE_LOBSTER' ] ; then |
|---|
| 142 | CONFS=GYREL |
|---|
| 143 | elif [ ${CONF} == 'ORCA2_LIM_PISCES' ] ; then |
|---|
| 144 | CONFS=OR2LP |
|---|
| 145 | elif [ ${CONF} == 'ORCA2_OFF_PISCES' ] ; then |
|---|
| 146 | CONFS=OR2OP |
|---|
| 147 | elif [ ${CONF} == 'ZAGRIF' ] ; then |
|---|
| 148 | CONFS=ZAGRI |
|---|
| 149 | else |
|---|
| 150 | CONFS=${CONF} |
|---|
| 151 | fi |
|---|
| 152 | |
|---|
| 153 | case ${TARGET} in |
|---|
| 154 | 'osxxlf' ) |
|---|
| 155 | cp jhd_${TARGET}_${RUN} jhd_${TARGET}_${RUN}_tmp ;; |
|---|
| 156 | 'ax6_mono' ) |
|---|
| 157 | CAR_JNAM=$( grep '\# @ job_name' jhd_${TARGET}_${RUN} ) |
|---|
| 158 | sed -e "s%$CAR_JNAM%\# @ job_name = ${RUN}${CONFS}%" \ |
|---|
| 159 | jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp ;; |
|---|
| 160 | 'aix6' ) |
|---|
| 161 | CAR_JNAM=$( grep '\# @ job_name' jhd_${TARGET}_${RUN} ) |
|---|
| 162 | CAR_JMPI=$( grep '\# @ total_tasks' jhd_${TARGET}_${RUN} ) |
|---|
| 163 | sed -e "s%$CAR_JNAM%\# @ job_name = ${RUN}${CONFS}%" \ |
|---|
| 164 | -e "s%$CAR_JMPI%\# @ total_tasks = ${PRC}%" \ |
|---|
| 165 | jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp ;; |
|---|
| 166 | 'sx8brodie' ) |
|---|
| 167 | CAR_JNAM=$( grep '\#PBS -N ' jhd_${TARGET}_${RUN} ) |
|---|
| 168 | CAR_JNAMO=$( grep '\#PBS -o ' jhd_${TARGET}_${RUN} ) |
|---|
| 169 | CAR_JNAME=$( grep '\#PBS -e ' jhd_${TARGET}_${RUN} ) |
|---|
| 170 | if [ ${RUN} = 'mpi' -o ${RUN} = 'omp' ] ; then |
|---|
| 171 | CAR_JMPI=$( grep '\#PBS -l cpunum_job' jhd_${TARGET}_${RUN} ) |
|---|
| 172 | sed -e "s%$CAR_JNAM%\#PBS -N ${RUN}${CONFS}%" \ |
|---|
| 173 | -e "s%$CAR_JNAMO%\#PBS -o ${RUN}${CONFS}_std%" \ |
|---|
| 174 | -e "s%$CAR_JNAME%\#PBS -e ${RUN}${CONFS}_err%" \ |
|---|
| 175 | -e "s%$CAR_JMPI%\#PBS -l cpunum_job=${PRC}%" \ |
|---|
| 176 | jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp |
|---|
| 177 | else |
|---|
| 178 | sed -e "s%$CAR_JNAM%\#PBS -N ${RUN}${CONFS}%" \ |
|---|
| 179 | -e "s%$CAR_JNAMO%\#PBS -o ${RUN}${CONFS}_std%" \ |
|---|
| 180 | -e "s%$CAR_JNAME%\#PBS -e ${RUN}${CONFS}_err%" \ |
|---|
| 181 | jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp |
|---|
| 182 | fi ;; |
|---|
| 183 | esac |
|---|
| 184 | |
|---|
| 185 | cat jhd_${TARGET}_${RUN}_tmp job_${CONF}.$$ > job_tmp_${RUN}.ksh |
|---|
| 186 | rm -f job_${CONF}.$$ jhd_${TARGET}_${RUN}_tmp |
|---|
| 187 | |
|---|
| 188 | # ----------------------------------------------------------------------------- |
|---|
| 189 | # III. Launch jobs |
|---|
| 190 | # The 3 following jobs are created for the run [type]=mon,mpi,omp : |
|---|
| 191 | # - "job_[type]_long.ksh" ---> long run |
|---|
| 192 | # - "job_[type]_1_short.ksh" ---> stream 1 run |
|---|
| 193 | # - "job_[type]_2_short.ksh" ---> stream 2 run |
|---|
| 194 | # - "job_[type]_gtime.ksh" ---> one year run |
|---|
| 195 | # The first 2 as the last one are immediatly launched whereas "job_[type]_2_short.ksh" |
|---|
| 196 | # is launched only when the "job_[type]_1_short.ksh" short run is finished |
|---|
| 197 | # ----------------------------------------------------------------------------- |
|---|
| 198 | jobnam[0]=long ; jobnam[1]=1_short ; jobnam[2]=2_short ; jobnam[3]=gtime |
|---|
| 199 | dirjob[0]=LONG ; dirjob[1]=1_SHORT ; dirjob[2]=2_SHORT ; dirjob[3]=GTIME |
|---|
| 200 | |
|---|
| 201 | case ${JOBTOL} in |
|---|
| 202 | 'all' ) |
|---|
| 203 | if [ ${MTIME} = 'timing' ] ; then |
|---|
| 204 | ind=0 |
|---|
| 205 | iloop=4 |
|---|
| 206 | else |
|---|
| 207 | ind=0 |
|---|
| 208 | iloop=3 |
|---|
| 209 | fi |
|---|
| 210 | ;; |
|---|
| 211 | 'long' ) |
|---|
| 212 | ind=0 |
|---|
| 213 | iloop=1 ;; |
|---|
| 214 | 'short' ) |
|---|
| 215 | ind=1 |
|---|
| 216 | iloop=3 ;; |
|---|
| 217 | 'gtime' ) |
|---|
| 218 | ind=3 |
|---|
| 219 | iloop=4 ;; |
|---|
| 220 | esac |
|---|
| 221 | |
|---|
| 222 | while [ $ind -lt ${iloop} ] |
|---|
| 223 | do |
|---|
| 224 | CAR_TYPE_tmp=$( grep 'type=' job_tmp_${RUN}.ksh ) |
|---|
| 225 | CAR_TYPE=`echo $CAR_TYPE_tmp | cut -f 1 -d " " ` |
|---|
| 226 | CAR_RUN_tmp=$( grep 'RUN=xxx' job_tmp_${RUN}.ksh ) |
|---|
| 227 | CAR_RUN=`echo $CAR_RUN_tmp | cut -f 1 -d " " ` |
|---|
| 228 | CAR_MPP_tmp=$( grep 'NB_PROCS=xxx' job_tmp_${RUN}.ksh ) |
|---|
| 229 | CAR_MPP=`echo $CAR_MPP_tmp | cut -f 1 -d " " ` |
|---|
| 230 | CAR_SUB_tmp=$( grep 'LAUNCH=xxx' job_tmp_${RUN}.ksh ) |
|---|
| 231 | CAR_SUB=`echo $CAR_SUB_tmp | cut -f 1 -d " " ` |
|---|
| 232 | CAR_EXE_tmp=$( grep 'LEXE=xxx' job_tmp_${RUN}.ksh ) |
|---|
| 233 | CAR_EXE=`echo $CAR_EXE_tmp | cut -f 1 -d " " ` |
|---|
| 234 | CAR_PER_tmp=$( grep 'CPERF=xxx' job_tmp_${RUN}.ksh ) |
|---|
| 235 | CAR_PER=`echo $CAR_PER_tmp | cut -f 1 -d " " ` |
|---|
| 236 | CAR_MAN_tmp=$( grep 'MAINDIR=xxx' job_tmp_${RUN}.ksh ) |
|---|
| 237 | CAR_MAN=`echo $CAR_MAN_tmp | cut -f 1 -d " " ` |
|---|
| 238 | CAR_INP_tmp=$( grep 'D_INPUT=xxx' job_tmp_${RUN}.ksh ) |
|---|
| 239 | CAR_INP=`echo $CAR_INP_tmp | cut -f 1 -d " " ` |
|---|
| 240 | CAR_GET_tmp=$( grep 'D_GET=xxx' job_tmp_${RUN}.ksh ) |
|---|
| 241 | CAR_GET=`echo $CAR_GET_tmp | cut -f 1 -d " " ` |
|---|
| 242 | CAR_RTG_tmp=$( grep 'REF_TAG=xxx' job_tmp_${RUN}.ksh ) |
|---|
| 243 | CAR_RTG=`echo $CAR_RTG_tmp | cut -f 1 -d " " ` |
|---|
| 244 | CAR_MYO_tmp=$( grep 'MYO_EXP=xxx' job_tmp_${RUN}.ksh ) |
|---|
| 245 | CAR_MYO=`echo $CAR_MYO_tmp | cut -f 1 -d " " ` |
|---|
| 246 | CAR_TSY_tmp=$( grep 'TSYMB=xxx' job_tmp_${RUN}.ksh ) |
|---|
| 247 | CAR_TSY=`echo $CAR_TSY_tmp | cut -f 1 -d " " ` |
|---|
| 248 | |
|---|
| 249 | sed -e "s%$CAR_TYPE%type=${jobnam[$ind]}%" \ |
|---|
| 250 | -e "s%$CAR_RUN%RUN=$RUN%" \ |
|---|
| 251 | -e "s%$CAR_MPP%NB_PROCS=$PRC%" \ |
|---|
| 252 | -e "s%$CAR_SUB%LAUNCH='$LAUN'%" \ |
|---|
| 253 | -e "s%$CAR_EXE%LEXE='$LSUB'%" \ |
|---|
| 254 | -e "s%$CAR_PER%CPERF='$LPERF'%" \ |
|---|
| 255 | -e "s%$CAR_INP%D_INPUT=$IODIR%" \ |
|---|
| 256 | -e "s%$CAR_GET%D_GET=$CMDGET%" \ |
|---|
| 257 | -e "s%$CAR_RTG%REF_TAG=$REFTAG%" \ |
|---|
| 258 | -e "s%$CAR_MYO%MYO_EXP=$OWNEXP%" \ |
|---|
| 259 | -e "s%$CAR_TSY%TSYMB=$SYMBOL%" \ |
|---|
| 260 | job_tmp_${RUN}.ksh > job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh |
|---|
| 261 | |
|---|
| 262 | if [ ${jobnam[$ind]} = 'gtime' ] ; then |
|---|
| 263 | |
|---|
| 264 | case ${TARGET} in |
|---|
| 265 | 'ax6_mono' | 'aix6' ) |
|---|
| 266 | CAR_JTIM=$( grep '\# @ wall_clock_limit' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh ) |
|---|
| 267 | sed -e "s%$CAR_JTIM%\# @ wall_clock_limit = ${LJTIM}%" \ |
|---|
| 268 | job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh > job_${RUN}_${jobnam[$ind]}.ksh_tmp ;; |
|---|
| 269 | 'sx8brodie' ) |
|---|
| 270 | CAR_JTIMP=$( grep '\#PBS -l cputim_prc' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh ) |
|---|
| 271 | CAR_JTIMJ=$( grep '\#PBS -l cputim_job' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh ) |
|---|
| 272 | sed -e "s%$CAR_JTIMP%\#PBS -l cputim_prc=${LJTIM}%" \ |
|---|
| 273 | -e "s%$CAR_JTIMJ%\#PBS -l cputim_job=${LJTIMJ}%" \ |
|---|
| 274 | job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh > job_${RUN}_${jobnam[$ind]}.ksh_tmp ;; |
|---|
| 275 | esac |
|---|
| 276 | mv job_${RUN}_${jobnam[$ind]}.ksh_tmp job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh |
|---|
| 277 | |
|---|
| 278 | fi |
|---|
| 279 | |
|---|
| 280 | chmod 750 job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh |
|---|
| 281 | |
|---|
| 282 | if [ ! -d ${dirjob[$ind]}/$OWNEXP ] ; then |
|---|
| 283 | mkdir ${dirjob[$ind]}/$OWNEXP |
|---|
| 284 | fi |
|---|
| 285 | mv job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh ${dirjob[$ind]}/${OWNEXP}/. |
|---|
| 286 | |
|---|
| 287 | if [ ${jobnam[$ind]} != '2_short' ] ; then |
|---|
| 288 | cd ${dirjob[$ind]}/${OWNEXP} |
|---|
| 289 | ${LAUN}job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh & |
|---|
| 290 | cd ../.. |
|---|
| 291 | fi |
|---|
| 292 | |
|---|
| 293 | let ind=$ind+1 |
|---|
| 294 | done |
|---|
| 295 | rm -f job_tmp_${RUN}.ksh |
|---|
| 296 | |
|---|
| 297 | # List all jobs submitted |
|---|
| 298 | ${LLJOBS} |
|---|
| 299 | |
|---|
| 300 | # ----------------------------------------------------------------------------- |
|---|
| 301 | # IV. Launch cron to be able to launch the assessment step |
|---|
| 302 | # ----------------------------------------------------------------------------- |
|---|
| 303 | if [ ${RUN} == 'mon' ] ; then |
|---|
| 304 | cd .. |
|---|
| 305 | |
|---|
| 306 | CAR_LJOB_tmp=$( grep 'LJOBS=' cron_jobs.ksh ) |
|---|
| 307 | CAR_LJOB=`echo $CAR_LJOB_tmp | cut -f 1 -d " " ` |
|---|
| 308 | |
|---|
| 309 | sed -e "s%$CAR_LJOB%LJOBS=${LLJOBS}%" cron_jobs.ksh > cron_jobs.$$ |
|---|
| 310 | mv cron_jobs.$$ cron_jobs.ksh && rm -rf cron_jobs.$$ |
|---|
| 311 | chmod 744 cron_jobs.ksh |
|---|
| 312 | |
|---|
| 313 | if [ ${TARGET} != 'osxxlf' ]; then |
|---|
| 314 | ./cron_jobs.ksh ${CONF} ${CONFS} & |
|---|
| 315 | fi |
|---|
| 316 | fi |
|---|
