#!/bin/ksh
# $Id$
######################################################
# Original : C. Talandier for NEMO team
# Contact  : nemo_st@locean-ipsl.upmc.fr
#
# INPUT ARGS: 4 + 1 optional
#     - CONF  : configuration name (ORCA2_LIM, GYRE ..)
#     - MTIME : timing/no timing (timing or notiming)
#     - JOBTOL: jobs to launch: all, long, 1_short or gtime
#     - OWNEXP: name of the experience performed
#     - REFTAG: (optional): reference tag name to which current 
#               version will be compare to
#
# WORK: This script aims to (for each standard configuration) :
#
#    I.   Take into account target's specificities 
#         Variables below must be filled for a new target
#         - W_XX   = name of target 
#         - LAUN   = name of jobs launcher command
#         - LLJOBS = name of jobs listing command
#         - LSUB   = name of running command
#         - LPERF  = name of timing command for the whole run
#         - LJTIM  = required time for a one year run (/proc)
#         - LJTIMJ = required time for a one year run (job)
#         - CMDGET = name of specific command to retrieve files
#         - SYMBOL = identifier associated to a platform which 
#                    is used to name all outputs files
#                    Ex: For Brodie, this symbol could be "B_" so
#                    file names looks like B_solver.stat ...
#
#    II.  Build jobs
#         - based on specific header depending on target, it must 
#           be provided for a new target following samples
#           jhd_[target_name]_mon for mono-processors runs
#           jhd_[target_name]_mpi for multi-processors MPI runs
#           jhd_[target_name]_omp for multi-processors OPen-MP runs
#
#    III. Launch jobs
#         to test both reproductibility and restartability
#         for each configuration NAME_CONF 
#         & for each run type [mon, mpi, omp] 
#         3 runs will be launched:
#              - LONG
#              - 1_SHORT
#              - 2_SHORT
#
#    IV. Launch cron job
#         Since no jobs (for a given configuration) are in 
#         the batch queue, this cron will launch the assessment.ksh 
#         script
#
##########################################################
#set -xv
##########################################################
##### Begin Users modifications
##########################################################
# IODIR     : directory where to get ORCA2_LIM_nemo_v3.1.tar 
# PRC       : total number of processors which will be used
##########################################################
IODIR=/u/rech/eee/reee512/NEMO/
PRC=8
TARGET=sx8brodie
##########################################################
##### End   Users modifications
##########################################################

CONF=$1
MTIME=$2
JOBTOL=$3
OWNEXP=$4
REFTAG=$5
RUN=$(basename `pwd`)

# ---------------------------------------------------------------
#  I. Take into account target's specificities 
#     Select appropriate target lines in Job_${CONF}.ksh
# ---------------------------------------------------------------
if   [ ${TARGET} = 'sx8brodie' ]; then
    W_XX='#-T- sx8brodie'
    LAUN='qsub '
    LLJOBS=qstat
    LSUB="mpirun -np ${PRC}"
    LPERF=
    LJTIM='01:20:00'
    LJTIMJ='01:40:00'
    CMDGET=mfget
    SYMBOL=B_
elif [ ${TARGET} = 'sx8mercure' ]; then
    W_XX='#-T- sx8mercure'
    LAUN='qsub '
    LLJOBS=qstat
    LSUB="mpirun -np ${PRC}"
    LPERF=
    LJTIM='01:20:00'
    LJTIMJ='01:40:00'
    CMDGET=cp
    SYMBOL=M_
elif [ ${TARGET} = 'aix6' ]; then
    W_XX='#-T- aix6'
    LAUN='llsubmit '
    LLJOBS=Qstat
    LSUB=
    LPERF="poe hpmcount -o perfs_mpi.txt" 
    LJTIM='01:10:00'
    LJTIMJ=
    CMDGET=mfget
    SYMBOL=V_
elif [ ${TARGET} = 'osxxlf' ]; then
    W_XX='#-T- osxxlf'
    LAUN='./'
    LLJOBS=
    LSUB="mpirun -np ${PRC}"
    LPERF=
    LJTIM=
    LJTIMJ=
    CMDGET=cp
    SYMBOL=D_
else
    echo '	#######################################'
    echo ' 	You must give a target name :'
    echo ' 	sx8brodie, aix6 or add one'
    echo ' 	Have a look in lance_batch.ksh script'
    echo '	#######################################'
    stop
fi

sed -e "s%^$W_XX *%%" job_${CONF}.ksh | grep -v '^#-T- ' > job_${CONF}.$$

# --------------------------------------------------------------
# II. Build jobs 
#     Adapt header (batch jobs) and concatanate it with core job
# --------------------------------------------------------------
if [ ${CONF} == 'ORCA2_LIM' ] ; then
CONFS=ORCA2
elif [ ${CONF} == 'GYRE' ] ; then
CONFS=GYREO
elif [ ${CONF} == 'GYRE_LOBSTER' ] ; then
CONFS=GYREL
elif [ ${CONF} == 'ORCA2_LIM_PISCES' ] ; then
CONFS=OR2LP
elif [ ${CONF} == 'ORCA2_OFF_PISCES' ] ; then
CONFS=OR2OP
elif [ ${CONF} == 'ZAGRIF' ] ; then
CONFS=ZAGRI
else
CONFS=${CONF}
fi

case ${TARGET} in
     'osxxlf' )
	  cp jhd_${TARGET}_${RUN} jhd_${TARGET}_${RUN}_tmp  ;;
     'aix6' )
          CAR_JNAM=$( grep '\# @ job_name' jhd_${TARGET}_${RUN} )
          CAR_JMPI=$( grep '\# @ total_tasks' jhd_${TARGET}_${RUN} )
          sed    -e "s%$CAR_JNAM%\# @ job_name = ${RUN}${CONFS}%" \
          -e "s%$CAR_JMPI%\# @ total_tasks = ${PRC}%" \
          jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp  ;;
     'sx8brodie' )
          CAR_JNAM=$( grep '\#PBS -N ' jhd_${TARGET}_${RUN} )
          CAR_JNAMO=$( grep '\#PBS -o ' jhd_${TARGET}_${RUN} )
          CAR_JNAME=$( grep '\#PBS -e ' jhd_${TARGET}_${RUN} )
          if [ ${RUN} = 'mpi' -o ${RUN} = 'omp' ] ; then
              CAR_JMPI=$( grep '\#PBS -l cpunum_job' jhd_${TARGET}_${RUN} )
              sed    -e "s%$CAR_JNAM%\#PBS -N  ${RUN}${CONFS}%" \
              -e "s%$CAR_JNAMO%\#PBS -o  ${RUN}${CONFS}_std%" \
              -e "s%$CAR_JNAME%\#PBS -e  ${RUN}${CONFS}_err%" \
              -e "s%$CAR_JMPI%\#PBS -l cpunum_job=${PRC}%" \
              jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp
          else
              sed    -e "s%$CAR_JNAM%\#PBS -N  ${RUN}${CONFS}%" \
              -e "s%$CAR_JNAMO%\#PBS -o  ${RUN}${CONFS}_std%" \
              -e "s%$CAR_JNAME%\#PBS -e  ${RUN}${CONFS}_err%" \
              jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp
          fi ;;
     'sx8mercure' )
          CAR_JNAM=$( grep '\#PBS -N ' jhd_${TARGET}_${RUN} )
          CAR_JNAMO=$( grep '\#PBS -o ' jhd_${TARGET}_${RUN} )
          CAR_JNAME=$( grep '\#PBS -e ' jhd_${TARGET}_${RUN} )
          if [ ${RUN} = 'mpi' -o ${RUN} = 'omp' ] ; then
              CAR_JMPI=$( grep '\#PBS -l cpunum_job' jhd_${TARGET}_${RUN} )
              sed    -e "s%$CAR_JNAM%\#PBS -N  ${RUN}${CONFS}%" \
              -e "s%$CAR_JNAMO%\#PBS -o  ${RUN}${CONFS}_std%" \
              -e "s%$CAR_JNAME%\#PBS -e  ${RUN}${CONFS}_err%" \
              -e "s%$CAR_JMPI%\#PBS -l cpunum_job=${PRC}%" \
              jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp
          else
              sed    -e "s%$CAR_JNAM%\#PBS -N  ${RUN}${CONFS}%" \
              -e "s%$CAR_JNAMO%\#PBS -o  ${RUN}${CONFS}_std%" \
              -e "s%$CAR_JNAME%\#PBS -e  ${RUN}${CONFS}_err%" \
              jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp
          fi ;;
esac

cat jhd_${TARGET}_${RUN}_tmp job_${CONF}.$$ > job_tmp_${RUN}.ksh
rm -f job_${CONF}.$$ jhd_${TARGET}_${RUN}_tmp

# -----------------------------------------------------------------------------
# III. Launch jobs
#      The 3 following jobs are created for the run [type]=mon,mpi,omp :
#      - "job_[type]_long.ksh"      ---> long run
#      - "job_[type]_1_short.ksh"   ---> stream 1 run
#      - "job_[type]_2_short.ksh"   ---> stream 2 run
#      - "job_[type]_gtime.ksh"     ---> one year run
#      The first 2 as the last one are immediatly launched whereas "job_[type]_2_short.ksh"
#      is launched only when the "job_[type]_1_short.ksh" short run is finished
# -----------------------------------------------------------------------------
jobnam[0]=long ; jobnam[1]=1_short ; jobnam[2]=2_short ; jobnam[3]=gtime 
dirjob[0]=LONG ; dirjob[1]=1_SHORT ; dirjob[2]=2_SHORT ; dirjob[3]=GTIME 

case ${JOBTOL} in
     'all' )
              if [ ${MTIME} = 'timing' ] ; then 
	          ind=0 
		  iloop=4
	      else
	          ind=0 
		  iloop=3
              fi 
	      ;;
     'long' )
              ind=0 
	      iloop=1 ;;
     'short' )
              ind=1 
	      iloop=3 ;;
     'gtime' )
              ind=3 
	      iloop=4 ;;
esac

while [ $ind -lt ${iloop} ]
do
       CAR_TYPE_tmp=$( grep 'type=' job_tmp_${RUN}.ksh )
       CAR_TYPE=`echo $CAR_TYPE_tmp | cut -f 1 -d " " `
       CAR_RUN_tmp=$( grep 'RUN=xxx' job_tmp_${RUN}.ksh )
       CAR_RUN=`echo $CAR_RUN_tmp | cut -f 1 -d " " `
       CAR_MPP_tmp=$( grep 'NB_PROCS=xxx' job_tmp_${RUN}.ksh )
       CAR_MPP=`echo $CAR_MPP_tmp | cut -f 1 -d " " `
       CAR_SUB_tmp=$( grep 'LAUNCH=xxx' job_tmp_${RUN}.ksh )
       CAR_SUB=`echo $CAR_SUB_tmp | cut -f 1 -d " " `
       CAR_EXE_tmp=$( grep 'LEXE=xxx' job_tmp_${RUN}.ksh )
       CAR_EXE=`echo $CAR_EXE_tmp | cut -f 1 -d " " `
       CAR_PER_tmp=$( grep 'CPERF=xxx' job_tmp_${RUN}.ksh )
       CAR_PER=`echo $CAR_PER_tmp | cut -f 1 -d " " `
       CAR_MAN_tmp=$( grep 'MAINDIR=xxx' job_tmp_${RUN}.ksh )
       CAR_MAN=`echo $CAR_MAN_tmp | cut -f 1 -d " " `
       CAR_INP_tmp=$( grep 'D_INPUT=xxx' job_tmp_${RUN}.ksh )
       CAR_INP=`echo $CAR_INP_tmp | cut -f 1 -d " " `
       CAR_GET_tmp=$( grep 'D_GET=xxx' job_tmp_${RUN}.ksh )
       CAR_GET=`echo $CAR_GET_tmp | cut -f 1 -d " " `
       CAR_RTG_tmp=$( grep 'REF_TAG=xxx' job_tmp_${RUN}.ksh )
       CAR_RTG=`echo $CAR_RTG_tmp | cut -f 1 -d " " `
       CAR_MYO_tmp=$( grep 'MYO_EXP=xxx' job_tmp_${RUN}.ksh )
       CAR_MYO=`echo $CAR_MYO_tmp | cut -f 1 -d " " `
       CAR_TSY_tmp=$( grep 'TSYMB=xxx' job_tmp_${RUN}.ksh )
       CAR_TSY=`echo $CAR_TSY_tmp | cut -f 1 -d " " `

       sed    -e "s%$CAR_TYPE%type=${jobnam[$ind]}%" \
       -e "s%$CAR_RUN%RUN=$RUN%" \
       -e "s%$CAR_MPP%NB_PROCS=$PRC%" \
       -e "s%$CAR_SUB%LAUNCH='$LAUN'%" \
       -e "s%$CAR_EXE%LEXE='$LSUB'%" \
       -e "s%$CAR_PER%CPERF='$LPERF'%" \
       -e "s%$CAR_INP%D_INPUT=$IODIR%" \
       -e "s%$CAR_GET%D_GET=$CMDGET%" \
       -e "s%$CAR_RTG%REF_TAG=$REFTAG%" \
       -e "s%$CAR_MYO%MYO_EXP=$OWNEXP%" \
       -e "s%$CAR_TSY%TSYMB=$SYMBOL%" \
       job_tmp_${RUN}.ksh > job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh

       if [ ${jobnam[$ind]} = 'gtime' ] ; then

           case ${TARGET} in
                'aix6' )
                     CAR_JTIM=$( grep '\# @ wall_clock_limit' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh )
                     sed    -e "s%$CAR_JTIM%\# @ wall_clock_limit = ${LJTIM}%" \
                     job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh > job_${RUN}_${jobnam[$ind]}.ksh_tmp  ;;
                'sx8brodie' )
                     CAR_JTIMP=$( grep '\#PBS -l cputim_prc' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh )
                     CAR_JTIMJ=$( grep '\#PBS -l cputim_job' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh )
                     sed    -e "s%$CAR_JTIMP%\#PBS -l cputim_prc=${LJTIM}%" \
                     -e "s%$CAR_JTIMJ%\#PBS -l cputim_job=${LJTIMJ}%" \
                     job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh > job_${RUN}_${jobnam[$ind]}.ksh_tmp  ;;
                'sx8mercure' )
                     CAR_JTIMP=$( grep '\#PBS -l cputim_prc' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh )
                     CAR_JTIMJ=$( grep '\#PBS -l cputim_job' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh )
                     sed    -e "s%$CAR_JTIMP%\#PBS -l cputim_prc=${LJTIM}%" \
                     -e "s%$CAR_JTIMJ%\#PBS -l cputim_job=${LJTIMJ}%" \
                     job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh > job_${RUN}_${jobnam[$ind]}.ksh_tmp  ;;
           esac
           mv job_${RUN}_${jobnam[$ind]}.ksh_tmp job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh

       fi

       chmod 750 job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh

       if [ ! -d ${dirjob[$ind]}/$OWNEXP ] ;  then
           mkdir ${dirjob[$ind]}/$OWNEXP
       fi
       mv job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh ${dirjob[$ind]}/${OWNEXP}/.

       if [ ${jobnam[$ind]} != '2_short' ] ; then
           cd ${dirjob[$ind]}/${OWNEXP}
           ${LAUN}job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh &
           cd ../..
       fi

       let ind=$ind+1
done
rm -f job_tmp_${RUN}.ksh

# List all jobs submitted
${LLJOBS}

# -----------------------------------------------------------------------------
# IV. Launch cron to be able to launch the assessment step
# -----------------------------------------------------------------------------
if [ ${RUN} == 'mon' ] ; then
    cd ..
    
    CAR_LJOB_tmp=$( grep 'LJOBS=' cron_jobs.ksh )
    CAR_LJOB=`echo $CAR_LJOB_tmp | cut -f 1 -d " " `
    
    sed    -e "s%$CAR_LJOB%LJOBS=${LLJOBS}%" cron_jobs.ksh > cron_jobs.$$
    mv cron_jobs.$$ cron_jobs.ksh &&  rm -rf cron_jobs.$$
    chmod 744 cron_jobs.ksh 

    if [ ${TARGET} != 'osxxlf' ]; then
        ./cron_jobs.ksh ${CONF} ${CONFS} &
    fi
fi
