#!/bin/ksh
######################################################
# Original : C. Talandier for ESOPA
# Contact  : opatlod@locean-ipsl.upmc.fr
#
# INPUT ARGS: 3 + 1 optional
#     - CONF  : configuration name (ORCA2_LIM, GYRE ..)
#     - MTIME : timing/no timing (timing or notiming)
#     - JOBTOL: jobs to launch: all, long, 1_short or gtime
#     - REFTAG: (optional): reference tag name to which current 
#               version will be compare to
#
# WORK: This script aims to (for each standard configuration) :
#
#    I.   Take into account target's specificities 
#         Variables below must be filled for a new target
#         - W_XX   = name of target 
#         - LAUN   = name of jobs launcher command
#         - LLJOBS = name of jobs listing command
#         - LSUB   = name of running command
#         - LPERF  = name of timing command for the whole run
#         - LJTIM  = required time for a one year run (/proc)
#         - LJTIMJ = required time for a one year run (job)
#         - CMDGET = name of specific command to retrieve files
#         - SYMBOL = identifier associated to a platform which 
#                    is used to name all outputs files
#                    Ex: For Brodie, this symbol could be "B_" so
#                    file names looks like B_solver.stat ...
#
#    II.  Build jobs
#         - based on specific header depending on target, it must 
#           be provided for a new target following samples
#           jhd_[target_name]_mon for mono-processors runs
#           jhd_[target_name]_mpi for multi-processors MPI runs
#           jhd_[target_name]_omp for multi-processors OPen-MP runs
#
#    III. Launch jobs
#         to test both reproductibility and restartability
#         for each configuration NAME_CONF 
#         & for each run type [mon, mpi, omp] 
#         3 runs will be launched:
#              - LONG
#              - 1_SHORT
#              - 2_SHORT
#
#    IV. Launch cron job
#         Since no jobs (for a given configuration) are in 
#         the batch queue, this cron will launch the assessment.ksh 
#         script
#
##########################################################
#set -xv
##########################################################
##### Begin Users modifications
##########################################################
# IODIR     : directory where to get ORCA2_LIM_nemo_v2.tar 
# PRC       : total number of processors which will be used
##########################################################
IODIR=/Volumes/FENUA/ORCA2_LIM_nemo_v2_IO
PRC=8
TARGET=osxxlf
##########################################################
##### End   Users modifications
##########################################################

CONF=$1
MTIME=$2
JOBTOL=$3
REFTAG=$4
RUN=$(basename `pwd`)

# Specific case of Zahir:
if [ ${RUN} = 'mon' -a ${TARGET} = 'aix' ] ; then TARGET=ax_mono ; fi

# ---------------------------------------------------------------
#  I. Take into account target's specificities 
#     Select appropriate target lines in Job_${CONF}.ksh
# ---------------------------------------------------------------
if   [ ${TARGET} = 'sx8brodie' ]; then
    W_XX='#-T- sx8brodie'
    LAUN='qsub '
    LLJOBS=qstat
    LSUB="mpirun -np ${PRC}"
    LPERF=
    LJTIM='01:20:00'
    LJTIMJ='01:40:00'
    CMDGET=mfget
    SYMBOL=B_
elif [ ${TARGET} = 'aix' ]; then
    W_XX='#-T- aix'
    LAUN='llsubmit '
    LLJOBS=Qstat
    LSUB=
    LPERF="poe hpmcount -o perfs_mpi.txt" 
    LJTIM=4000
    LJTIMJ=
    CMDGET=mfget
    SYMBOL=Z_
elif [ ${TARGET} = 'ax_mono' ]; then
    W_XX='#-T- ax_mono'
    LAUN='llsubmit '
    LLJOBS=Qstat
    LSUB=
    LPERF="hpmcount -o perfs_mon.txt" 
    LJTIM=30000
    LJTIMJ=
    CMDGET=mfget
    SYMBOL=Z_
elif [ ${TARGET} = 'osxxlf' ]; then
    W_XX='#-T- osxxlf'
    LAUN='./'
    LLJOBS=
    LSUB="mpirun -np ${PRC}"
    LPERF=
    LJTIM=
    LJTIMJ=
    CMDGET=cp
    SYMBOL=D_
else
    echo '	#######################################'
    echo ' 	You must give a target name :'
    echo ' 	sx8brodie, aix, ax_mono or add one'
    echo ' 	Have a look in lance_batch.ksh script'
    echo '	#######################################'
    stop
fi

sed -e "s/^$W_XX *//" job_${CONF}.ksh | grep -v '^#-T- ' > job_${CONF}.$$

# --------------------------------------------------------------
# II. Build jobs 
#     Adapt header (batch jobs) and concatanate it with core job
# --------------------------------------------------------------
case ${TARGET} in
     'osxxlf' )
	  cp jhd_${TARGET}_${RUN} jhd_${TARGET}_${RUN}_tmp  ;;
     'ax_mono' )
          CAR_JNAM=$( grep '\# @ job_name' jhd_${TARGET}_${RUN} )
          sed    -e "s/$CAR_JNAM/\# @ job_name = ${RUN}${CONF}/" \
          jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp  ;;
     'aix' )
          CAR_JNAM=$( grep '\# @ job_name' jhd_${TARGET}_${RUN} )
          CAR_JMPI=$( grep '\# @ total_tasks' jhd_${TARGET}_${RUN} )
          sed    -e "s/$CAR_JNAM/\# @ job_name = ${RUN}${CONF}/" \
          -e "s/$CAR_JMPI/\# @ total_tasks = ${PRC}/" \
          jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp  ;;
     'sx8brodie' )
          CAR_JNAM=$( grep '\#PBS -N ' jhd_${TARGET}_${RUN} )
          CAR_JNAMO=$( grep '\#PBS -o ' jhd_${TARGET}_${RUN} )
          CAR_JNAME=$( grep '\#PBS -e ' jhd_${TARGET}_${RUN} )
          if [ ${RUN} = 'mpi' -o ${RUN} = 'omp' ] ; then
              CAR_JMPI=$( grep '\#PBS -l cpunum_job' jhd_${TARGET}_${RUN} )
              sed    -e "s/$CAR_JNAM/\#PBS -N  ${RUN}${CONF}/" \
              -e "s/$CAR_JNAMO/\#PBS -o  ${RUN}${CONF}_std/" \
              -e "s/$CAR_JNAME/\#PBS -e  ${RUN}${CONF}_err/" \
              -e "s/$CAR_JMPI/\#PBS -l cpunum_job=${PRC}/" \
              jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp
          else
              sed    -e "s/$CAR_JNAM/\#PBS -N  ${RUN}${CONF}/" \
              -e "s/$CAR_JNAMO/\#PBS -o  ${RUN}${CONF}_std/" \
              -e "s/$CAR_JNAME/\#PBS -e  ${RUN}${CONF}_err/" \
              jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp
          fi ;;
esac

cat jhd_${TARGET}_${RUN}_tmp job_${CONF}.$$ > job_tmp_${RUN}.ksh
rm -f job_${CONF}.$$ jhd_${TARGET}_${RUN}_tmp

# -----------------------------------------------------------------------------
# III. Launch jobs
#      The 3 following jobs are created for the run [type]=mon,mpi,omp :
#      - "job_[type]_long.ksh"      ---> long run
#      - "job_[type]_1_short.ksh"   ---> stream 1 run
#      - "job_[type]_2_short.ksh"   ---> stream 2 run
#      - "job_[type]_gtime.ksh"     ---> one year run
#      The first 2 as the last one are immediatly launched whereas "job_[type]_2_short.ksh"
#      is launched only when the "job_[type]_1_short.ksh" short run is finished
# -----------------------------------------------------------------------------
jobnam[0]=long ; jobnam[1]=1_short ; jobnam[2]=2_short ; jobnam[3]=gtime 
dirjob[0]=LONG ; dirjob[1]=1_SHORT ; dirjob[2]=2_SHORT ; dirjob[3]=GTIME 

case ${JOBTOL} in
     'all' )
              if [ ${MTIME} = 'timing' ] ; then 
	          ind=0 
		  iloop=4
	      else
	          ind=0 
		  iloop=3
              fi 
	      ;;
     'long' )
              ind=0 
	      iloop=1 ;;
     'short' )
              ind=1 
	      iloop=3 ;;
     'gtime' )
              ind=3 
	      iloop=4 ;;
esac

while [ $ind -lt ${iloop} ]
do
       CAR_TYPE_tmp=$( grep 'type=' job_tmp_${RUN}.ksh )
       CAR_TYPE=`echo $CAR_TYPE_tmp | cut -f 1 -d " " `
       CAR_RUN_tmp=$( grep 'RUN=xxx' job_tmp_${RUN}.ksh )
       CAR_RUN=`echo $CAR_RUN_tmp | cut -f 1 -d " " `
       CAR_MPP_tmp=$( grep 'NB_PROCS=xxx' job_tmp_${RUN}.ksh )
       CAR_MPP=`echo $CAR_MPP_tmp | cut -f 1 -d " " `
       CAR_SUB_tmp=$( grep 'LAUNCH=xxx' job_tmp_${RUN}.ksh )
       CAR_SUB=`echo $CAR_SUB_tmp | cut -f 1 -d " " `
       CAR_EXE_tmp=$( grep 'LEXE=xxx' job_tmp_${RUN}.ksh )
       CAR_EXE=`echo $CAR_EXE_tmp | cut -f 1 -d " " `
       CAR_PER_tmp=$( grep 'CPERF=xxx' job_tmp_${RUN}.ksh )
       CAR_PER=`echo $CAR_PER_tmp | cut -f 1 -d " " `
       CAR_MAN_tmp=$( grep 'MAINDIR=xxx' job_tmp_${RUN}.ksh )
       CAR_MAN=`echo $CAR_MAN_tmp | cut -f 1 -d " " `
       CAR_INP_tmp=$( grep 'D_INPUT=xxx' job_tmp_${RUN}.ksh )
       CAR_INP=`echo $CAR_INP_tmp | cut -f 1 -d " " `
       CAR_GET_tmp=$( grep 'D_GET=xxx' job_tmp_${RUN}.ksh )
       CAR_GET=`echo $CAR_GET_tmp | cut -f 1 -d " " `
       CAR_RTG_tmp=$( grep 'REF_TAG=xxx' job_tmp_${RUN}.ksh )
       CAR_RTG=`echo $CAR_RTG_tmp | cut -f 1 -d " " `
       CAR_TSY_tmp=$( grep 'TSYMB=xxx' job_tmp_${RUN}.ksh )
       CAR_TSY=`echo $CAR_TSY_tmp | cut -f 1 -d " " `

       sed    -e "s/$CAR_TYPE/type=${jobnam[$ind]}/" \
       -e "s/$CAR_RUN/RUN=$RUN/" \
       -e "s/$CAR_MPP/NB_PROCS=$PRC/" \
       -e "s@$CAR_SUB@LAUNCH='$LAUN'@" \
       -e "s/$CAR_EXE/LEXE='$LSUB'/" \
       -e "s/$CAR_PER/CPERF='$LPERF'/" \
       -e "s?$CAR_INP?D_INPUT=$IODIR?" \
       -e "s/$CAR_GET/D_GET=$CMDGET/" \
       -e "s/$CAR_RTG/REF_TAG=$REFTAG/" \
       -e "s/$CAR_TSY/TSYMB=$SYMBOL/" \
       job_tmp_${RUN}.ksh > job_${RUN}_${jobnam[$ind]}.ksh

       if [ ${jobnam[$ind]} = 'gtime' ] ; then

           case ${TARGET} in
                'ax_mono' | 'aix' )
                     CAR_JTIM=$( grep '\# @ cpu_limit' job_${RUN}_${jobnam[$ind]}.ksh )
                     sed    -e "s/$CAR_JTIM/\# @ cpu_limit = ${LJTIM}/" \
                     job_${RUN}_${jobnam[$ind]}.ksh > job_${RUN}_${jobnam[$ind]}.ksh_tmp  ;;
                'sx8brodie' )
                     CAR_JTIMP=$( grep '\#PBS -l cputim_prc' job_${RUN}_${jobnam[$ind]}.ksh )
                     CAR_JTIMJ=$( grep '\#PBS -l cputim_job' job_${RUN}_${jobnam[$ind]}.ksh )
                     sed    -e "s/$CAR_JTIMP/\#PBS -l cputim_prc=${LJTIM}/" \
                     -e "s/$CAR_JTIMJ/\#PBS -l cputim_job=${LJTIMJ}/" \
                     job_${RUN}_${jobnam[$ind]}.ksh > job_${RUN}_${jobnam[$ind]}.ksh_tmp  ;;
           esac
           mv job_${RUN}_${jobnam[$ind]}.ksh_tmp job_${RUN}_${jobnam[$ind]}.ksh

       fi

       chmod 750 job_${RUN}_${jobnam[$ind]}.ksh
       mv job_${RUN}_${jobnam[$ind]}.ksh ${dirjob[$ind]}/.

       if [ ${jobnam[$ind]} != '2_short' ] ; then
           cd ${dirjob[$ind]}
           ${LAUN}job_${RUN}_${jobnam[$ind]}.ksh &
           cd ../
       fi

       let ind=$ind+1
done
rm -f job_tmp_${RUN}.ksh

# List all jobs submitted
${LLJOBS}

# -----------------------------------------------------------------------------
# IV. Launch cron to be able to launch the assessment step
# -----------------------------------------------------------------------------
if [ ${RUN} == 'mon' ] ; then
    cd ..
    
    CAR_LJOB_tmp=$( grep 'LJOBS=' cron_jobs.ksh )
    CAR_LJOB=`echo $CAR_LJOB_tmp | cut -f 1 -d " " `
    
    sed    -e "s/$CAR_LJOB/LJOBS=${LLJOBS}/" cron_jobs.ksh > cron_jobs.$$
    mv cron_jobs.$$ cron_jobs.ksh &&  rm -rf cron_jobs.$$
    chmod 744 cron_jobs.ksh 

    if [ ${TARGET} != 'osxxlf' ]; then
        ./cron_jobs.ksh ${CONF} &
    fi
fi