#!/bin/ksh
# $Id$
######################################################
# Original : C. Talandier for NEMO team
# Contact  : nemo_st@locean-ipsl.upmc.fr
#
# INPUT ARGS: 9
#     - CONF  : configuration name (ORCA2_LIM, GYRE ..)
#     - MTIME : timing/no timing (timing or notiming)
#     - JOBTOL: jobs to launch: all, long, 1_short or gtime
#     - OWNEXP: name of the experience performed
#     - PRC   : total number of processors
#     - REFTAG: reference tag name to be compare to
#     - USE_IOMPUT: true of false (default false)
#     - USE_IOSERVER: true of false (default false)
#     - NB_PROCS_IOSERVER: (default 0)
#
# WORK: This script aims to (for each standard configuration) :
#
#    I.   Take into account target's specificities 
#         Variables below must be filled for a new target
#         - W_XX   = name of target 
#         - LAUN   = name of jobs launcher command
#         - LLJOBS = name of jobs listing command
#         - LSUB   = name of running command
#         - LPERF  = name of timing command for the whole run
#         - LJTIM  = required time for a one year run (/proc)
#         - LJTIMJ = required time for a one year run (job)
#         - CMDGET = name of specific command to retrieve files
#         - SYMBOL = identifier associated to a platform which 
#                    is used to name all outputs files
#                    Ex: For Brodie, this symbol could be "B_" so
#                    file names looks like B_solver.stat ...
#
#    II.  Build jobs
#         - based on specific header depending on target, it must 
#           be provided for a new target following samples
#           jhd_[target_name]_mon for mono-processors runs
#           jhd_[target_name]_mpi for multi-processors MPI runs
#           jhd_[target_name]_omp for multi-processors OPen-MP runs
#
#    III. Launch jobs
#         to test both reproductibility and restartability
#         for each configuration NAME_CONF 
#         & for each run type [mon, mpi, omp] 
#         3 runs will be launched:
#              - LONG
#              - 1_SHORT
#              - 2_SHORT
#
#    IV. Launch cron job
#         Since no jobs (for a given configuration) are in 
#         the batch queue, this cron will launch the assessment.ksh 
#         script
#
##########################################################
#set -xv
##########################################################
##### Begin Users modifications
##########################################################
# IODIR     : directory where to get ORCA2_LIM_nemo_v3.1.tar 
##########################################################
IODIR=/u/rech/eee/reee512/NEMO/
TARGET=sx8brodie
##########################################################
##### End   Users modifications
##########################################################

CONF=$1
MTIME=$2
JOBTOL=$3
OWNEXP=$4
PRC=$5
REFTAG=$6
USE_IOMPUT=${7:-false}
USE_IOSERVER=${8:-false}
NB_PROCS_IOSERVER=${9:-0}
[ "${USE_IOMPUT}" = "false" ] && USE_IOSERVER=false
[ "${USE_IOSERVER}" = "false" ] && NB_PROCS_IOSERVER=0
RUN=$(basename `pwd`)
[ "${RUN}" = "mon" ] && PRC=1
if [[ ( "${TARGET}" = "sx8brodie" ) || ( "${TARGET}" = "sx8mercure" ) ]]
then
    if [ $(( ${PRC} + ${NB_PROCS_IOSERVER} )) -gt 8 ]
    then
	echo NVTK is not made to use more than 1 node on SX8 machines...
	echo please, reduce the number of cpu used by NEMO or/and the ioserver
	exit 1
    fi
fi
# ---------------------------------------------------------------
#  I. Take into account target's specificities 
#     Select appropriate target lines in Job_${CONF}.ksh
# ---------------------------------------------------------------
if   [ ${TARGET} = 'sx8brodie' ]; then
    W_XX='#-T- sx8brodie'
    LAUN='qsub '
    LLJOBS=qstat
    LSUB="mpirun -np ${PRC}"
    LPERF=
    LJTIM='01:20:00'
    LJTIMJ='01:40:00'
    CMDGET=mfget
    SYMBOL=B_
elif [ ${TARGET} = 'sx8mercure' ]; then
    W_XX='#-T- sx8mercure'
    LAUN='qsub '
    LLJOBS=qstat
    LSUB="mpirun -np ${PRC}"
    LPERF=
    LJTIM='01:20:00'
    LJTIMJ='01:40:00'
    CMDGET=cp
    SYMBOL=M_
elif [ ${TARGET} = 'aix6' ]; then
    W_XX='#-T- aix6'
    LAUN='llsubmit '
    LLJOBS=Qstat
    LSUB=
    LPERF="poe hpmcount -o perfs_mpi.txt" 
    LJTIM='01:10:00'
    LJTIMJ=
    CMDGET=mfget
    SYMBOL=V_
elif [ ${TARGET} = 'osxxlf' ]; then
    W_XX='#-T- osxxlf'
    LAUN='./'
    LLJOBS=
    LSUB="mpirun -np ${PRC}"
    LPERF=
    LJTIM=
    LJTIMJ=
    CMDGET=cp
    SYMBOL=D_
else
    echo '	#######################################'
    echo ' 	You must give a target name :'
    echo ' 	sx8brodie, aix6 or add one'
    echo ' 	Have a look in lance_batch.ksh script'
    echo '	#######################################'
    stop
fi

sed -e "s%^$W_XX *%%" job_${CONF}.ksh | grep -v '^#-T- ' > job_${CONF}.$$

# --------------------------------------------------------------
# II. Build jobs 
#     Adapt header (batch jobs) and concatanate it with core job
# --------------------------------------------------------------
if [ ${CONF} == 'ORCA2_LIM' ] ; then
    CONFS=ORCA2
elif [ ${CONF} == 'ORCA2_LIM3' ] ; then
    CONFS=OR2L3
elif [ ${CONF} == 'GYRE' ] ; then
    CONFS=GYREO
elif [ ${CONF} == 'GYRE_LOBSTER' ] ; then
    CONFS=GYREL
elif [ ${CONF} == 'ORCA2_LIM_PISCES' ] ; then
    CONFS=OR2LP
elif [ ${CONF} == 'ORCA2_OFF_PISCES' ] ; then
    CONFS=OR2OP
elif [ ${CONF} == 'ZAGRIF' ] ; then
    CONFS=ZAGRI
else
    CONFS=${CONF}
fi

case ${TARGET} in
    'osxxlf' )
	cp jhd_${TARGET}_${RUN} jhd_${TARGET}_${RUN}_tmp  ;;
    'aix6' )
	CAR_JNAM=$( grep '\# @ job_name' jhd_${TARGET}_${RUN} )
	CAR_JMPI=$( grep '\# @ total_tasks' jhd_${TARGET}_${RUN} )
	sed -e "s%$CAR_JNAM%\# @ job_name = ${RUN}${CONFS}%" \
	    -e "s%$CAR_JMPI%\# @ total_tasks = $(( ${PRC} + ${NB_PROCS_IOSERVER} ))%" \
	    jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp  ;;
    'sx8brodie' )
	CAR_JNAM=$( grep '\#PBS -N ' jhd_${TARGET}_${RUN} )
	CAR_JNAMO=$( grep '\#PBS -o ' jhd_${TARGET}_${RUN} )
	CAR_JNAME=$( grep '\#PBS -e ' jhd_${TARGET}_${RUN} )
	if [ ${RUN} = 'mpi' -o ${RUN} = 'omp' ] ; then
	    CAR_JMPI=$( grep '\#PBS -l cpunum_job' jhd_${TARGET}_${RUN} )
	    sed -e "s%$CAR_JNAM%\#PBS -N  ${RUN}${CONFS}%" \
		-e "s%$CAR_JNAMO%\#PBS -o  ${RUN}${CONFS}_std%" \
		-e "s%$CAR_JNAME%\#PBS -e  ${RUN}${CONFS}_err%" \
		-e "s%$CAR_JMPI%\#PBS -l cpunum_job=$(( ${PRC} + ${NB_PROCS_IOSERVER} ))%" \
		jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp
	else
	    sed -e "s%$CAR_JNAM%\#PBS -N  ${RUN}${CONFS}%" \
		-e "s%$CAR_JNAMO%\#PBS -o  ${RUN}${CONFS}_std%" \
		-e "s%$CAR_JNAME%\#PBS -e  ${RUN}${CONFS}_err%" \
		jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp
	fi ;;
    'sx8mercure' )
	CAR_JNAM=$( grep '\#PBS -N ' jhd_${TARGET}_${RUN} )
	CAR_JNAMO=$( grep '\#PBS -o ' jhd_${TARGET}_${RUN} )
	CAR_JNAME=$( grep '\#PBS -e ' jhd_${TARGET}_${RUN} )
	if [ ${RUN} = 'mpi' -o ${RUN} = 'omp' ] ; then
	    CAR_JMPI=$( grep '\#PBS -l cpunum_job' jhd_${TARGET}_${RUN} )
	    sed -e "s%$CAR_JNAM%\#PBS -N  ${RUN}${CONFS}%" \
		-e "s%$CAR_JNAMO%\#PBS -o  ${RUN}${CONFS}_std%" \
		-e "s%$CAR_JNAME%\#PBS -e  ${RUN}${CONFS}_err%" \
		-e "s%$CAR_JMPI%\#PBS -l cpunum_job=$(( ${PRC} + ${NB_PROCS_IOSERVER} ))%" \
		jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp
	else
	    sed -e "s%$CAR_JNAM%\#PBS -N  ${RUN}${CONFS}%" \
		-e "s%$CAR_JNAMO%\#PBS -o  ${RUN}${CONFS}_std%" \
		-e "s%$CAR_JNAME%\#PBS -e  ${RUN}${CONFS}_err%" \
		jhd_${TARGET}_${RUN} > jhd_${TARGET}_${RUN}_tmp
	fi ;;
esac

cat jhd_${TARGET}_${RUN}_tmp job_${CONF}.$$ > job_tmp_${RUN}.ksh
rm -f job_${CONF}.$$ jhd_${TARGET}_${RUN}_tmp

# -----------------------------------------------------------------------------
# III. Launch jobs
#      The 3 following jobs are created for the run [type]=mon,mpi,omp :
#      - "job_[type]_long.ksh"      ---> long run
#      - "job_[type]_1_short.ksh"   ---> stream 1 run
#      - "job_[type]_2_short.ksh"   ---> stream 2 run
#      - "job_[type]_gtime.ksh"     ---> one year run
#      The first 2 as the last one are immediatly launched whereas "job_[type]_2_short.ksh"
#      is launched only when the "job_[type]_1_short.ksh" short run is finished
# -----------------------------------------------------------------------------
jobnam[0]=long ; jobnam[1]=1_short ; jobnam[2]=2_short ; jobnam[3]=gtime 
dirjob[0]=LONG ; dirjob[1]=1_SHORT ; dirjob[2]=2_SHORT ; dirjob[3]=GTIME 

case ${JOBTOL} in
    'all' )
	if [ ${MTIME} = 'timing' ] ; then 
	    ind=0 
	    iloop=4
	else
	    ind=0 
	    iloop=3
	fi 
	;;
    'long' )
	ind=0 
	iloop=1 ;;
    'short' )
	ind=1 
	iloop=3 ;;
    'gtime' )
	ind=3 
	iloop=4 ;;
esac

chwd () { 
    word=$( grep "^ *${1}" job_tmp_${RUN}.ksh | cut -f 1 -d " " )
    if [ "${word}" != "" ] 
	then
	sed -e "s@${word}@${1%=*}=\"${2}\"@" job_tmp_${RUN}.ksh > tmp$$
	mv tmp$$ job_tmp_${RUN}.ksh
    fi
}

chwd "RUN=xxx"              "${RUN}"
chwd "NB_PROCS=xxx"          ${PRC}
chwd "LAUNCH=xxx"           "${LAUN}"
chwd "LEXE=xxx"             "${LSUB}"
chwd "CPERF=xxx"            "${LPERF}"
chwd "D_INPUT=xxx"          "${IODIR}"
chwd "D_GET=xxx"            "${CMDGET}"
chwd "REF_TAG=xxx"          "${REFTAG}" 
chwd "MYO_EXP=xxx"          "${OWNEXP}"
chwd "TSYMB=xxx"            "${SYMBOL}"
chwd "USE_IOMPUT=xxx"        ${USE_IOMPUT}
chwd "USE_IOSERVER=xxx"      ${USE_IOSERVER}
chwd "NB_PROCS_IOSERVER=xxx" ${NB_PROCS_IOSERVER}

while [ $ind -lt ${iloop} ]
  do
  
  chwd "type=" "${jobnam[$ind]}"  
  cp job_tmp_${RUN}.ksh job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh
  
  if [ ${jobnam[$ind]} = 'gtime' ] ; then
      
      case ${TARGET} in
	  'aix6' )
	      CAR_JTIM=$( grep '\# @ wall_clock_limit' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh )
	      sed -e "s%$CAR_JTIM%\# @ wall_clock_limit = ${LJTIM}%" \
		  job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh > job_${RUN}_${jobnam[$ind]}.ksh_tmp  ;;
	  'sx8brodie' )
	      CAR_JTIMP=$( grep '\#PBS -l cputim_prc' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh )
	      CAR_JTIMJ=$( grep '\#PBS -l cputim_job' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh )
	      sed -e "s%$CAR_JTIMP%\#PBS -l cputim_prc=${LJTIM}%" \
		  -e "s%$CAR_JTIMJ%\#PBS -l cputim_job=${LJTIMJ}%" \
		  job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh > job_${RUN}_${jobnam[$ind]}.ksh_tmp  ;;
	  'sx8mercure' )
	      CAR_JTIMP=$( grep '\#PBS -l cputim_prc' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh )
	      CAR_JTIMJ=$( grep '\#PBS -l cputim_job' job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh )
	      sed -e "s%$CAR_JTIMP%\#PBS -l cputim_prc=${LJTIM}%" \
		  -e "s%$CAR_JTIMJ%\#PBS -l cputim_job=${LJTIMJ}%" \
		  job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh > job_${RUN}_${jobnam[$ind]}.ksh_tmp  ;;
      esac
      mv job_${RUN}_${jobnam[$ind]}.ksh_tmp job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh
      
  fi
  
  chmod 750 job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh
  
  if [ ! -d ${dirjob[$ind]}/$OWNEXP ] ;  then
      mkdir ${dirjob[$ind]}/$OWNEXP
  fi
  mv job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh ${dirjob[$ind]}/${OWNEXP}/.
  
  if [ ${jobnam[$ind]} != '2_short' ] ; then
      cd ${dirjob[$ind]}/${OWNEXP}
      ${LAUN}job_${RUN}_${jobnam[$ind]}_${OWNEXP}.ksh &
      cd ../..
  fi
  
  let ind=$ind+1
done
rm -f job_tmp_${RUN}.ksh

# List all jobs submitted
${LLJOBS}

# -----------------------------------------------------------------------------
# IV. Launch cron to be able to launch the assessment step
# -----------------------------------------------------------------------------
if [ ${RUN} == 'mon' ] ; then
    cd ..
    
    CAR_LJOB_tmp=$( grep 'LJOBS=' cron_jobs.ksh )
    CAR_LJOB=`echo $CAR_LJOB_tmp | cut -f 1 -d " " `
    
    sed    -e "s%$CAR_LJOB%LJOBS=${LLJOBS}%" cron_jobs.ksh > cron_jobs.$$
    mv cron_jobs.$$ cron_jobs.ksh &&  rm -rf cron_jobs.$$
    chmod 744 cron_jobs.ksh 
    
    if [ ${TARGET} != 'osxxlf' ]; then
        ./cron_jobs.ksh ${OWNEXP} ${CONFS} &
    fi
fi