#!/bin/bash
#  
#  BENCH launching scripts for beaufix, Meteo-France
#  To be modified for other machines
#
set -u
set -vx
#
cores=$1
ncore_node=$2
resolution=$3
machine=$( hostname | sed -e "s/[0-9]*//g" )
######################################################################
#
# number of processes for each executable
    nproc_exe1=$( echo $cores | bc )
  (( nproc = $nproc_exe1 ))
(( nnode = $nproc / $ncore_node ))
[ $nnode -lt 1 ] && nnode=1

nproc5=$( printf "%05d\n" ${nproc_exe1} )

if [ "$machine" == "beaufix" ]
then
    
cat > Log/run_bench << EOF
#!/bin/bash
#SBATCH --time=00:1:00
#SBATCH -p  normal64         # partition/queue
#SBATCH --job-name=bench      # job name
#SBATCH -N $nnode            # number of nodes
#SBATCH -n $nproc            # number of procs
#SBATCH -o /scratch/work/cglo315/ESIWACE/dev_r9759_HPC09_ESIWACE/tests/LBENCH_RN/EXP00/Log/job.out%j
#SBATCH -e /scratch/work/cglo315/ESIWACE/dev_r9759_HPC09_ESIWACE/tests/LBENCH_RN/EXP00/Log/job.out%j
#SBATCH --exclusive

module unload intelmpi intel grib_api
module load intel/16.1.150 intelmpi/5.1.2.150

itac=0
xpmpi=0

if [ \$xpmpi == 1 ]; then
  module load bullxde
  module load xPMPI/1.1_intelmpi
fi

[ \$itac == 1 ] && module load itac/2017.2.028

set -vx

cd \${TMPDIR}
cp /scratch/work/cglo315/ESIWACE/dev_r9759_HPC09_ESIWACE/tests/LBENCH_RN/EXP00/* .

# Best decompositions BENCH-1
jpni=${cores/\**/}
jpni=${cores/?*\*/}

sed -e "s/jpni *=.*/jpni = \${jpni}/" -e "s/jpnj *=.*/jpnj = \${jpnj}/" namelist_cfg_orca${resolution}_like > namelist_cfg

export OMP_NUM_THREADS=1
ulimit -s unlimited
#
if [ \$itac == 1 ]; then
  source /opt/softs/intel/2017/update_1/itac_latest/bin/itacvars.sh
  time mpirun -ordered-output -prepend-rank -trace -np $nproc_exe1 ./nemo > jobout 2>joberr
else
  time mpirun -ordered-output -prepend-rank -np $nproc_exe1 ./nemo > jobout_${resolution}_${nproc5}
fi
/opt/softs/bin/ja

if [ \$xpmpi == 1 ]; then
  module unload xPMPI/1.1_intelmpi
  module unload bullxde
fi
#
EOF


######################################################################
###
### 4. Execute the model

    echo 'Submitting the job to queue using sbatch'
    sbatch Log/run_bench
    squeue -u cglo315

echo 'is executed or submitted to queue.'

######################################################################

fi

if [ "$machine" == "curie" ]
then

    if [ $nproc_exe1 -lt 256 ]
    then
	timejob=10000
    else
	timejob=1800
    fi
    
    jobname=$HOME/binrun/jobbench
    cat > $jobname << EOF
#!/bin/bash
#MSUB -r jobbench${nproc5}
#MSUB -n ${nproc_exe1}
#MSUB -T $timejob
#MSUB -e bench_${resolution}_${nproc5}_%I.eo
#MSUB -o bench_${resolution}_${nproc5}_%I.eo
#MSUB -j oe
#MSUB -x
#MSUB -q standard
#MSUB -A gen6895
#==========================================
set -u
#

cd \${BRIDGE_MSUB_PWD}/..

rsync -av --exclude="*eo" EXPREF/ EXP_${resolution}_${nproc5}/
rsync -av EXP00/nemo EXP_${resolution}_${nproc5}/nemo
cd EXP_${resolution}_${nproc5}/

jpni=${cores/\**/}
jpnj=${cores/?*\*/}

sed -e "s/jpni *=.*/jpni = \${jpni}/" -e "s/jpnj *=.*/jpnj = \${jpnj}/" namelist_cfg_orca${resolution}_like > namelist_cfg

time ccc_mprun -n \${BRIDGE_MSUB_NPROC} ./nemo > jobout_${resolution}_${nproc5}

EOF

    ccc_msub $jobname

fi