source: NEMO/branches/2019/dev_r10984_HPC-13_IRRMANN_BDY_optimization/tests/BENCH/EXPREF/submit_bench @ 11364

#!/bin/bash
#  
#  BENCH launching scripts for beaufix, Meteo-France
#                          and curie, TGCC
#  To be modified for other machines
#
set -u
#set -vx
#
cores=$1
ncore_node=$2
resolution=$3
dateref=$4
machine=$( hostname | sed -e "s/[0-9]*//g" )
#
# number of processes for each executable
nproc_exe1=$( echo $cores | bc )
nproc=$nproc_exe1
nnode=$(( ( $nproc + $ncore_node - 1 )  / $ncore_node ))

nproc5=$( printf "%05d\n" ${nproc_exe1} )

case ${resolution} in
    "1")
   if [ $nproc_exe1 -lt 50 ]
   then
       timejob=3600
   elif [ $nproc_exe1 -lt 100 ]
   then
       timejob=1800
   elif [ $nproc_exe1 -lt 200 ]
   then
       timejob=900
   else
       timejob=600
   fi
   ;;
    "025")
   if [ $nproc_exe1 -lt 50 ]
   then
       timejob=15000
   elif [ $nproc_exe1 -lt 100 ]
   then
       timejob=7000
   elif [ $nproc_exe1 -lt 200 ]
   then
       timejob=3600
   elif [ $nproc_exe1 -lt 400 ]
   then
       timejob=2000
   elif [ $nproc_exe1 -lt 800 ]
   then
       timejob=1000
   else
       timejob=600
   fi
   ;;
    "12")
   if [ $nproc_exe1 -lt 200 ]
   then
       timejob=30000
   elif [ $nproc_exe1 -lt 400 ]
   then
       timejob=15000
   elif [ $nproc_exe1 -lt 800 ]
   then
       timejob=8000
   elif [ $nproc_exe1 -lt 1600 ]
   then
       timejob=5000
   elif [ $nproc_exe1 -lt 3200 ]
   then
       timejob=2500
   else
       timejob=1200
   fi
   ;;
esac


######################################################################
### beaufixlogin
######################################################################

if [ "$machine" == "beaufixlogin" ]
then
    
cat > Log/run_bench << EOF
#!/bin/bash
#SBATCH --time=00:1:00
#SBATCH -p  normal64         # partition/queue
#SBATCH --job-name=bench      # job name
#SBATCH -N $nnode            # number of nodes
#SBATCH -n $nproc            # number of procs
#SBATCH -o /scratch/work/cglo315/ESIWACE/dev_r9759_HPC09_ESIWACE/tests/LBENCH_RN/EXP00/Log/job.out%j
#SBATCH -e /scratch/work/cglo315/ESIWACE/dev_r9759_HPC09_ESIWACE/tests/LBENCH_RN/EXP00/Log/job.out%j
#SBATCH --exclusive

module unload intelmpi intel grib_api
module load intel/16.1.150 intelmpi/5.1.2.150

itac=0
xpmpi=0

if [ \$xpmpi == 1 ]; then
  module load bullxde
  module load xPMPI/1.1_intelmpi
fi

[ \$itac == 1 ] && module load itac/2017.2.028

set -vx

cd \${TMPDIR}
cp /scratch/work/cglo315/ESIWACE/dev_r9759_HPC09_ESIWACE/tests/LBENCH_RN/EXP00/* .

# Best decompositions BENCH-1
jpni=${cores/\**/}
jpnj=${cores/?*\*/}

sed -e "s/jpni *=.*/jpni = \${jpni}/" -e "s/jpnj *=.*/jpnj = \${jpnj}/" namelist_cfg_orca${resolution}_like > namelist_cfg

export OMP_NUM_THREADS=1
ulimit -s unlimited
#
if [ \$itac == 1 ]; then
  source /opt/softs/intel/2017/update_1/itac_latest/bin/itacvars.sh
  time mpirun -ordered-output -prepend-rank -trace -np $nproc_exe1 ./nemo > jobout 2>joberr
else
  time mpirun -ordered-output -prepend-rank -np $nproc_exe1 ./nemo > jobout_${resolution}_${nproc5}
fi
/opt/softs/bin/ja

if [ \$xpmpi == 1 ]; then
  module unload xPMPI/1.1_intelmpi
  module unload bullxde
fi
#
EOF


### 4. Execute the model

    echo 'Submitting the job to queue using sbatch'
    sbatch Log/run_bench
    squeue -u cglo315

echo 'is executed or submitted to queue.'


fi

######################################################################
### curie or irene
######################################################################

if [[ ( "$machine" == "curie" ) || ( "$machine" == "irene" ) ]]
then

    [ "$machine" == "curie" ] && queuename=standard || queuename=skylake
    
    EXPjob=../EXP_${resolution}_${nproc5}_${dateref}
    mkdir -p ${EXPjob}
    cd ${EXPjob}
    jobname=jobbench
    cat > $jobname << EOF
#!/bin/bash
#MSUB -r bench${nproc5}
#MSUB -n ${nproc_exe1}
#MSUB -T $timejob
#MSUB -e bench_${resolution}_${nproc5}_%I.eo
#MSUB -o bench_${resolution}_${nproc5}_%I.eo
#MSUB -j oe
#MSUB -x
#MSUB -q ${queuename}
#MSUB -A gen6895
#==========================================
set -u
#

cd \${BRIDGE_MSUB_PWD}

for ff in \${BRIDGE_MSUB_PWD}/../EXPREF/namelist_*cfg \${BRIDGE_MSUB_PWD}/../EXPREF/namelist_*ref \${BRIDGE_MSUB_PWD}/../BLD/bin/nemo.exe
do
    cp \$ff .
done

jpni=${cores/\**/}
jpnj=${cores/?*\*/}

sed -e "s/jpni *=.*/jpni = \${jpni}/" \
    -e "s/jpnj *=.*/jpnj = \${jpnj}/"\
    -e "s/ln_timing *= *.false./ln_timing   =  .true./" \
     \${BRIDGE_MSUB_PWD}/../EXPREF/namelist_cfg_orca${resolution}_like > namelist_cfg

time ccc_mprun -n \${BRIDGE_MSUB_NPROC} ./nemo.exe > jobout_${resolution}_${nproc5}_${dateref} 2>&1

EOF

    ccc_msub $jobname

fi

######################################################################
### Jean-Zay
######################################################################

if [ "$machine" == "jean-zay" ]
then
    hh=$( printf "%02d\n" $(( ${timejob} / 3600 )) )
    mm=$( printf "%02d\n" $(( ( ${timejob} % 3600 ) / 60 )) )
    ss=$( printf "%02d\n" $(( ( ${timejob} % 3600 ) % 60 )) )

    EXPjob=../EXP_${resolution}_${nproc5}_${dateref}
    mkdir -p ${EXPjob}
    cd ${EXPjob}
    jobname=jobbench
    cat > $jobname << EOF
#!/bin/bash
#SBATCH --job-name=Seq        # nom du job
#SBATCH --partition=cpu_dev   # demande d'allocation sur la partition CPU
#SBATCH --nodes=${nnode}             # nombre de noeuds
#SBATCH --ntasks-per-node=${ncore_node}  # nombre de taches MPI par noeud
#SBATCH --ntasks-per-core=1        # 1 processus MPI par coeur physique (pas d'hyperthreading)
#SBATCH --time=${hh}:${mm}:${ss}       # temps d execution maximum demande (HH:MM:SS)
#SBATCH --output=bench_${resolution}_${nproc5}_%j.eo        # nom du fichier de sortie
#SBATCH --error=bench_${resolution}_${nproc5}_%j.eo         # nom du fichier d'erreur (ici en commun avec la sortie)
#==========================================
set -u
#set -xv
#
#cd \${SLURM_SUBMIT_DIR}
cd \${JOBSCRATCH}
pwd

for ff in \${SLURM_SUBMIT_DIR}/../EXPREF/namelist_*cfg \${SLURM_SUBMIT_DIR}/../EXPREF/namelist_*ref \${SLURM_SUBMIT_DIR}/../BLD/bin/nemo.exe
do
    cp \$ff .
done

jpni=${cores/\**/}
jpnj=${cores/?*\*/}

sed -e "s/jpni *=.*/jpni = \${jpni}/" \
    -e "s/jpnj *=.*/jpnj = \${jpnj}/" \
    -e "s/ln_timing *= *.false./ln_timing   =  .true./" \
    \${SLURM_SUBMIT_DIR}/../EXPREF/namelist_cfg_orca${resolution}_like > namelist_cfg

ls -l

echo 
echo 
echo " =========== start the model ==========="
echo 
echo 

time srun --mpi=pmi2 --cpu-bind=cores -K1 -n ${nproc} ./nemo.exe > jobout_${resolution}_${nproc5}_${dateref} 2>&1

ls -l

rsync -av namelist_cfg time.step ocean.output out_run.txt communication_report.txt layout.dat timing.output output.namelist* $( pwd )

EOF

    sbatch $jobname

fi