#!/bin/bash # # BENCH launching scripts for beaufix, Meteo-France # and curie, TGCC # To be modified for other machines # set -u #set -vx # cores=$1 ncore_node=$2 resolution=$3 dateref=$4 machine=$( hostname | sed -e "s/[0-9]*//g" ) # # number of processes for each executable nproc_exe1=$( echo $cores | bc ) nproc=$nproc_exe1 nnode=$(( ( $nproc + $ncore_node - 1 ) / $ncore_node )) nproc5=$( printf "%05d\n" ${nproc_exe1} ) case ${resolution} in "1") if [ $nproc_exe1 -lt 50 ] then timejob=3600 elif [ $nproc_exe1 -lt 100 ] then timejob=1800 elif [ $nproc_exe1 -lt 200 ] then timejob=900 else timejob=600 fi ;; "025") if [ $nproc_exe1 -lt 50 ] then timejob=15000 elif [ $nproc_exe1 -lt 100 ] then timejob=7000 elif [ $nproc_exe1 -lt 200 ] then timejob=3600 elif [ $nproc_exe1 -lt 400 ] then timejob=2000 elif [ $nproc_exe1 -lt 800 ] then timejob=1000 else timejob=600 fi ;; "12") if [ $nproc_exe1 -lt 200 ] then timejob=30000 elif [ $nproc_exe1 -lt 400 ] then timejob=15000 elif [ $nproc_exe1 -lt 800 ] then timejob=20000 elif [ $nproc_exe1 -lt 1600 ] then timejob=15000 elif [ $nproc_exe1 -lt 3200 ] then timejob=7500 elif [ $nproc_exe1 -lt 10000 ] then timejob=5000 elif [ $nproc_exe1 -lt 20000 ] then timejob=2500 else timejob=1200 fi ;; esac ###################################################################### ### beaufixlogin ###################################################################### if [ "$machine" == "beaufixlogin" ] then cat > Log/run_bench << EOF #!/bin/bash #SBATCH --time=00:1:00 #SBATCH -p normal64 # partition/queue #SBATCH --job-name=bench # job name #SBATCH -N $nnode # number of nodes #SBATCH -n $nproc # number of procs #SBATCH -o /scratch/work/cglo315/ESIWACE/dev_r9759_HPC09_ESIWACE/tests/LBENCH_RN/EXP00/Log/job.out%j #SBATCH -e /scratch/work/cglo315/ESIWACE/dev_r9759_HPC09_ESIWACE/tests/LBENCH_RN/EXP00/Log/job.out%j #SBATCH --exclusive module unload intelmpi intel grib_api module load intel/16.1.150 intelmpi/5.1.2.150 itac=0 xpmpi=0 if [ \$xpmpi == 1 ]; then module load bullxde module load xPMPI/1.1_intelmpi fi [ \$itac == 1 ] && module load itac/2017.2.028 set -vx cd \${TMPDIR} cp /scratch/work/cglo315/ESIWACE/dev_r9759_HPC09_ESIWACE/tests/LBENCH_RN/EXP00/* . # Best decompositions BENCH-1 jpni=${cores/\**/} jpnj=${cores/?*\*/} sed -e "s/jpni *=.*/jpni = \${jpni}/" -e "s/jpnj *=.*/jpnj = \${jpnj}/" namelist_cfg_orca${resolution}_like > namelist_cfg export OMP_NUM_THREADS=1 ulimit -s unlimited # if [ \$itac == 1 ]; then source /opt/softs/intel/2017/update_1/itac_latest/bin/itacvars.sh time mpirun -ordered-output -prepend-rank -trace -np $nproc_exe1 ./nemo > jobout 2>joberr else time mpirun -ordered-output -prepend-rank -np $nproc_exe1 ./nemo > jobout_${resolution}_${nproc5} fi /opt/softs/bin/ja if [ \$xpmpi == 1 ]; then module unload xPMPI/1.1_intelmpi module unload bullxde fi # EOF ### 4. Execute the model echo 'Submitting the job to queue using sbatch' sbatch Log/run_bench squeue -u cglo315 echo 'is executed or submitted to queue.' fi ###################################################################### ### curie or irene ###################################################################### if [[ ( "$machine" == "curie" ) || ( "$machine" == "irene" ) ]] then [ "$machine" == "curie" ] && queuename=standard || queuename=skylake EXPjob=../EXP_${resolution}_${nproc5}_${dateref} mkdir -p ${EXPjob} cd ${EXPjob} jobname=jobbench cat > $jobname << EOF #!/bin/bash #MSUB -r bench${nproc5} #MSUB -n ${nproc_exe1} #MSUB -T $timejob #MSUB -e bench_${resolution}_${nproc5}_%I.eo #MSUB -o bench_${resolution}_${nproc5}_%I.eo #MSUB -j oe #MSUB -x #MSUB -q ${queuename} #MSUB -A gen6895 #========================================== set -u # cd \${BRIDGE_MSUB_PWD} for ff in \${BRIDGE_MSUB_PWD}/../EXPREF/namelist_*cfg \${BRIDGE_MSUB_PWD}/../EXPREF/namelist_*ref \${BRIDGE_MSUB_PWD}/../BLD/bin/nemo.exe do cp \$ff . done jpni=${cores/\**/} jpnj=${cores/?*\*/} sed -e "s/jpni *=.*/jpni = \${jpni}/" \ -e "s/jpnj *=.*/jpnj = \${jpnj}/"\ -e "s/ln_timing *= *.false./ln_timing = .true./" \ \${BRIDGE_MSUB_PWD}/../EXPREF/namelist_cfg_orca${resolution}_like > namelist_cfg time ccc_mprun -n \${BRIDGE_MSUB_NPROC} ./nemo.exe > jobout_${resolution}_${nproc5}_${dateref} 2>&1 EOF ccc_msub $jobname fi ###################################################################### ### Jean-Zay ###################################################################### if [ "$machine" == "jean-zay" ] then hh=$( printf "%02d\n" $(( ${timejob} / 3600 )) ) mm=$( printf "%02d\n" $(( ( ${timejob} % 3600 ) / 60 )) ) ss=$( printf "%02d\n" $(( ( ${timejob} % 3600 ) % 60 )) ) EXPjob=../EXP_${resolution}_${nproc5}_${dateref} mkdir -p ${EXPjob} cd ${EXPjob} jobname=jobbench cat > $jobname << EOF #!/bin/bash #SBATCH --job-name=Seq # nom du job #SBATCH --partition=cpu_gct3 # demande d'allocation sur la partition CPU #SBATCH --nodes=${nnode} # nombre de noeuds #SBATCH --ntasks-per-node=${ncore_node} # nombre de taches MPI par noeud #SBATCH --ntasks-per-core=1 # 1 processus MPI par coeur physique (pas d'hyperthreading) #SBATCH --time=${hh}:${mm}:${ss} # temps d execution maximum demande (HH:MM:SS) #SBATCH --output=bench_${resolution}_${nproc5}_%j.eo # nom du fichier de sortie #SBATCH --error=bench_${resolution}_${nproc5}_%j.eo # nom du fichier d'erreur (ici en commun avec la sortie) #========================================== set -u #set -xv # #cd \${SLURM_SUBMIT_DIR} cd \${JOBSCRATCH} pwd for ff in \${SLURM_SUBMIT_DIR}/../EXPREF/namelist_*cfg \${SLURM_SUBMIT_DIR}/../EXPREF/namelist_*ref \${SLURM_SUBMIT_DIR}/../BLD/bin/nemo.exe do cp \$ff . done jpni=${cores/\**/} jpnj=${cores/?*\*/} sed -e "s/jpni *=.*/jpni = \${jpni}/" \ -e "s/jpnj *=.*/jpnj = \${jpnj}/" \ -e "s/ln_timing *= *.false./ln_timing = .true./" \ \${SLURM_SUBMIT_DIR}/../EXPREF/namelist_cfg_orca${resolution}_like > namelist_cfg ls -l echo echo echo " =========== start the model ===========" echo echo time srun --mpi=pmi2 --cpu-bind=cores -K1 -n ${nproc} ./nemo.exe > jobout_${resolution}_${nproc5}_${dateref} 2>&1 ls -l if [ "\$( pwd )" != "\${SLURM_SUBMIT_DIR}" ] then rsync -av namelist_cfg time.step ocean.output jobout_${resolution}_${nproc5}_${dateref} communication_report.txt layout.dat timing.output output.namelist* \${SLURM_SUBMIT_DIR} fi EOF sbatch $jobname fi