#!/bin/bash

# This script provides site-specific functions
# called by the site-independent scripts
# create_runs.sh and submit.sh

# NB : we must use the queue 'normal' because there seems to be a limit
# on the number of jobs that can be submitted to queue 'test'

#----------------------------- split_XXX ----------------------------

function split_mpi_32() {
    setvar nbp 33 nsplit_i 2 nsplit_j 4
}
function split_serial_40() {
    setvar nbp 41 nsplit_i 2 nsplit_j 2
}
function split_mpi_40() {
    setvar nbp 41 nsplit_i 2 nsplit_j 3
}
function split_mpi_80() {
    setvar nbp 81 nsplit_i 4 nsplit_j 3
}
function split_mpi_omp_40() {
    setvar nbp 41 nsplit_i 2 nsplit_j 2 omp_level_size 4
}

#----------------- job submission ------------------

function submit_job_X64_CURIE() # $1=script $* = SLURM OPTIONS
{
    TMP=$(mktemp)
    ccc_msub $1 > $TMP
    cat $TMP 1>&2
    cat $TMP | awk '{print $NF }'
    rm -f $TMP
}

#------------------------------ job_XXX -----------------------------

# Serial jobs
function job_serial() { # EXP_NAME
    job_gen $1 1 1
}
function job_serial_40() { # EXP_NAME
    job_serial $1
}

# MPI jobs
function job_mpi() { #EXP_NAME NB_MPI
    job_gen $1 $2 1
}
function job_mpi_32() { # EXP_NAME
    job_mpi $1 80
}
function job_mpi_40() { # EXP_NAME
    job_mpi $1 60
}
function job_mpi_80() { # EXP_NAME
    job_mpi $1 120
}

# MPI-OMP jobs
function job_mpi_omp() { #EXP_NAME NB_MPI NB_OMP
    job_gen $1 $2 $3
}
function job_mpi_omp_40() { # EXP_NAME
    job_mpi_omp $1 40 4
}

# Generic
function job_gen() { # EXP_NAME MPI_TASKS OMP_TASKS MPIRUN
MPI=$2
OMP=$3
MPI_OMP=$((MPI*OMP))
NODES=$(((MPI_OMP+63)/64))
VMEM=$((2*MPI_OMP))
    cat <<EOF
#!/bin/bash
#PBS -N $1
#PBS -q std
#PBS -n
#PBS -l nodes=$NODES:ppn=64
#PBS -l walltime=00:15:00
#PBS -l mem=${VMEM}gb
#PBS -l vmem=${VMEM}gb

#####comment:PBS -l mem=31922327552,ncpus=60
#Jobs start in the HOME directory, cd to submitted directory
cd "\$PBS_O_WORKDIR"

############################################################
NCPU=\$(wc -l < \$PBS_NODEFILE)
awk "NR % $OMP == 0" \$PBS_NODEFILE | head -n $MPI > nodelist.txt

echo ------------------JOB SUMMARY--------------------------
echo 'PBS OUT: Job allocated on: '\${NCPU}' cpu(s)'
echo 'PBS OUT: Job running on the following node(s): '
cat nodelist.txt
echo -------------------------------------------------------
echo PBS OUT: Job running on: \$PBS_O_HOST
echo PBS OUT: Originating queue: \$PBS_O_QUEUE
echo PBS OUT: Executing queue: \$PBS_QUEUE
echo PBS OUT: Work directory: \$PBS_O_WORKDIR
echo PBS OUT: Execution mode: \$PBS_ENVIRONMENT
echo PBS OUT: Job identifier: \$PBS_JOBID
echo PBS OUT: Job name: \$PBS_JOBNAME
echo PBS OUT: Node file: \$PBS_NODEFILE
echo PBS OUT: Current home directory: \$PBS_O_HOME
echo PBS OUT: PATH= \$PBS_O_PATH
echo -------------------------------------------------------


export OMP_NUM_THREADS=$3
export OMP_STACKSIZE=128M
ulimit -s unlimited

$(cat $ROOT/DYNAMICO/arch/arch-ifort_CICLAD.env)

rm -rf gcm.log logs *.nc netcdf
mpirun -np $MPI -cpus-per-proc \$OMP_NUM_THREADS -bycore -bind-to-core -report-bindings ./gcm.exe > gcm.log 2>&1
$4 >> gcm.log
date >> gcm.log

mkdir -p netcdf
cp gcm.log *.def netcdf
mv *.nc netcdf

mkdir -p logs
cp *.xml logs
cp -pr $(readlink -e gcm.exe) logs
cp -pr $(readlink -e build/src) logs
mv xios_client_*.err xios_client_*.out gcm.log logs

EOF
}

#------------------------------ post-processing job -----------------------------

function job_post_X64_CURIE() # LIST
{
    DEPLIST=afterany
    for ID in $* ; do
	DEPLIST="$DEPLIST:$ID"
    done
    cat <<EOF
#!/bin/bash
## Request name
#MSUB -r testing
#MSUB -q standard 
#MSUB -A gen7548
#MSUB -E "--dependency=$DEPLIST"
## Number of tasks (=MPI processes) to use
#MSUB -n 1
## Elapsed time limit in seconds
#MSUB -T 600
## Quality of Service required (long [3 days], normal [1 day], test [30 min])
#MSUB -Q normal

export OMP_NUM_THREADS=1
# this script is submitted from $ROOT/logs
cd \${BRIDGE_MSUB_PWD}/..

$(cat $ROOT/DYNAMICO/arch/arch-X64_CURIE.env)
module load python

bash/post.sh
EOF
}