#!/bin/bash

# This script provides site-specific functions
# called by the site-independent scripts
# create_runs.sh and submit.sh

# NB : we must use the queue 'normal' because there seems to be a limit
# on the number of jobs that can be submitted to queue 'test'

#----------------------------- split_XXX ----------------------------

function split_serial_40() {
    setvar nbp 41 nsplit_i 2 nsplit_j 2
}
function split_mpi_40() {
    setvar nbp 41 nsplit_i 2 nsplit_j 2
}
function split_mpi_80() {
    setvar nbp 81 nsplit_i 4 nsplit_j 2
}
function split_mpi_omp_40() {
    setvar nbp 41 nsplit_i 2 nsplit_j 2 omp_level_size 4
}

#----------------- job submission ------------------

function submit_job_X64_CURIE() # $1=script $* = SLURM OPTIONS
{
    TMP=$(mktemp)
    ccc_msub $1 > $TMP
    cat $TMP 1>&2
    cat $TMP | awk '{print $NF }'
    rm -f $TMP
}

#------------------------------ job_XXX -----------------------------

# Serial jobs
function job_serial() { # EXP_NAME
    job_X64_CURIE $1 1 1 "./gcm.exe"
}
function job_serial_40() { # EXP_NAME
    job_serial $1
}

# MPI jobs
function job_mpi() { #EXP_NAME NB_MPI
    job_X64_CURIE $1 $2 1 "ccc_mprun ./gcm.exe"
}
function job_mpi_40() { # EXP_NAME
    job_mpi $1 40
}
function job_mpi_80() { # EXP_NAME
    job_mpi $1 80
}

# MPI-OMP jobs
function job_mpi_omp() { #EXP_NAME NB_MPI NB_OMP
    job_X64_CURIE $1 $2 $3 "ccc_mprun ./gcm.exe"
}
function job_mpi_omp_40() { # EXP_NAME
    job_mpi_omp $1 40 4
}

# Generic
function job_X64_CURIE() { # EXP_NAME MPI_TASKS OMP_TASKS MPIRUN
    cat <<EOF
#!/bin/bash
## Request name
#MSUB -r $1
#MSUB -q standard 
#MSUB -A gen7548
#MSUB -x
## Number of tasks (=MPI processes) to use
#MSUB -n $2
## Number of OpenMP threads
#MSUB -c $3
## Elapsed time limit in seconds
#MSUB -T 600
## Quality of Service required (long [3 days], normal [1 day], test [30 min])
#MSUB -Q normal

export OMP_NUM_THREADS=$3
export OMP_STACKSIZE=128M

cd \${BRIDGE_MSUB_PWD} 

$(cat $ROOT/DYNAMICO/arch/arch-X64_CURIE.env)

rm -rf gcm.log logs *.nc netcdf
date > gcm.log
ulimit -s unlimited
$4 >> gcm.log
date >> gcm.log

mkdir -p netcdf
cp gcm.log *.def netcdf
mv *.nc netcdf

mkdir -p logs
cp *.xml logs
mv xios_client_*.err xios_client_*.out gcm.log logs

EOF
}

#------------------------------ post-processing job -----------------------------

function job_post_X64_CURIE() # LIST
{
    DEPLIST=afterany
    for ID in $* ; do
	DEPLIST="$DEPLIST:$ID"
    done
    cat <<EOF
#!/bin/bash
## Request name
#MSUB -r testing
#MSUB -q standard 
#MSUB -A gen7548
#MSUB -E "--dependency=$DEPLIST"
## Number of tasks (=MPI processes) to use
#MSUB -n 1
## Elapsed time limit in seconds
#MSUB -T 600
## Quality of Service required (long [3 days], normal [1 day], test [30 min])
#MSUB -Q normal

export OMP_NUM_THREADS=1
# this script is submitted from $ROOT/logs
cd \${BRIDGE_MSUB_PWD}/..

$(cat $ROOT/DYNAMICO/arch/arch-X64_CURIE.env)
module load python

bash/post.sh
EOF
}