#!/bin/bash
############################################################
# Author : Italo Epicoco - CMCC
# Contact: italo.epicoco@unisalento.it
# 2014   : A.C. Coward added new namelist settings for GYRE configuration
#
# sette_xios.sh   : additional script of SET TEsts for XIOS within NEMO
# ----------------------------------------------------------------------
# NEMO/SETTE , NEMO Consortium (2014)
# Software governed by the CeCILL licence     (NEMOGCM/NEMO_CeCILL.txt)
# ----------------------------------------------------------------------
#
#############################################################
#set -x
set -o posix
#set -u
#set -e
# ===========
# DESCRIPTION
# ===========
#
# Variables to be checked by user:
#
# COMPILER          : name of compiler as defined in NEMOGCM/ARCH directory 
# BATCH_COMMAND_PAR :  name of the command for submitting parallel batch jobs
# BATCH_COMMAND_SEQ :  name of the command for submitting sequential batch jobs  
# INTERACT_FLAG     : flag to run in interactive mode "yes"
#                           to run in batch mode "no"
# MPIRUN_FLAG       : flag to run in parallel (MPI) "yes"
#                           to run in sequential mode (NB_PROC = 1) "no"
# USING_MPMD        : flag to control the use of stand-alone IO servers
#                      "true" to run in MPMD (detached) mode with stand-alone IO servers
#                      "false"  to run in SPMD (attached) mode without separate IO servers 
# NUM_XIOSERVERS    : number of stand-alone IO servers to employ
#                     set to zero if USING_MPMD="false"
#
# Principal script is sette_xios.sh, that calls 
#
#  makenemo  : to create successive exectuables in ${CONFIG_NAME}/BLD/bin/nemo.exe 
#              and links to opa in ${CONFIG_NAME}/EXP00)
#
#  param.cfg : sets and loads following directories:
#
#   FORCING_DIR         : is the directory for forcing files (tarfile)
#   INPUT_DIR           : is the directory for input files storing 
#   TMPDIR              : is the temporary directory (if needed)
#   NEMO_VALIDATION_DIR : is the validation directory
#
#   (NOTE: this file is the same for all configrations to be tested with sette)
#
#   all_functions.sh : loads functions used by sette (note: new functions can be added here)
#   set_namelist     : function declared in all_functions that sets namelist parameters 
#   post_test_tidyup : creates validation storage directory and copies required output files 
#                      (solver.stat and ocean.output) in it after execution of test.
#
#  VALIDATION tree is:
#
#   NEMO_VALIDATION_DIR/WCONFIG_NAME/WCOMPILER_NAME/TEST_NAME/REVISION_NUMBER(or DATE)
#
#  prepare_exe_dir.sh : defines and creates directory where the test is executed
#                       execution directory takes name of TEST_NAME defined for every test 
#                       in sette.sh. (each test in executed in its own directory)
#
#  prepare_job.sh     : to generate the script run_job.sh
#
#  fcm_job.sh         : run in batch (INTERACT_FLAG="no") or interactive (INTERACT_FLAG="yes")
#                        see sette.sh and BATCH_TEMPLATE directory
#
#  NOTE: jobs requiring initial or forcing data need to have an input_CONFIG.cfg in which 
#        can be found paths to the input tar file)
#  NOTE: if job is not launched for any reason you have the executable ready in ${EXE_DIR} 
#        directory
#  NOTE: the changed namelists are left in ${EXE_DIR} directory whereas original namelists 
#        remain in ${NEW_CONF}/EXP00
# 
#  NOTE: a log file, output.sette, is created in ${SETTE_DIR} with the echoes of 
#        executed commands
#
#  NOTE: if sette_xios.sh is stopped in output.sette there is written the last command 
#        executed by sette_xios.sh
#
# example use: ./sette_xios.sh 
#########################################################################################
#
# Compiler among those in NEMOGCM/ARCH
COMPILER=
export BATCH_COMMAND_PAR="llsubmit"
export BATCH_COMMAND_SEQ=$BATCH_COMMAND_PAR
export INTERACT_FLAG="no"
export MPIRUN_FLAG="yes"
#

# Directory to run the tests
SETTE_DIR=$(cd $(dirname "$0"); pwd)
MAIN_DIR=${SETTE_DIR%/SETTE}
CONFIG_DIR=${MAIN_DIR}/CONFIG
TOOLS_DIR=${MAIN_DIR}/TOOLS
COMPIL_DIR=${TOOLS_DIR}/COMPILE

CMP_NAM=${1:-$COMPILER}
#
#=================================================================================
# Copy job_batch_COMPILER file for specific compiler into job_batch_template
# Note this batch template needs to be capable of launching both SPMD and MPMD
# tasks with internal selection depending on the value of NUM_XIOSERVERS
# (0=SPMD; >0 = MPMD)
#=================================================================================
#
cd ${SETTE_DIR}
cp BATCH_TEMPLATE/batch-${COMPILER} job_batch_template || exit

for config in 1 2  

do

#==========================================================
# TESTS FOR XIOS USING GYRE CONFIGURATION AT LOW RESOLUTION
#==========================================================

if [ ${config} -eq 1 ] ;  then
    ## Test of XIOS configured in attached mode with multiple output files (one for each process). 
    ## $NPROC processes are used 

    NPROC=4
    jp_cfg=1
    jpni=2
    jpnj=2
    export NUM_XIOSERVERS=0
    export USING_MPMD="false"

    export TEST_NAME="ATTACHED_MULTIPLE"

    cd ${SETTE_DIR}
    . ../CONFIG/makenemo -m ${CMP_NAM} -n GYRE_XIOS_LR -r GYRE_XIOS -j 8 
    cd ${SETTE_DIR}
    . ./param.cfg
    . ./all_functions.sh
    . ./prepare_exe_dir.sh
    JOB_FILE=${EXE_DIR}/run_job.sh
    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
    cd ${EXE_DIR}  
    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_LR\"
    set_namelist namelist_cfg nn_it000 1
    set_namelist namelist_cfg nn_itend 120
    set_namelist namelist_cfg nn_stock 120
    set_namelist namelist_cfg ln_clobber .true.
    set_namelist namelist_cfg jp_cfg $jp_cfg
    set_namelist namelist_cfg jpidta $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjdta $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpiglo $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjglo $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpni $jpni
    set_namelist namelist_cfg jpnj $jpnj
    set_namelist namelist_cfg jpnij $NPROC

    set_xio_using_server iodef.xml $USING_MPMD
    set_xio_file_type    iodef.xml multiple_file

    cd ${SETTE_DIR}
    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}

    cd ${SETTE_DIR}
    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}


    ## Test of XIOS configured in attached mode with a single output file. 

    export TEST_NAME="ATTACHED_ONE"
    export NUM_XIOSERVERS=0
    export USING_MPMD="false"
    cd ${SETTE_DIR} 
    . ./prepare_exe_dir.sh
    JOB_FILE=${EXE_DIR}/run_job.sh
    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
    cd ${EXE_DIR}
    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_LR\"
    set_namelist namelist_cfg nn_it000 1
    set_namelist namelist_cfg nn_itend 120
    set_namelist namelist_cfg nn_stock 120
    set_namelist namelist_cfg ln_clobber .true.
    set_namelist namelist_cfg jp_cfg $jp_cfg
    set_namelist namelist_cfg jpidta $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjdta $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpiglo $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjglo $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpni $jpni
    set_namelist namelist_cfg jpnj $jpnj
    set_namelist namelist_cfg jpnij $NPROC

    set_xio_using_server iodef.xml $USING_MPMD
    set_xio_file_type    iodef.xml one_file

    cd ${SETTE_DIR}
    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}

    cd ${SETTE_DIR}
    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}


    ## Test of XIOS configured in detached mode with a single output file. 
    ## $NUM_XIOSERVERS IO server are used.
    ## the total number of allocated cores is $NUM_XIOSERVERS + $NPROC 

    export TEST_NAME="DETACHED_ONE"
    export NUM_XIOSERVERS=2
    export USING_MPMD="true"
    cd ${SETTE_DIR}
#
    . ./prepare_exe_dir.sh
    JOB_FILE=${EXE_DIR}/run_job.sh
    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
    cd ${EXE_DIR}
    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_LR\"
    set_namelist namelist_cfg nn_it000 1
    set_namelist namelist_cfg nn_itend 120
    set_namelist namelist_cfg nn_stock 120
    set_namelist namelist_cfg ln_clobber .true.
    set_namelist namelist_cfg jp_cfg $jp_cfg
    set_namelist namelist_cfg jpidta $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjdta $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpiglo $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjglo $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpni $jpni
    set_namelist namelist_cfg jpnj $jpnj
    set_namelist namelist_cfg jpnij $NPROC

    set_xio_using_server iodef.xml $USING_MPMD
    set_xio_file_type    iodef.xml one_file

    cd ${SETTE_DIR}
    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}

    cd ${SETTE_DIR}
    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}

    ## Test of XIOS configured in detached mode with multiple output files (one for each IO server). 
    ## $NUM_XIOSERVERS IO server are used.
    ## the total number of allocated cores is $NUM_XIOSERVERS + $NPROC 


    export TEST_NAME="DETACHED_MULTIPLE"
    export NUM_XIOSERVERS=2
    export USING_MPMD="true"
    cd ${SETTE_DIR}
    . ./prepare_exe_dir.sh
    JOB_FILE=${EXE_DIR}/run_job.sh
    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
    cd ${EXE_DIR}
    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_LR\"
    set_namelist namelist_cfg nn_it000 1
    set_namelist namelist_cfg nn_itend 120
    set_namelist namelist_cfg nn_stock 120
    set_namelist namelist_cfg ln_clobber .true.
    set_namelist namelist_cfg jp_cfg $jp_cfg
    set_namelist namelist_cfg jpidta $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjdta $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpiglo $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjglo $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpni $jpni
    set_namelist namelist_cfg jpnj $jpnj
    set_namelist namelist_cfg jpnij $NPROC

    set_xio_using_server iodef.xml $USING_MPMD
    set_xio_file_type    iodef.xml multiple_file

    cd ${SETTE_DIR}
    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}

    cd ${SETTE_DIR}
    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}

fi

#==========================================================
# TESTS FOR XIOS USING GYRE CONFIGURATION AT HIGH RESOLUTION
#==========================================================

if [ ${config} -eq 2 ] ;  then
    ## Test of XIOS configured in attached mode with multiple output files (one for each process). 
    ## $NPROC processes are used 

    NPROC=64
    jp_cfg=30
    jpni=8
    jpnj=8
    export NUM_XIOSERVERS=0
    export USING_MPMD="false"

    export TEST_NAME="ATTACHED_MULTIPLE"

    cd ${SETTE_DIR}
    . ../CONFIG/makenemo -m ${CMP_NAM} -n GYRE_XIOS_HR -r GYRE_XIOS -j 8
    cd ${SETTE_DIR}
#
    . ./param.cfg
    . ./all_functions.sh
    . ./prepare_exe_dir.sh
#
    JOB_FILE=${EXE_DIR}/run_job.sh
    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
    cd ${EXE_DIR}
    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_HR\"
    set_namelist namelist_cfg nn_it000 1
    set_namelist namelist_cfg nn_itend 120
    set_namelist namelist_cfg nn_stock 120
    set_namelist namelist_cfg nn_bench 1
    set_namelist namelist_cfg ln_clobber .true.
    set_namelist namelist_cfg jp_cfg $jp_cfg
    set_namelist namelist_cfg jpidta $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjdta $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpiglo $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjglo $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpni $jpni
    set_namelist namelist_cfg jpnj $jpnj
    set_namelist namelist_cfg jpnij $NPROC

    set_xio_using_server iodef.xml $USING_MPMD
    set_xio_file_type    iodef.xml multiple_file

    cd ${SETTE_DIR}
    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}

    cd ${SETTE_DIR}
    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}


    ## Test of XIOS configured in attached mode with a single output file. 

    export TEST_NAME="ATTACHED_ONE"
    export NUM_XIOSERVERS=0
    export USING_MPMD="false"
    cd ${SETTE_DIR}
#
    . ./prepare_exe_dir.sh
#
    JOB_FILE=${EXE_DIR}/run_job.sh
    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
    cd ${EXE_DIR}
    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_HR\"
    set_namelist namelist_cfg nn_it000 1
    set_namelist namelist_cfg nn_itend 120
    set_namelist namelist_cfg nn_stock 120
    set_namelist namelist_cfg nn_bench 1
    set_namelist namelist_cfg ln_clobber .true.
    set_namelist namelist_cfg jp_cfg $jp_cfg
    set_namelist namelist_cfg jpidta $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjdta $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpiglo $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjglo $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpni $jpni
    set_namelist namelist_cfg jpnj $jpnj
    set_namelist namelist_cfg jpnij $NPROC

    set_xio_using_server iodef.xml $USING_MPMD
    set_xio_file_type    iodef.xml one_file

    cd ${SETTE_DIR}
    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}

    cd ${SETTE_DIR}
    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}


    ## Test of XIOS configured in detached mode with a single output file. 
    ## $NUM_XIOSERVERS IO server are used.
    ## the total number of allocated cores is $NUM_XIOSERVERS + $NPROC 

    export TEST_NAME="DETACHED_ONE"
    export NUM_XIOSERVERS=8
    export USING_MPMD="true"
    cd ${SETTE_DIR}
#
    . ./prepare_exe_dir.sh
#
    JOB_FILE=${EXE_DIR}/run_job.sh
    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
    cd ${EXE_DIR}
    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_HR\"
    set_namelist namelist_cfg nn_it000 1
    set_namelist namelist_cfg nn_itend 120
    set_namelist namelist_cfg nn_stock 120
    set_namelist namelist_cfg nn_bench 1
    set_namelist namelist_cfg ln_clobber .true.
    set_namelist namelist_cfg jp_cfg $jp_cfg
    set_namelist namelist_cfg jpidta $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjdta $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpiglo $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjglo $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpni $jpni
    set_namelist namelist_cfg jpnj $jpnj
    set_namelist namelist_cfg jpnij $NPROC

    set_xio_using_server iodef.xml $USING_MPMD
    set_xio_file_type    iodef.xml one_file

    cd ${SETTE_DIR}
    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}

    cd ${SETTE_DIR}
    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}

    ## Test of XIOS configured in detached mode with multiple output files (one for each IO server). 
    ## $NUM_XIOSERVERS IO server are used.
    ## the total number of allocated cores is $NUM_XIOSERVERS + $NPROC 

    export TEST_NAME="DETACHED_MULTIPLE"
    export NUM_XIOSERVERS=8
    export USING_MPMD="true"
    cd ${SETTE_DIR}
    . ./prepare_exe_dir.sh
    JOB_FILE=${EXE_DIR}/run_job.sh
    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
    cd ${EXE_DIR}
    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_HR\"
    set_namelist namelist_cfg nn_it000 1
    set_namelist namelist_cfg nn_itend 120
    set_namelist namelist_cfg nn_stock 120
    set_namelist namelist_cfg nn_bench 1
    set_namelist namelist_cfg ln_clobber .true.
    set_namelist namelist_cfg jp_cfg $jp_cfg
    set_namelist namelist_cfg jpidta $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjdta $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpiglo $(( $jp_cfg * 30 + 2 ))
    set_namelist namelist_cfg jpjglo $(( $jp_cfg * 20 + 2 ))
    set_namelist namelist_cfg jpni $jpni
    set_namelist namelist_cfg jpnj $jpnj
    set_namelist namelist_cfg jpnij $NPROC

    set_xio_using_server iodef.xml $USING_MPMD
    set_xio_file_type    iodef.xml multiple_file

    cd ${SETTE_DIR}
    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}

    cd ${SETTE_DIR}
    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}

fi

done