source: branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/GYRE_benchmark.sh @ 7230

#!/bin/bash
############################################################
# Author : Tim Graham
# Contact: tim.graham@metoffice.gov.uk
#
# ----------------------------------------------------------------------
# NEMO Consortium (2015)
# Software governed by the CeCILL licence     (NEMOGCM/NEMO_CeCILL.txt)
# ----------------------------------------------------------------------
#
#############################################################
#set -x
set -o posix
#set -u
#set -e
# ===========
# DESCRIPTION
# ===========
#
##############################################################

#
# Compiler among those in NEMOGCM/ARCH
COMPILER=XC40_METO_perf_tools
export BATCH_COMMAND_PAR="qsub"
export BATCH_COMMAND_SEQ=$BATCH_COMMAND_PAR
export INTERACT_FLAG="no"
export MPIRUN_FLAG="yes"
export USING_XIOS="yes"
export PERF_REGIONS_HOME=/home/tigrah/perf_regions

#Number of cores on a single node of HPC
export CORES_PER_NODE=32

# List of papi_counters to be used
export LIST_COUNTERS=PAPI_L1_ICM


# Make a link to perf_regions_fortran.f90 in MY_SRC directory
# I think it should be possible to just include the .mod file but it's failing on XC40 at the moment
ln -sf $PERF_REGIONS_HOME/src/perf_regions_fortran.F90 MY_SRC/perf_regions_fortran.F90

#
export DEL_KEYS="key_iomput"
if [ ${USING_XIOS} == "yes" ] 
 then 
   export DEL_KEYS=""
fi
#
# Settings which control the use of stand alone servers (only relevant if using xios)
#
export USING_MPMD="no"
export NUM_XIOSERVERS=4
export JOB_PREFIX=batch-mpmd
#
if [ ${USING_MPMD} == "no" ] 
 then
   export NUM_XIOSERVERS=0
   export JOB_PREFIX=batch
fi
#
#
if [ ${USING_MPMD} == "yes" ] && [ ${USING_XIOS} == "no"]
 then
   echo "Incompatible choices. MPMD mode requires the XIOS server"
   exit
fi
#
#########################################
# Architecture specific settings
# control HPC behaviour for benchmarking
#########################################
module load cray-netcdf


#############################
# Directory to run the tests
#############################
SETTE_DIR=$(cd $(dirname "$0"); pwd)
MAIN_DIR=${SETTE_DIR%/BENCHMARK}
CONFIG_DIR=${MAIN_DIR}/CONFIG
TOOLS_DIR=${MAIN_DIR}/TOOLS
COMPIL_DIR=${TOOLS_DIR}/COMPILE

CMP_NAM=${1:-$COMPILER}
# Copy job_batch_COMPILER file for specific compiler into job_batch_template
cd ${SETTE_DIR}
cp BATCH_TEMPLATE/${JOB_PREFIX}-${COMPILER} job_batch_template || exit


#Cleanup instrumentation from old run
. ./perf_regions_instrumentation.py cleanup

#Instrument OPA_SRC code before compilation
. ./perf_regions_instrumentation.py

# GYRE CONFIGURATION
export TEST_NAME="BENCHMARK"
. ./param.cfg
. ./all_functions.sh
cd ${CONFIG_DIR}
. ./makenemo -m ${CMP_NAM} -n GYRE_BENCHMARK -e /home/tigrah/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/MY_SRC -r GYRE -j 8 del_key "key_mpp_mpi key_iomput"
echo $PWD
cd GYRE_BENCHMARK
mkdir -p ${TEST_NAME}
export EXE_DIR=${CONFIG_DIR}/${NEW_CONF}/${TEST_NAME}
JOB_FILE=${EXE_DIR}/run_job.sh
proc=1
cd $EXE_DIR
  cp -rL ${CONFIG_DIR}/${NEW_CONF}/EXP00/* ${EXE_DIR}/.
  cp -r ${SETTE_DIR}/iodef_sette.xml ${EXE_DIR}/iodef.xml
  cp -r ${SETTE_DIR}/perf_region_list.txt ${EXE_DIR}/perf_region_list.txt

NPROC=1
cd ${EXE_DIR}  
if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
set_namelist namelist_cfg cn_exp \"GYRE_LONG\"
set_namelist namelist_cfg nn_it000 1
set_namelist namelist_cfg nn_itend 500
set_namelist namelist_cfg nn_stock 10000
set_namelist namelist_cfg ln_clobber .true.
set_namelist namelist_cfg nn_fwb 0
set_namelist namelist_cfg jpni 1
set_namelist namelist_cfg jpnj 1
set_namelist namelist_cfg jpnij 1
set_namelist namelist_cfg jpiglo  50
set_namelist namelist_cfg jpjglo  40
set_namelist namelist_cfg jpidta  50
set_namelist namelist_cfg jpjdta  40
set_namelist namelist_cfg jp_cfg  1
if [ ${USING_MPMD} == "yes" ] ; then
   set_xio_using_server iodef.xml true
else
   set_xio_using_server iodef.xml false
fi
cd ${SETTE_DIR}
. ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}
cd ${SETTE_DIR} #Prepare job seems to leave us in a different working directory
. ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}