#!/bin/bash 
#!
#PBS -N NEMO
#PBS -l select=NODES:ncpus=8:mpiprocs=4
#PBS -l place=scatter:excl
#PBS -l walltime=00:59:00
#PBS -q nemo
#PBS -W group_list=nemo
#PBS -j oe
#
  OCEANCORES=NPROCS
  XIOCORES=NXIOPROCS
  export SETTE_DIR=DEF_SETTE_DIR
#
# load sette functions (only post_test_tidyup needed)
#
  . ${SETTE_DIR}/all_functions.sh
###############################################################
#
# set up mpp computing environment
#
#
# Local settings. These settings are for a particular machine (the ALTIX ICE system at NOCS)
# at a particular stage of that machine's evolution. This template file is provided for
# illustration purposes only and will not work on any other machine. There should, however,
# be sufficient similarity with other MPP platforms and batch systems for this example to provide
# a useful guide for experienced users
#
  module load mpt/2.03
  export MPT_HOME=/opt/sgi/mpt/mpt-2.03
  export NC4_HOME=/fibre/omfman/NETCDF_PAR
  export HD5_HOME=/fibre/omfman/NETCDF_PAR
  export XIO_HOME=/fibre/omfman/XIOS
  export SVN_HOME=/nerc/packages/subversion/v1.7.2
  export FTN_LIB=/sw/Intel/fce/11.1.072/lib/intel64
  export ICC_LIB=/sw/Intel/cce/11.1.072/lib/intel64
#
  echo Running on host `hostname`
  echo Time is `date`
  echo PBS job ID is $PBS_JOBID
  echo This jobs runs on the following machines:
  echo `cat $PBS_NODEFILE | uniq`
# 
# Create a machine file for MPI
  cat $PBS_NODEFILE | uniq > host.file.$PBS_JOBID
#
  echo export LD_LIBRARY_PATH=${MPT_HOME}/lib:${ICC_LIB}:${FTN_LIB}:${NC4_HOME}/lib:${HD5_HOME}/lib:${SVN_HOME}/lib:${LD_LIBRARY_PATH}
  export LD_LIBRARY_PATH=${MPT_HOME}/lib:${ICC_LIB}:${FTN_LIB}:${NC4_HOME}/lib:${HD5_HOME}/lib:${SVN_HOME}/lib:${LD_LIBRARY_PATH}
# 
  echo export PATH=${MPT_HOME}/bin:${SVN_HOME}/bin:$PATH
  export PATH=${MPT_HOME}/bin:${SVN_HOME}/bin:$PATH
#
  export PSM_SHAREDPORTS=1
  echo export MPI_PPN=4
  echo export MPI_PPN_CLIST="2-3-6-7-0-1-4-5"
  echo export OMP_NUM_THREADS=1 
#
# Don't remove neither change the following line
# BODY
#
# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
# (via sed operating on this template job file). Note that the number of compute nodes required
# (half-populated on the NOCS ALTIX system) is also set by the fcm_job.sh on the PBS select
# header line above.
#
# These variables are needed by post_test_tidyup function in all_functions.sh
#
  export INPUT_DIR=DEF_INPUT_DIR
  export CONFIG_DIR=DEF_CONFIG_DIR
  export TOOLS_DIR=DEF_TOOLS_DIR
  export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION
  export NEW_CONF=DEF_NEW_CONF
  export CMP_NAM=DEF_CMP_NAM
  export TEST_NAME=DEF_TEST_NAME
  export EXE_DIR=DEF_EXE_DIR
#
# end of set up
###############################################################
#
# change to the working directory 
#
# cd $PBS_O_WORKDIR
  cd $EXE_DIR
  echo Directory is `pwd`
  if [ ! -f ./xios_server.exe ] && [ -f ${XIO_HOME}/bin/xios_server.exe ] 
   then
     cp ${XIO_HOME}/bin/xios_server.exe .
  fi
  if [ ! -f ./xios_server.exe ]
   then
     echo "./xios_server.exe not found"
     echo "run aborted"
     exit
  fi
#
#
#  Run the parallel MPI executable 
#
  echo "Running time ${MPT_HOME}/bin/mpiexec_mpt -np " $XIOCORES " ./xios_server.exe : -np " $OCEANCORES " ./nemo"
#
  time ${MPT_HOME}/bin/mpiexec_mpt -np $XIOCORES ./xios_server.exe :  -np $OCEANCORES ./nemo
#
  post_test_tidyup
# END_BODY
# Don't remove neither change the previous line
  exit