#!/bin/bash
#SBATCH -N NODES
#SBATCH --time=03:59:00 

module purge
module load shared
module load gcc
module load noc/intel/17.0.4
module load openmpi/intel/64/1.6.5
module load slurm/16.05.8
module load noc/subversion/1.7.16

#
  export OMP_NUM_THREADS=1
  export OCORES=NPROCS
  export XCORES=NXIOPROCS
  export SETTE_DIR=DEF_SETTE_DIR
  export XIO_HOME=/hpcdata/working/smueller/xios-2.5
#
# load sette functions (only post_test_tidyup needed)
#
  . ${SETTE_DIR}/all_functions.sh
###############################################################
#
# set up mpp computing environment
#
#
# Local settings. These settings are for a particular machine (the MOBILIS ClusterVision 
# system at NOCS) at a particular stage of that machine's evolution. This template file
# is provided for illustration purposes only and will not work on any other machine. There
# should, however, be sufficient similarity with other MPP platforms and batch systems
# for this example to provide a useful guide for experienced users
#
# Don't remove neither change the following line
# BODY
#
# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
# (via sed operating on this template job file). Note that the number of compute nodes required
# is also set by the fcm_job.sh on the PBS select header line above.
#
# These variables are needed by post_test_tidyup function in all_functions.sh
#
  export INPUT_DIR=DEF_INPUT_DIR
  export CONFIG_DIR=DEF_CONFIG_DIR
  export TOOLS_DIR=DEF_TOOLS_DIR
  export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION
  export NEW_CONF=DEF_NEW_CONF
  export CMP_NAM=DEF_CMP_NAM
  export TEST_NAME=DEF_TEST_NAME
  export EXE_DIR=DEF_EXE_DIR
#
# end of set up
###############################################################
#
# change to the working directory 
#
  cd $EXE_DIR
  echo Directory is `pwd`
  if [ $XCORES -eq 0 ]; then
#
# Run SPMD case
#
       echo time `which mpirun` --report-bindings -x MALLOC_MMAP_MAX_=-1                 \
                                -x MALLOC_TRIM_THRESHOLD_=33554432 -np $OCORES --bynode  \
                                --mca mpi_paffinity_alone 1 ./nemo 
            time `which mpirun` --report-bindings -x MALLOC_MMAP_MAX_=-1                 \
                                -x MALLOC_TRIM_THRESHOLD_=33554432 -np $OCORES --bynode  \
                                --mca mpi_paffinity_alone 1 ./nemo 
  else
       if [ ! -f ./xios_server.exe ] && [ -f ${XIO_HOME}/bin/xios_server.exe ]; then
          cp ${XIO_HOME}/bin/xios_server.exe .
       fi
       if [ ! -f ./xios_server.exe ]; then
          echo "./xios_server.exe not found"
          echo "run aborted"
          exit
       fi
#
#  Run MPMD case
#
       echo time `which mpirun` --report-bindings -x MALLOC_MMAP_MAX_=-1 \
                                -x MALLOC_TRIM_THRESHOLD_=33554432       \
                                -np $XCORES ./xios_server.exe            \
                              : -np $OCORES --bynode  --mca mpi_paffinity_alone 1 ./nemo 
#
            time `which mpirun` --report-bindings -x MALLOC_MMAP_MAX_=-1 \
                                -x MALLOC_TRIM_THRESHOLD_=33554432       \
                                -np $XCORES ./xios_server.exe            \
                              : -np $OCORES --bynode  --mca mpi_paffinity_alone 1 ./nemo 
#
  fi
#
  post_test_tidyup
# END_BODY
# Don't remove neither change the previous line
  exit