Index: /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/BATCH_TEMPLATE/batch-XC40_METO
===================================================================
--- /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/BATCH_TEMPLATE/batch-XC40_METO	(revision 7230)
+++ /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/BATCH_TEMPLATE/batch-XC40_METO	(revision 7230)
@@ -0,0 +1,68 @@
+#/bin/bash 
+#!
+#PBS -N nemo_sette
+#PBS -l walltime=00:08:00
+#PBS -j oe
+#PBS -q shared
+#PBS -l select=1:ncpus=32
+
+  export PBS_O_WORKDIR=$(readlink -f $PBS_O_WORKDIR)
+  export OMP_NUM_THREADS=1
+  cd $PBS_O_WORKDIR
+  export XIO_HOME=/projects/ocean/nemo/xios/xios_1.0_r618_20150619/XIOS
+#
+  echo " ";
+  export OMP_NUM_THREADS=1
+  O_PER_NODE=32
+  X_PER_NODE=4
+  OCORES=NPROCS
+  XCORES=NXIOPROCS
+  if [ $OCORES -le 32 ] ; then O_PER_NODE=$OCORES; fi
+  export SETTE_DIR=DEF_SETTE_DIR
+
+###############################################################
+#
+#
+# load sette functions (only post_test_tidyup needed)
+#
+  . ${SETTE_DIR}/all_functions.sh
+###############################################################
+#
+# Don't remove neither change the following line
+# BODY
+#
+# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
+# (via sed operating on this template job file). Note that the number of compute nodes required
+# is also set by the fcm_job.sh on the PBS select header line above.
+#
+# These variables are needed by post_test_tidyup function in all_functions.sh
+#
+  export INPUT_DIR=DEF_INPUT_DIR
+  export CONFIG_DIR=DEF_CONFIG_DIR
+  export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION
+  export NEW_CONF=DEF_NEW_CONF
+  export CMP_NAM=DEF_CMP_NAM
+  export TEST_NAME=DEF_TEST_NAME
+  export EXE_DIR=DEF_EXE_DIR
+  ulimit -c unlimited
+  ulimit -s unlimited
+  export FORT_FMT_RECL=132
+#
+# end of set up
+###############################################################
+#
+# change to the working directory 
+#
+  cd $EXE_DIR
+  proc=1
+  while [ $proc -le 32 ]
+  do
+    echo Directory is `pwd`
+    mpiexec -n 1 ./opa
+  done
+
+#
+# END_BODY
+# Don't remove neither change the previous line
+  exit
+
Index: /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/BATCH_TEMPLATE/batch-XC40_METO_perf_tools
===================================================================
--- /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/BATCH_TEMPLATE/batch-XC40_METO_perf_tools	(revision 7230)
+++ /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/BATCH_TEMPLATE/batch-XC40_METO_perf_tools	(revision 7230)
@@ -0,0 +1,76 @@
+#/bin/bash 
+#!
+#PBS -N nemo_sette
+#PBS -l walltime=00:08:00
+#PBS -j oe
+#PBS -q shared
+#PBS -l select=1:ncpus=32
+
+  export PBS_O_WORKDIR=$(readlink -f $PBS_O_WORKDIR)
+  export OMP_NUM_THREADS=1
+  cd $PBS_O_WORKDIR
+  export XIO_HOME=/projects/ocean/nemo/xios/xios_1.0_r618_20150619/XIOS
+#
+  echo " ";
+  export OMP_NUM_THREADS=1
+  O_PER_NODE=32
+  X_PER_NODE=4
+  OCORES=NPROCS
+  XCORES=NXIOPROCS
+  if [ $OCORES -le 32 ] ; then O_PER_NODE=$OCORES; fi
+  export SETTE_DIR=DEF_SETTE_DIR
+
+  export LD_LIBRARY_PATH=$LD_LIBRARY_PATH:DEF_LD_LIB
+  export LIST_COUNTERS=DEF_COUNTERS
+
+###############################################################
+#
+#
+# load sette functions (only post_test_tidyup needed)
+#
+  . ${SETTE_DIR}/all_functions.sh
+###############################################################
+#
+# Don't remove neither change the following line
+# BODY
+#
+# Test specific settings. Do not hand edit these lines; the fcm_job.sh script will set these
+# (via sed operating on this template job file). Note that the number of compute nodes required
+# is also set by the fcm_job.sh on the PBS select header line above.
+#
+# These variables are needed by post_test_tidyup function in all_functions.sh
+#
+  export INPUT_DIR=DEF_INPUT_DIR
+  export CONFIG_DIR=DEF_CONFIG_DIR
+  export NEMO_VALIDATION_DIR=DEF_NEMO_VALIDATION
+  export NEW_CONF=DEF_NEW_CONF
+  export CMP_NAM=DEF_CMP_NAM
+  export TEST_NAME=DEF_TEST_NAME
+  export EXE_DIR=DEF_EXE_DIR
+  ulimit -c unlimited
+  ulimit -s unlimited
+  export FORT_FMT_RECL=132
+#
+# end of set up
+###############################################################
+#
+# change to the working directory 
+#
+  cd $EXE_DIR
+  proc=1
+  hostname
+  for proc in `seq 1 DEF_CORES_PER_NODE`;
+  do
+    date
+    echo Directory is `pwd`
+    taskset -c $proc ./opa &
+    date
+#    ./opa &
+  done
+  wait
+
+#
+# END_BODY
+# Don't remove neither change the previous line
+  exit
+
Index: /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/GYRE_benchmark.sh
===================================================================
--- /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/GYRE_benchmark.sh	(revision 7229)
+++ /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/GYRE_benchmark.sh	(revision 7230)
@@ -22,5 +22,5 @@
 #
 # Compiler among those in NEMOGCM/ARCH
-COMPILER=XC40_METO
+COMPILER=XC40_METO_perf_tools
 export BATCH_COMMAND_PAR="qsub"
 export BATCH_COMMAND_SEQ=$BATCH_COMMAND_PAR
@@ -28,4 +28,17 @@
 export MPIRUN_FLAG="yes"
 export USING_XIOS="yes"
+export PERF_REGIONS_HOME=/home/tigrah/perf_regions
+
+#Number of cores on a single node of HPC
+export CORES_PER_NODE=32
+
+# List of papi_counters to be used
+export LIST_COUNTERS=PAPI_L1_ICM
+
+
+# Make a link to perf_regions_fortran.f90 in MY_SRC directory
+# I think it should be possible to just include the .mod file but it's failing on XC40 at the moment
+ln -sf $PERF_REGIONS_HOME/src/perf_regions_fortran.F90 MY_SRC/perf_regions_fortran.F90
+
 #
 export DEL_KEYS="key_iomput"
@@ -58,5 +71,4 @@
 # control HPC behaviour for benchmarking
 #########################################
-module load perftools-lite
 module load cray-netcdf
 
@@ -77,18 +89,27 @@
 
 
+#Cleanup instrumentation from old run
+. ./perf_regions_instrumentation.py cleanup
+
+#Instrument OPA_SRC code before compilation
+. ./perf_regions_instrumentation.py
+
 # GYRE CONFIGURATION
 export TEST_NAME="BENCHMARK"
-export NUM_COUNTERS=2
-export LIST_COUNTERS=FP_COMP_OPS_EXE,PAPI_L3_TCM
-
 . ./param.cfg
 . ./all_functions.sh
 cd ${CONFIG_DIR}
-. ./makenemo -m ${CMP_NAM} -n GYRE_BENCHMARK -r GYRE -j 8 del_key key_mpp_mpi
-mkdir -p ${NEW_CONF}/${TEST_NAME}
+. ./makenemo -m ${CMP_NAM} -n GYRE_BENCHMARK -e /home/tigrah/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/MY_SRC -r GYRE -j 8 del_key "key_mpp_mpi key_iomput"
+echo $PWD
+cd GYRE_BENCHMARK
+mkdir -p ${TEST_NAME}
 export EXE_DIR=${CONFIG_DIR}/${NEW_CONF}/${TEST_NAME}
-cp -rL ${CONFIG_DIR}/${NEW_CONF}/EXP00/* ${EXE_DIR}/.
-cp -r ${SETTE_DIR}/iodef_sette.xml ${EXE_DIR}/iodef.xml
 JOB_FILE=${EXE_DIR}/run_job.sh
+proc=1
+cd $EXE_DIR
+  cp -rL ${CONFIG_DIR}/${NEW_CONF}/EXP00/* ${EXE_DIR}/.
+  cp -r ${SETTE_DIR}/iodef_sette.xml ${EXE_DIR}/iodef.xml
+  cp -r ${SETTE_DIR}/perf_region_list.txt ${EXE_DIR}/perf_region_list.txt
+
 NPROC=1
 cd ${EXE_DIR}  
Index: /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/perf_regions_instrumentation.py
===================================================================
--- /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/perf_regions_instrumentation.py	(revision 7230)
+++ /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/perf_regions_instrumentation.py	(revision 7230)
@@ -0,0 +1,43 @@
+#!/usr/bin/env python2.7
+
+
+import sys
+import os
+path = os.path.expandvars('$PERF_REGIONS_HOME')+'/scripts'
+sys.path.append(path)
+import perf_regions
+
+pf = perf_regions.perf_regions(
+		["../NEMO/OPA_SRC"],	# list with source directories
+		[
+			".*timing_init.*",		# initialization of timing
+			".*timing_finalize.*",	# shutdown of timing
+
+			".*USE timing.*",	# include part
+
+			".*timing_start\(\'(.*)\'\)",	# start of timing
+			".*timing_stop\(\'(.*)\'\)",	# end of timing
+			".*timing_reset.*"		# reset of timing
+		],
+#		'../../',	# perf region root directory
+		'./',		# output directory of perf region tools
+		'fortran',
+		'../NEMO/OPA_SRC/timing.F90'    # Don't modify timing.F90
+	)
+
+
+
+if len(sys.argv) > 1:
+	if sys.argv[1] == 'preprocess':
+		print("PREPROCESS")
+		pf.preprocessor()
+
+	elif sys.argv[1] == 'cleanup':
+		print("CLEANUP")
+		pf.cleanup()
+
+	else:
+		print("Unsupported argument "+sys.argv[1])
+
+else:
+	pf.preprocessor()
Index: /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/prepare_job.sh
===================================================================
--- /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/prepare_job.sh	(revision 7229)
+++ /branches/2016/dev_HPC_Gyre_benchmark_test/NEMOGCM/BENCHMARK/prepare_job.sh	(revision 7230)
@@ -251,5 +251,7 @@
              -e"s:MPI_FLAG:${MPI_FLAG}:" \
              -e"s:DEF_NEMO_VALIDATION:${NEMO_VALIDATION_DIR}:" -e"s:DEF_NEW_CONF:${NEW_CONF}:" \
-             -e"s:DEF_CMP_NAM:${CMP_NAM}:" -e"s:DEF_TEST_NAME:${TEST_NAME}:" > run_sette_test.job
+             -e"s:DEF_CMP_NAM:${CMP_NAM}:" -e"s:DEF_TEST_NAME:${TEST_NAME}:" \
+             -e"s:DEF_LD_LIB:${PERF_REGIONS_HOME}/build/:" -e"s:DEF_COUNTERS:${LIST_COUNTERS}:" \
+             -e"s:DEF_CORES_PER_NODE:$CORES_PER_NODE:" > run_sette_test.job
 
         case ${COMPILER} in
