Index: branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/BATCH_TEMPLATE/batch-gfortran_athena_xios
===================================================================
--- branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/BATCH_TEMPLATE/batch-gfortran_athena_xios	(revision 4328)
+++ branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/BATCH_TEMPLATE/batch-gfortran_athena_xios	(revision 4373)
@@ -2,5 +2,5 @@
 #!
 #BSUB -q poe_short
-#BSUB -n NPROCS
+#BSUB -n TOTAL_NPROCS
 #BSUB -a poe
 #BSUB -J MPI_config
@@ -15,4 +15,5 @@
 #
   OCEANCORES=NPROCS
+  XIOS_NUMPROCS=NXIOPROCS
   export SETTE_DIR=DEF_SETTE_DIR
 
@@ -27,4 +28,30 @@
 
 export MPIRUN="mpirun.lsf"
+export LANG=en_US
+export NLSPATH=/opt/ibmhpc/pecurrent/base/msg/%L/%N:$NLSPATH
+export MP_LABELIO=yes
+export MP_STDOUTMODE=unordered
+export MP_EUILIB=us
+export MP_INFOLEVEL=0
+export MP_EUIDEVELOP=min
+export MP_INSTANCES=1
+export MP_SHARED_MEMORY=yes
+export MP_EAGER_LIMIT_LOCAL=2000000000
+export MP_EUIDEVICE=sn_single
+export MP_BULK_MIN_MSG_SIZE=16384
+export MP_POLLING_INTERVAL=1000000
+export MP_DEVTYPE=ib
+export MP_EAGER_LIMIT=262144
+export MP_TASK_AFFINITY=core
+export MP_BINDPROC=yes
+export MP_DEBUG_CHECK_PARAM=no
+export MP_MSG_API=mpi
+export MP_USE_BULK_XFER=yes
+export MP_WAIT_MODE=poll
+export LDR_CNTRL=TEXTPSIZE=64K@STACKPSIZE=64K@DATAPSIZE=64K
+export LD_LIBRARY_PATH=/users/home/ans040/local/lib:${LD_LIBRARY_PATH}
+
+XIOS_SERVER_PATHNAME="/users/home/ans040/SOFTWARE/XIOS/trunk/bin/xios_server.exe"
+
 
 #
@@ -68,5 +95,25 @@
 
   if [ MPI_FLAG == "yes" ]; then
-     time ${MPIRUN} ./opa
+     if [ $XIOS_NUMPROCS -eq 0 ]; then
+       time ${MPIRUN} ./opa
+     else
+
+       export MP_PGMMODEL=mpmd
+       export MP_CMDFILE=xioscmdfile
+       rm xioscmdfile &> /dev/null
+       touch xioscmdfile
+       k=0
+       while (( k < $OCEANCORES )) ; do
+          echo ./opa >> xioscmdfile
+          k=`expr $k + 1`
+       done
+       k=0
+       while (( k < $XIOS_NUMPROCS )) ; do
+          echo $XIOS_SERVER_PATHNAME >> xioscmdfile
+          k=`expr $k + 1`
+       done
+
+       time ${MPIRUN}
+     fi
   else
      time ./opa
Index: branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/param.cfg
===================================================================
--- branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/param.cfg	(revision 4328)
+++ branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/param.cfg	(revision 4373)
@@ -1,4 +1,4 @@
 #- forcing files storing 
-FORCING_DIR=${WORKDIR}/FORCING
+FORCING_DIR=~/FORCING
 #- input files storing 
 INPUT_DIR=${CONFIG_DIR}/${NEW_CONF}/EXP00
@@ -6,3 +6,3 @@
 #TMPDIR=${CONFIG_DIR}/${NEW_CONF}/EXP00
 #- VALIDATION files storing 
-NEMO_VALIDATION_DIR=${WORKDIR}/NEMO_VALIDATION
+NEMO_VALIDATION_DIR=~/NEMO_VALIDATION
Index: branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/prepare_exe_dir.sh
===================================================================
--- branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/prepare_exe_dir.sh	(revision 4328)
+++ branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/prepare_exe_dir.sh	(revision 4373)
@@ -71,4 +71,4 @@
 
 cp -r ${CONFIG_DIR}/${NEW_CONF}/EXP00/* ${EXE_DIR}/.
-cp -r ${SETTE_DIR}/iodef_sette.xml ${EXE_DIR}/iodef.xml
+#cp -r ${SETTE_DIR}/iodef_sette.xml ${EXE_DIR}/iodef.xml
 cd ${EXE_DIR}
Index: branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/prepare_job.sh
===================================================================
--- branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/prepare_job.sh	(revision 4328)
+++ branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/prepare_job.sh	(revision 4373)
@@ -231,5 +231,7 @@
 # Pass settings into job file by using sed to edit predefined strings
 #
+        TOTAL_NPROCS=$(( $NB_PROC + $NXIO_PROC ))
         cat ${SETTE_DIR}/job_batch_template | sed -e"s/NODES/${NB_NODES}/" \
+             -e"s/TOTAL_NPROCS/${TOTAL_NPROCS}/" \
              -e"s/NPROCS/${NB_PROC}/" \
              -e"s/NXIOPROCS/${NXIO_PROC}/" \
Index: branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/sette_xios.sh
===================================================================
--- branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/sette_xios.sh	(revision 4373)
+++ branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/sette_xios.sh	(revision 4373)
@@ -0,0 +1,362 @@
+#!/bin/bash
+############################################################
+# Author : Simona Flavoni for NEMO
+# Contact: sflod@locean-ipsl.upmc.fr
+# 2013   : A.C. Coward added options for testing with XIOS in dettached mode
+#
+# sette.sh   : principal script of SET TEsts for NEMO (SETTE)
+# ----------------------------------------------------------------------
+# NEMO/SETTE , NEMO Consortium (2010)
+# Software governed by the CeCILL licence     (NEMOGCM/NEMO_CeCILL.txt)
+# ----------------------------------------------------------------------
+#
+#############################################################
+#set -x
+set -o posix
+#set -u
+#set -e
+# ===========
+# DESCRIPTION
+# ===========
+#
+# Variables to be checked by user:
+#
+# COMPILER          : name of compiler as defined in NEMOGCM/ARCH directory 
+# BATCH_COMMAND_PAR :  name of the command for submitting parallel batch jobs
+# BATCH_COMMAND_SEQ :  name of the command for submitting sequential batch jobs  
+# INTERACT_FLAG     : flag to run in interactive mode "yes"
+#                           to run in batch mode "no"
+# MPIRUN_FLAG       : flag to run in parallel (MPI) "yes"
+#                           to run in sequential mode (NB_PROC = 1) "no"
+# USING_XIOS        : flag to control the activation of key_iomput
+#                      "yes" to compile using key_iomput and link to the external XIOS library
+#                      "no"  to compile without key_iomput and link to the old IOIPSL library
+# USING_MPMD        : flag to control the use of stand-alone IO servers
+#                     requires USING_XIOS="yes"
+#                      "yes" to run in MPMD (detached) mode with stand-alone IO servers
+#                      "no"  to run in SPMD (attached) mode without separate IO servers 
+# NUM_XIOSERVERS    : number of stand-alone IO servers to employ
+#                     set to zero if USING_MPMD="no"
+#
+# Principal script is sette.sh, that calls 
+#
+#  makenemo  : to create successive exectuables in ${CONFIG_NAME}/BLD/bin/nemo.exe 
+#              and links to opa in ${CONFIG_NAME}/EXP00)
+#
+#  param.cfg : sets and loads following directories:
+#
+#   FORCING_DIR         : is the directory for forcing files (tarfile)
+#   INPUT_DIR           : is the directory for input files storing 
+#   TMPDIR              : is the temporary directory (if needed)
+#   NEMO_VALIDATION_DIR : is the validation directory
+#
+#   (NOTE: this file is the same for all configrations to be tested with sette)
+#
+#   all_functions.sh : loads functions used by sette (note: new functions can be added here)
+#   set_namelist     : function declared in all_functions that sets namelist parameters 
+#   post_test_tidyup : creates validation storage directory and copies required output files 
+#                      (solver.stat and ocean.output) in it after execution of test.
+#
+#  VALIDATION tree is:
+#
+#   NEMO_VALIDATION_DIR/WCONFIG_NAME/WCOMPILER_NAME/TEST_NAME/REVISION_NUMBER(or DATE)
+#
+#  prepare_exe_dir.sh : defines and creates directory where the test is executed
+#                       execution directory takes name of TEST_NAME defined for every test 
+#                       in sette.sh. (each test in executed in its own directory)
+#
+#  prepare_job.sh     : to generate the script run_job.sh
+#
+#  fcm_job.sh         : run in batch (INTERACT_FLAG="no") or interactive (INTERACT_FLAG="yes")
+#                        see sette.sh and BATCH_TEMPLATE directory
+#
+#  NOTE: jobs requiring initial or forcing data need to have an input_CONFIG.cfg in which 
+#        can be found paths to the input tar file)
+#  NOTE: if job is not launched for any reason you have the executable ready in ${EXE_DIR} 
+#        directory
+#  NOTE: the changed namelists are left in ${EXE_DIR} directory whereas original namelists 
+#        remain in ${NEW_CONF}/EXP00
+# 
+#  NOTE: a log file, output.sette, is created in ${SETTE_DIR} with the echoes of 
+#        executed commands
+#
+#  NOTE: if sette.sh is stopped in output.sette there is written the last command 
+#        executed by sette.sh
+#
+# example use: ./sette.sh 
+#########################################################################################
+#
+# Compiler among those in NEMOGCM/ARCH
+COMPILER=tobedefined
+export BATCH_COMMAND_PAR="llsubmit"
+export BATCH_COMMAND_SEQ=$BATCH_COMMAND_PAR
+export INTERACT_FLAG="no"
+export MPIRUN_FLAG="yes"
+#
+
+# Directory to run the tests
+SETTE_DIR=$(cd $(dirname "$0"); pwd)
+MAIN_DIR=${SETTE_DIR%/SETTE}
+CONFIG_DIR=${MAIN_DIR}/CONFIG
+TOOLS_DIR=${MAIN_DIR}/TOOLS
+COMPIL_DIR=${TOOLS_DIR}/COMPILE
+
+CMP_NAM=${1:-$COMPILER}
+# Copy job_batch_COMPILER file for specific compiler into job_batch_template
+cd ${SETTE_DIR}
+cp BATCH_TEMPLATE/batch-${COMPILER} job_batch_template || exit
+
+for config in 2  
+
+do
+
+# TESTS FOR GYRE CONFIGURATION
+if [ ${config} -eq 1 ] ;  then
+    ## Restartability tests for GYRE
+
+    NPROC=4
+    export NUM_XIOSERVERS=0
+    export USING_MPMD="false"
+
+    export TEST_NAME="ATTACHED_MULTIPLE"
+
+    cd ${SETTE_DIR}
+    . ../CONFIG/makenemo -m ${CMP_NAM} -n GYRE_XIOS_LR -r GYRE_XIOS -j 8 
+    cd ${SETTE_DIR}
+    . param.cfg
+    . all_functions.sh
+    . prepare_exe_dir.sh
+    JOB_FILE=${EXE_DIR}/run_job.sh
+    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
+    cd ${EXE_DIR}  
+    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_LR\"
+    set_namelist namelist_cfg nn_it000 1
+    set_namelist namelist_cfg nn_itend 120
+    set_namelist namelist_cfg nn_stock 60
+    set_namelist namelist_cfg ln_clobber .true.
+    set_namelist namelist_cfg jp_cfg 1
+    set_namelist namelist_cfg jpni 2
+    set_namelist namelist_cfg jpnj 2
+    set_namelist namelist_cfg jpnij $NPROC
+
+    set_xio_using_server iodef.xml $USING_MPMD
+    set_xio_file_type    iodef.xml multiple_file
+
+    cd ${SETTE_DIR}
+    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}
+
+    cd ${SETTE_DIR}
+    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}
+
+
+    export TEST_NAME="ATTACHED_ONE"
+    export NUM_XIOSERVERS=0
+    export USING_MPMD="false"
+    cd ${SETTE_DIR} 
+    . prepare_exe_dir.sh
+    JOB_FILE=${EXE_DIR}/run_job.sh
+    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
+    cd ${EXE_DIR}
+    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_LR\"
+    set_namelist namelist_cfg nn_it000 1
+    set_namelist namelist_cfg nn_itend 120
+    set_namelist namelist_cfg nn_stock 60
+    set_namelist namelist_cfg ln_clobber .true.
+    set_namelist namelist_cfg jp_cfg 1
+    set_namelist namelist_cfg jpni 2
+    set_namelist namelist_cfg jpnj 2
+    set_namelist namelist_cfg jpnij $NPROC
+
+    set_xio_using_server iodef.xml $USING_MPMD
+    set_xio_file_type    iodef.xml one_file
+
+    cd ${SETTE_DIR}
+    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}
+
+    cd ${SETTE_DIR}
+    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}
+
+
+    export TEST_NAME="DETACHED_ONE"
+    export NUM_XIOSERVERS=2
+    export USING_MPMD="true"
+    cd ${SETTE_DIR}
+    . prepare_exe_dir.sh
+    JOB_FILE=${EXE_DIR}/run_job.sh
+    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
+    cd ${EXE_DIR}
+    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_LR\"
+    set_namelist namelist_cfg nn_it000 1
+    set_namelist namelist_cfg nn_itend 120
+    set_namelist namelist_cfg nn_stock 60
+    set_namelist namelist_cfg ln_clobber .true.
+    set_namelist namelist_cfg jp_cfg 1
+    set_namelist namelist_cfg jpni 2
+    set_namelist namelist_cfg jpnj 2
+    set_namelist namelist_cfg jpnij $NPROC
+
+    set_xio_using_server iodef.xml $USING_MPMD
+    set_xio_file_type    iodef.xml one_file
+
+    cd ${SETTE_DIR}
+    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}
+
+    cd ${SETTE_DIR}
+    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}
+
+    export TEST_NAME="DETACHED_MULTIPLE"
+    export NUM_XIOSERVERS=2
+    export USING_MPMD="true"
+    cd ${SETTE_DIR}
+    . prepare_exe_dir.sh
+    JOB_FILE=${EXE_DIR}/run_job.sh
+    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
+    cd ${EXE_DIR}
+    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_LR\"
+    set_namelist namelist_cfg nn_it000 1
+    set_namelist namelist_cfg nn_itend 120
+    set_namelist namelist_cfg nn_stock 60
+    set_namelist namelist_cfg ln_clobber .true.
+    set_namelist namelist_cfg jp_cfg 1
+    set_namelist namelist_cfg jpni 2
+    set_namelist namelist_cfg jpnj 2
+    set_namelist namelist_cfg jpnij $NPROC
+
+    set_xio_using_server iodef.xml $USING_MPMD
+    set_xio_file_type    iodef.xml multiple_file
+
+    cd ${SETTE_DIR}
+    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}
+
+    cd ${SETTE_DIR}
+    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}
+
+fi
+
+if [ ${config} -eq 2 ] ;  then
+    ## Restartability tests for GYRE
+
+    NPROC=152
+    export NUM_XIOSERVERS=0
+    export USING_MPMD="false"
+
+    export TEST_NAME="ATTACHED_MULTIPLE"
+
+    cd ${SETTE_DIR}
+    . ../CONFIG/makenemo -m ${CMP_NAM} -n GYRE_XIOS_HR -r GYRE_XIOS -j 8
+    cd ${SETTE_DIR}
+    . param.cfg
+    . all_functions.sh
+    . prepare_exe_dir.sh
+    JOB_FILE=${EXE_DIR}/run_job.sh
+    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
+    cd ${EXE_DIR}
+    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_HR\"
+    set_namelist namelist_cfg nn_it000 1
+    set_namelist namelist_cfg nn_itend 120
+    set_namelist namelist_cfg nn_stock 120
+    set_namelist namelist_cfg nn_bench 1
+    set_namelist namelist_cfg ln_clobber .true.
+    set_namelist namelist_cfg jp_cfg 144
+    set_namelist namelist_cfg jpni 19
+    set_namelist namelist_cfg jpnj 8
+    set_namelist namelist_cfg jpnij $NPROC
+
+    set_xio_using_server iodef.xml $USING_MPMD
+    set_xio_file_type    iodef.xml multiple_file
+
+    cd ${SETTE_DIR}
+    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}
+
+    cd ${SETTE_DIR}
+    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}
+
+
+
+    export TEST_NAME="ATTACHED_ONE"
+    export NUM_XIOSERVERS=0
+    export USING_MPMD="false"
+    cd ${SETTE_DIR}
+    . prepare_exe_dir.sh
+    JOB_FILE=${EXE_DIR}/run_job.sh
+    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
+    cd ${EXE_DIR}
+    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_HR\"
+    set_namelist namelist_cfg nn_it000 1
+    set_namelist namelist_cfg nn_itend 120
+    set_namelist namelist_cfg nn_stock 120
+    set_namelist namelist_cfg nn_bench 1
+    set_namelist namelist_cfg ln_clobber .true.
+    set_namelist namelist_cfg jp_cfg 144
+    set_namelist namelist_cfg jpni 19
+    set_namelist namelist_cfg jpnj 8
+    set_namelist namelist_cfg jpnij $NPROC
+
+    set_xio_using_server iodef.xml $USING_MPMD
+    set_xio_file_type    iodef.xml one_file
+
+    cd ${SETTE_DIR}
+    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}
+
+    cd ${SETTE_DIR}
+    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}
+
+    export TEST_NAME="DETACHED_ONE"
+    export NUM_XIOSERVERS=8
+    export USING_MPMD="true"
+    cd ${SETTE_DIR}
+    . prepare_exe_dir.sh
+    JOB_FILE=${EXE_DIR}/run_job.sh
+    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
+    cd ${EXE_DIR}
+    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_HR\"
+    set_namelist namelist_cfg nn_it000 1
+    set_namelist namelist_cfg nn_itend 120
+    set_namelist namelist_cfg nn_stock 120
+    set_namelist namelist_cfg nn_bench 1
+    set_namelist namelist_cfg ln_clobber .true.
+    set_namelist namelist_cfg jp_cfg 144
+    set_namelist namelist_cfg jpni 19
+    set_namelist namelist_cfg jpnj 8
+    set_namelist namelist_cfg jpnij $NPROC
+
+    set_xio_using_server iodef.xml $USING_MPMD
+    set_xio_file_type    iodef.xml one_file
+
+    cd ${SETTE_DIR}
+    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}
+
+    cd ${SETTE_DIR}
+    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}
+
+    export TEST_NAME="DETACHED_MULTIPLE"
+    export NUM_XIOSERVERS=8
+    export USING_MPMD="true"
+    cd ${SETTE_DIR}
+    . prepare_exe_dir.sh
+    JOB_FILE=${EXE_DIR}/run_job.sh
+    if [ -f ${JOB_FILE} ] ; then \rm ${JOB_FILE} ; fi
+    cd ${EXE_DIR}
+    set_namelist namelist_cfg cn_exp \"GYRE_XIOS_HR\"
+    set_namelist namelist_cfg nn_it000 1
+    set_namelist namelist_cfg nn_itend 120
+    set_namelist namelist_cfg nn_stock 120
+    set_namelist namelist_cfg nn_bench 1
+    set_namelist namelist_cfg ln_clobber .true.
+    set_namelist namelist_cfg jp_cfg 144
+    set_namelist namelist_cfg jpni 19
+    set_namelist namelist_cfg jpnj 8
+    set_namelist namelist_cfg jpnij $NPROC
+
+    set_xio_using_server iodef.xml $USING_MPMD
+    set_xio_file_type    iodef.xml multiple_file
+
+    cd ${SETTE_DIR}
+    . ./prepare_job.sh input_GYRE.cfg $NPROC ${TEST_NAME} ${MPIRUN_FLAG} ${JOB_FILE} ${NUM_XIOSERVERS}
+
+    cd ${SETTE_DIR}
+    . ./fcm_job.sh $NPROC ${JOB_FILE} ${INTERACT_FLAG} ${MPIRUN_FLAG}
+
+fi
+
+done
