Changeset 4373 for branches/2013/dev_MERGE_2013

Timestamp:

2014-01-26T10:51:51+01:00 (10 years ago)

Author:

epico

Message:

new SETTE test for testing XIOS in its 4 modes Attached/Detached?, One/Multiple? with both Low and High resolution.
A modification of BATCH file for each machine is required to enable MPMD jobs. See batch-gfortran_athena_xios as example

Location:

branches/2013/dev_MERGE_2013/NEMOGCM

Files:

• CONFIG/GYRE_XIOS (added)
• CONFIG/GYRE_XIOS/EXP00 (added)
• CONFIG/GYRE_XIOS/EXP00/AA_job (added)
• CONFIG/GYRE_XIOS/EXP00/domain_def.xml (added)
• CONFIG/GYRE_XIOS/EXP00/field_def.xml (added)
• CONFIG/GYRE_XIOS/EXP00/iodef.xml (added)
• CONFIG/GYRE_XIOS/EXP00/namelist_cfg (added)
• CONFIG/GYRE_XIOS/EXP00/namelist_ref (added)
• CONFIG/GYRE_XIOS/cpp_GYRE_XIOS.fcm (added)
• CONFIG/cfg.txt (modified) (1 diff)
• SETTE/BATCH_TEMPLATE/batch-gfortran_athena_xios (modified) (4 diffs)
• SETTE/param.cfg (modified) (2 diffs)
• SETTE/prepare_exe_dir.sh (modified) (1 diff)
• SETTE/prepare_job.sh (modified) (1 diff)
• SETTE/sette_xios.sh (added)

branches/2013/dev_MERGE_2013/NEMOGCM/CONFIG/cfg.txt

@@ -3 +3 @@
 ORCA2_LIM_CFC_C14b OPA_SRC LIM_SRC_2 NST_SRC TOP_SRC
 GYRE OPA_SRC
+GYRE_XIOS OPA_SRC
 ORCA2_OFF_PISCES OPA_SRC OFF_SRC TOP_SRC
 ORCA2_SAS_LIM OPA_SRC SAS_SRC LIM_SRC_2 NST_SRC

branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/BATCH_TEMPLATE/batch-gfortran_athena_xios

@@ -2 +2 @@
 #!
 #BSUB -q poe_short
-#BSUB -n NPROCS
+#BSUB -n TOTAL_NPROCS
 #BSUB -a poe
 #BSUB -J MPI_config
@@ -15 +15 @@
 #
   OCEANCORES=NPROCS
+  XIOS_NUMPROCS=NXIOPROCS
   export SETTE_DIR=DEF_SETTE_DIR
 
@@ -27 +28 @@
 
 export MPIRUN="mpirun.lsf"
+export LANG=en_US
+export NLSPATH=/opt/ibmhpc/pecurrent/base/msg/%L/%N:$NLSPATH
+export MP_LABELIO=yes
+export MP_STDOUTMODE=unordered
+export MP_EUILIB=us
+export MP_INFOLEVEL=0
+export MP_EUIDEVELOP=min
+export MP_INSTANCES=1
+export MP_SHARED_MEMORY=yes
+export MP_EAGER_LIMIT_LOCAL=2000000000
+export MP_EUIDEVICE=sn_single
+export MP_BULK_MIN_MSG_SIZE=16384
+export MP_POLLING_INTERVAL=1000000
+export MP_DEVTYPE=ib
+export MP_EAGER_LIMIT=262144
+export MP_TASK_AFFINITY=core
+export MP_BINDPROC=yes
+export MP_DEBUG_CHECK_PARAM=no
+export MP_MSG_API=mpi
+export MP_USE_BULK_XFER=yes
+export MP_WAIT_MODE=poll
+export LDR_CNTRL=TEXTPSIZE=64K@STACKPSIZE=64K@DATAPSIZE=64K
+export LD_LIBRARY_PATH=/users/home/ans040/local/lib:${LD_LIBRARY_PATH}
+
+XIOS_SERVER_PATHNAME="/users/home/ans040/SOFTWARE/XIOS/trunk/bin/xios_server.exe"
+
 
 #
@@ -68 +95 @@
 
   if [ MPI_FLAG == "yes" ]; then
-     time ${MPIRUN} ./opa
+     if [ $XIOS_NUMPROCS -eq 0 ]; then
+       time ${MPIRUN} ./opa
+     else
+
+       export MP_PGMMODEL=mpmd
+       export MP_CMDFILE=xioscmdfile
+       rm xioscmdfile &> /dev/null
+       touch xioscmdfile
+       k=0
+       while (( k < $OCEANCORES )) ; do
+          echo ./opa >> xioscmdfile
+          k=`expr $k + 1`
+       done
+       k=0
+       while (( k < $XIOS_NUMPROCS )) ; do
+          echo $XIOS_SERVER_PATHNAME >> xioscmdfile
+          k=`expr $k + 1`
+       done
+
+       time ${MPIRUN}
+     fi
   else
      time ./opa

branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/param.cfg

@@ -1 +1 @@
 #- forcing files storing 
-FORCING_DIR=${WORKDIR}/FORCING
+FORCING_DIR=~/FORCING
 #- input files storing 
 INPUT_DIR=${CONFIG_DIR}/${NEW_CONF}/EXP00
@@ -6 +6 @@
 #TMPDIR=${CONFIG_DIR}/${NEW_CONF}/EXP00
 #- VALIDATION files storing 
-NEMO_VALIDATION_DIR=${WORKDIR}/NEMO_VALIDATION
+NEMO_VALIDATION_DIR=~/NEMO_VALIDATION

branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/prepare_exe_dir.sh

@@ -71 +71 @@
 
 cp -r ${CONFIG_DIR}/${NEW_CONF}/EXP00/* ${EXE_DIR}/.
-cp -r ${SETTE_DIR}/iodef_sette.xml ${EXE_DIR}/iodef.xml
+#cp -r ${SETTE_DIR}/iodef_sette.xml ${EXE_DIR}/iodef.xml
 cd ${EXE_DIR}

branches/2013/dev_MERGE_2013/NEMOGCM/SETTE/prepare_job.sh

@@ -231 +231 @@
 # Pass settings into job file by using sed to edit predefined strings
 #
+        TOTAL_NPROCS=$(( $NB_PROC + $NXIO_PROC ))
         cat ${SETTE_DIR}/job_batch_template | sed -e"s/NODES/${NB_NODES}/" \
+             -e"s/TOTAL_NPROCS/${TOTAL_NPROCS}/" \
              -e"s/NPROCS/${NB_PROC}/" \
              -e"s/NXIOPROCS/${NXIO_PROC}/" \