Index: branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh
===================================================================
--- branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh	(revision 5292)
+++ branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh	(revision 5355)
@@ -9,63 +9,78 @@
 
 # Set defaults
+#---------------------------------------------------
+NEMO_TRUS=$( pwd $( dirname $0 ) )
 DEBUG=0; PUBLISH=0; status='OK'
 
-# Get options to replace some initials settings
+# Get options for replacing some initials settings
+#---------------------------------------------------
 while [ $# -ne 0 ]; do
     case $1 in
         -b|--branch ) NEMO_BRAN=$2         ; shift 2;; -d|--debug  ) set -vx; DEBUG=1     ; shift  ;;
 	-e|--email  ) MAIL=$2              ; shift 2;; -j|--job    ) NPROC=$2             ; shift 2;;
-	-h|--help   ) cat trusting_help.txt; exit   ;; -m|--machine) CFG_ARCH=$2          ; shift 2;;
-	-n|--newconf) CONF_TEST=$2         ; shift 2;; -r|--refconf) CONF_REFE=$2         ; shift 2;;
+	-h|--help   ) cat trusting_help.txt; exit  1;; -m|--machine) CFG_ARCH=$2          ; shift 2;;
+	-n|--newconf) TEST_CONF=$2         ; shift 2;; -r|--refconf) REFE_CONF=$2         ; shift 2;;
 	-t|--time   ) TIME_LIMI=$2         ; shift 2;; -p|--publish) PUBLISH=1            ; shift  ;;
         -u|--user   ) CFG_USER=$2          ; shift 2;; -v|--version) NEMO_REVI=$2         ; shift 2;;
-                   *) cat trusting_help.txt; exit   ;;
+                   *) cat trusting_help.txt; exit  1;;
     esac
 done
 
-# Set super-computer & user environment
-[[ ! -z "${CFG_ARCH}" && ! -z "${CFG_USER}" ]] && . trusting.env || { ./trusting.sh -h && exit; }
-export DEBUG PUBLISH
-
-# Initialization
-#=============================================
-
-echo ${DIR_TEST}
-mkdir -p ${DIR_TEST} ${DIR_REFE}
-cd ${DIR_TEST}
-datestr1=$( LC_ALL=C date -d"$DATE_STR" +%Y-%m-%dT%H:%M )
-printf "Date\n%s\n"       $datestr1 > mesg_01_date_${CFG_USER}_${CFG_ARCH}.txt
-
-printf "Status\nFAILED\n"           > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt # Only at end status will change to OK
-
-. trusting_func.sh
-#=============================================
+#export DEBUG PUBLISH
 
 
-# 
-#=============================================
+# Initialization (super-computer & user environment)
+#---------------------------------------------------
+if [[ ! -z "${CFG_ARCH}" && ! -z "${CFG_USER}" ]]; then
+
+    if   [ ! $( find ${NEMO_TRUS}/cfg -name ${CFG_USER}.cfg ) ]; then
+	echo 'No configuration file in '${NEMO_TRUS}'/cfg for '${CFG_USER}; exit 1
+    elif [ ! $( find ${NEMO_TRUS}/cfg -name ${CFG_ARCH}.cfg ) ]; then
+	echo 'No configuration file in '${NEMO_TRUS}'/cfg for '${CFG_ARCH}; exit 1
+    else
+	. ${NEMO_TRUS}/trusting.env
+    fi
+
+else
+    cat ${NEMO_TRUS}/trusting_help.txt; exit 1
+fi
+
+datestr1=$( LC_ALL=C date -d"${DATE_STR}" +%Y-%m-%dT%H:%M )
+. ./trusting_func.sh
+
+mkdir -p ${TEST_DIR} ${REFE_DIR}
+echo ${TEST_DIR} && cd ${TEST_DIR}
+printf "Date\n%s\n"       $datestr1 > mesg_01_date_${CFG_USER}_${CFG_ARCH}.txt
+# Only at end status will change to OK
+printf "Status\nFAILED\n"           > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt
+
+
+# Check softwares versions
+#---------------------------------------------------
+printf "Step......................\nGet softwares release\n"                     | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
 for rel in $CDO $COMPILER $MPI $NETCDF; do
-    printf "Step......................\nGet $rel release\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
                              arch_rel=$( echo $LOADEDMODULES | sed "s#.*:$rel/\([^:]*\).*#\1#" )
     [ -z "${arch_rel}"  ] && arch_rel=$( echo $PATH          | sed "s#.*$rel/\([^/]*\).*#\1#"  )
-    [ $( echo $rel | grep "$CDO\|$COMPILER" ) ] && arch_rel=$( $rel --version | grep -m1 -o ' [0-9.]* ' )
+#    [ $( echo $rel | grep $COMPILER ) ] && arch_rel=$( $rel --version | grep -m1 -o ' [0-9.]* ' >/dev/null )
     echo $rel ${arch_rel} >> model.log
 done
 
-sed -n 1p model.log | sed 'i\Fortran compiler' > mesg_04_compiler_${CFG_USER}_${CFG_ARCH}.txt
-sed -n 2p model.log | sed 'i\MPI libraries   ' > mesg_05_mpi_${CFG_USER}_${CFG_ARCH}.txt
-sed -n 3p model.log | sed 'i\NetCDF libraries' > mesg_06_netcdf_${CFG_USER}_${CFG_ARCH}.txt
+sed -n 2p model.log | sed 'i\Fortran compiler' > mesg_04_compiler_${CFG_USER}_${CFG_ARCH}.txt
+sed -n 3p model.log | sed 'i\MPI libraries   ' > mesg_05_mpi_${CFG_USER}_${CFG_ARCH}.txt
+sed -n 4p model.log | sed 'i\NetCDF libraries' > mesg_06_netcdf_${CFG_USER}_${CFG_ARCH}.txt
 
 
-#---------------------------------------------
-printf "Step......................\nCheckout XIOS, FCM-make(/EXTERNAL/fcm), NEMO engines (/NEMO) & configuration (/CONFIG)\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
+# SVN action on XIOS & NEMO essentials directories
+#---------------------------------------------------
 svn_cmd='svn status'
-
 # If -v|--version option has been set, modify svn command
 [ $( echo ${NEMO_VERS} | grep  "up\|update"         ) ] && svn_cmd='svn update'
 [ $( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) ] && svn_cmd='svn update -r '$( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' )
 
+printf "Step......................\nGet NEMO SVN revision with: ${svn_cmd}\n"    | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
 rev=0
+
 for dir in ${DIR_XIOS} ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_EFCM} ${NEMO_ENGI} ${NEMO_IPSL} ${NEMO_TRUS} ${NEMO_TCMP}; do
+    # For time being, just get revision number from XIOS
     [ $dir == ${DIR_XIOS} ] && echo 'XIOS '$( svn info $dir | awk '(NR == 9) {print $NF}' ) >> model.log && continue
     ${svn_cmd} $dir
@@ -76,22 +91,24 @@
 
 
-#---------------------------------------------
-printf "Step......................\nCompilation NEMO configuration %s\n" ${CONF_REF} > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
+# NEMO config compiled from scratch
+#---------------------------------------------------
+printf "Step......................\nCompiling ${CONF_REF} configuration\n"       | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
 cd ${NEMO_CONF}
-. ${ARCH_ENV}     # Sourcing architecture environment
-#[ -d ${CONF_TEST} ] && ./makenemo -n ${CONF_TEST} clean
-./makenemo -n ${CONF_TEST} -r ${CONF_REFE} -m $ARCH -j $NPROC #> /dev/null 2>&1
-[ ! -e ${CONF_TEST}/BLD/bin/nemo.exe ] && getout
+. ${ARCH_ENV} > /dev/null    # Sourcing architecture environment
+#[ -d ${TEST_CONF} ] && ./makenemo -n ${TEST_CONF} clean
+./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m $ARCH -j $NPROC > /dev/null 2>&1
+[ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out
 
 
-#---------------------------------------------
-printf "Step......................\nPreparing job\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
-cd ${DIR_TEST}
-find ${NEMO_CONF}/${CONF_TEST}/EXP00 -regex '.*\(cfg\|opa\|ref\|xml\)' -exec cp {} . \;
+# Get namelists, xml & forced files for running
+#---------------------------------------------------
+printf "Step......................\nPreparing job\n"                             | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
+cd ${TEST_DIR}
+find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(cfg\|opa\|ref\|xml\)' -exec cp {} . \;
 
 if [ ! -z "${NEMO_FORC}" ]; then
     cp ${NEMO_FORC}/${FORC_TAR} .
-    tar -tvf ${FORC_TAR} > inputs_list.txt && tar -vxf ${FORC_TAR}
-    [ $? -ne 0 ] && getout
+    tar -tvf ${FORC_TAR} > inputs_list.txt && tar -vxf ${FORC_TAR} > /dev/null
+    [ $? -ne 0 ] && get_out
     gunzip *.gz
 else
@@ -100,13 +117,13 @@
 
 
-#---------------------------------------------
-printf "Step......................\nRunning\n"       > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
-#${JOB_SUB} ${DIR_TEST}/../../batch.sh
-job_num=$( ${JOB_SUB} ${DIR_TEST}/../../batch.sh | awk '{print $NF}')
-[ $? -ne 0 ] && getout
+# Job submission & computing
+#---------------------------------------------------
+printf "Step......................\nRunning\n"                                   | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt; echo ${JOB_SUBM}
+job_num=$( ${JOB_SUBM} | awk '{print $NF}' )
+[ $? -ne 0 ] && get_out
 
-time_elapsed=0; time_increment=300
+time_elapsed=0; time_increment=30
 sleep ${time_increment}
-while [[ $( ${JOB_LIST} | grep $job_num ) && ${time_elapsed} -lt ${TIME_LIMI} ]] ; do
+while [[ $( ${JOB_LIST} | grep $job_num ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do
     printf "\n####################################################" >> computation.log
     ${JOB_VIEW} $job_num                                            >> computation.log
@@ -116,52 +133,67 @@
 
 
-#---------------------------------------------
-printf "Step......................\nTest if ASCII output files exist\n"   > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
-if      [ ! -f ${output_test} ]; then
-
-    getout
-else if [  ]
-
-    grep 'E R R O R' ocean.output    >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
-    sed 's/^/Time step :/' time.step >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
+# Eventual crash during computing
+#---------------------------------------------------
+printf "Step......................\nTest if ASCII output files exist\n"          | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
+if   [ ! -f ocean.output                  ]; then
+    printf "Step......................\nOpa crashed at initialization\n"         | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
+    status='FAILED'
+elif [ $( grep 'E R R O R' ocean.output ) ]; then
+    printf "Step......................\nOpa crashed\n"                           | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
+    grep 'E R R O R'       ocean.output >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
+    sed 's/^/Time step :/' time.step    >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
+    status='FAILED'
 fi
 
 
-#---------------------------------------------
-printf "Step......................\nTest ASCII output files difference\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
+# Inspect output text files
+#---------------------------------------------------
+printf "Step......................\nTest ASCII output files difference\n"        | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
 for file in ocean.output solver.stat tracer.stat; do
-    diff $file ${DIR_REFE}/$file
+    diff $file ${REFE_DIR}/$file
     [ $? -ne 0 ] && status='FAILED'
 done
 
 
-#---------------------------------------------
-printf "Step......................\nTest last restart files difference\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
-diff_trusting 2>/dev/null
+# Inspect output NetCDF files
+#---------------------------------------------------
+printf "Step......................\nTest last restart NetCDF files difference\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
+diff_restart 2>/dev/null
 [ $? -ne 0 ] && status='FAILED'
 
 
-#---------------------------------------------
+# Comments
+#---------------------------------------------------
 echo 'Comments'                    >  mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
 grep 'W A R N I N G:' ocean.output >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
 
 
-#---------------------------------------------
+# Finals tests
+#---------------------------------------------------
 if [ $status != 'FAILED' ]; then
-    printf "Step......................\nGet Real CPU time\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
-    real_cpu_time=$( ${JOB_VIEW} $job_num | awk 'END{print $9}')
+    printf "Step......................\nGet real CPU time\n"                     | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
+    real_cpu_time=$( ${JOB_VIEW} $job_num | awk 'END{print $9}' )
     printf "Real CPU time\n%s\n" ${real_cpu_time} > mesg_08_realcputime_${CFG_USER}_${CFG_ARCH}.txt
 else
-    printf "Step......................\nTest InputFiles difference\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
-    diff inputs_list.txt ${DIR_REFE}/inputs_list.txt
-    [ $? -ne 0 ] && getout
-    [[ $? -ne 0 && printf "InputFiles\n%s\n" "Different" > mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt ]] || printf "InputFiles\n%s\n" "Same" > mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt
+    printf "Step......................\nTest input files difference\n"           | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
+    diff inputs_list.txt ${REFE_DIR}/inputs_list.txt
+
+    if [ $? -ne 0]; then
+	printf "Input files\nDifferent" > mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt
+    else
+	printf "Input files\nSame"      > mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt
+    fi
+
+    get_out
 fi
 
 
-#---------------------------------------------
+# So far, everything is good
+#---------------------------------------------------
 printf "Status\nOK\n" > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt
-printf "Step......................\nCode is reliable\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
+printf "Step......................\nCode is reliable\n"                          | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
 
-#---------------------------------------------
-getout
+
+# End
+#---------------------------------------------------
+get_out
