Changeset 5455

Timestamp:

2015-06-19T22:57:06+02:00 (9 years ago)

Author:

nicolasmartin

Message:

dev_r5092_CNRS_SETTE Ping pong updates between Curie & Ada to finalize

Location:

branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST

Files:

• trusting.sh (modified) (9 diffs)
• trusting_func.sh (modified) (6 diffs)

branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh

@@ -1 +1 @@
 #!/bin/bash
-#
-# Author: Patrick Brockmann
-# Contact: Patrick.Brockmann@cea.fr
-# History:
-# Modification: Nicolas-thierry.Martin@locean-ipsl.upmc.fr
-#
 
 
@@ -11 +5 @@
 #---------------------------------------------------
 NEMO_TRUS=$( pwd $( dirname $0 ) )
-DEBUG=0; PUBLISH=0; STATUS='OK'
+DEBUG=0; PUBLISH=0
+STATUS='FAILED'; STEP=''# Only at end status will change to OK
 
 # Get options for replacing some initials settings
@@ -51 +46 @@
 datestr1=$( LC_ALL=C date -d"${DATE_STR}" +%Y-%m-%dT%H:%M )
 printf "Date\n$datestr1\n" > mesg_01_date_${CFG_USER}_${CFG_ARCH}.txt
-# Only at end status will change to OK
-STATUS='FAILED'; STEP=''
 
 # SVN action on XIOS & NEMO essentials directories
 #---------------------------------------------------
-print_step "Get NEMO SVN state with: ${SVN_OPT}"
+print_step 'Get NEMO code state / repository'
 get_nemo_rev
 
@@ -70 +63 @@
 print_step 'Compiling XIOS'
 cd ${DIR_XIOS}
-#./make_xios --arch $ARCH -job $NPROC >& /dev/null
+./make_xios --full --arch $ARCH -job $NPROC #>& /dev/null
+[ ! -e ${DIR_XIOS}/lib/libxios.a ] && get_out $STATUS $STEP
 
 # NEMO config compiled from scratch
@@ -76 +70 @@
 print_step "Compiling ${CONF_REF} configuration"
 cd ${NEMO_CONF}
-#[ -d ${TEST_CONF} ] && ./makenemo -n ${TEST_CONF} clean
-./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m $ARCH -j $NPROC >& /dev/null
+[ -d ${TEST_CONF} ] && ./makenemo -n ${TEST_CONF} clean
+./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m $ARCH -j $NPROC #>& /dev/null
 [ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out $STATUS $STEP
 
 # Get namelists, xml & forced files for running
 #---------------------------------------------------
-print_step 'Preparing job'
+print_step "Preparing job by copying ${CONF_REF} input files"
 cd ${TEST_DIR}
 find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(cfg\|opa\|ref\|xml\)' -exec cp {} . \;
@@ -94 +88 @@
 ${cmd_iol} > inputs_list.txt && ${cmd_iof} > /dev/null
 [ $? -ne 0 ] && get_out $STATUS $STEP
+
 [ $( find . -name '*.gz' -print -quit ) ] && gunzip *.gz
 
 # Check input files in all cases
 #---------------------------------------------------
-print_step 'Test input files difference'
+print_step 'Difference between input files'
 diff_inputs
 
 # Job submission & computing
 #---------------------------------------------------
-print_step 'Running'
+print_step 'Submitting job'
 JOB_ID=$( ${JOB_SUBM} | awk '{print $NF}' )
-export JOB_ID
 [ $? -ne 0 ] && get_out $STATUS $STEP
 
+print_step 'Pending/Running job'
 time_elapsed=0; time_increment=30
 sleep ${time_increment}
@@ -118 +113 @@
 sleep ${time_increment}
 
+if (( ${time_elapsed} = ${TIME_LIMI} )); then
+    print_step 'Killing job'
+    ${JOB_DELE} ${JOB_ID} &> /dev/null
+    get_out $STATUS 'Job timeout'
+fi
+print_step 'Job finished'
+
 # Eventual crash during computing
 #---------------------------------------------------
-print_step 'Test if ASCII output files exist'
+print_step 'Test if job crashed'
 if   [[ ! -e ocean.output || $( grep 'E R R O R' ocean.output ) ]]; then
-    ${JOB_DELE} ${JOB_ID} &> /dev/null
-    [ ! -e ocean.output                  ] && STEP='nemo.exe crashed at initialization'
-    [ $( grep 'E R R O R' ocean.output ) ] && STEP='nemo.exe crashed'
     comments 'E R R O R'
-    ${JOB_DELE} ${JOB_ID} >/dev/null 2>&1
     get_out $STATUS $STEP
 else
@@ -136 +134 @@
     for file in ocean.output solver.stat tracer.stat; do
    diff $file ${REFE_DIR}/$file
-   [ $? -ne 0 ] && STATUS='FAILED' && STEP='ASCII output file differ'
+   [ $? -ne 0 ] && get_out 'FAILED' 'ASCII output file differ'
     done
 
@@ -142 +140 @@
     #---------------------------------------------------
     print_step 'Test last restart NetCDF files diff'
-    diff_restart #2> /dev/null
-    [ $? -ne 0 ] && STATUS='FAILED'
+    diff_restart
+    [ $? -ne 0 ] && get_out 'FAILED' "$CDO diff failed"
 
     # Get computation duration

branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting_func.sh

@@ -3 +3 @@
 
 comments() {
-    [ -e ocean.output ] && line=$( grep $1 ocean.output )
+    state=$1
 
-    # Check environment architecture modification
-    #---------------------------------------------------
-    [ $( diff -q env.log ${REFE_DIR}/env.log ) ] && line=$( diff env.log ${REFE_DIR}/env.log )
+    if [ $state == 'E R R O R' ]; then
+   if [ -e ocean.output ]; then
+       last_time_step=$( cat ${TEST_DIR}/time.step | tr -d [:space:] )
+       STEP='nemo.exe crashed at '${last_time_step} && export STEP
+       line=$( grep -m1 "$state" ocean.output | tr -d '\n' )
+   else
+       STEP='nemo.exe crashed at initialization'    && export STEP
+       return
+   fi
+    fi
+
+    if [ $( diff -q env.log ${REFE_DIR}/env.log ) ]; then
+   line=$( diff env.log ${REFE_DIR}/env.log  | tr -d '\n' )
+    fi
 
     printf "Comments\n$line\n" | tee mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt
@@ -47 +58 @@
        cmp ${TEST_DIR}/${base_name}_$comp.nc ${REFE_DIR}/${base_name}_$comp.nc
        if [ $? -ne 0 ]; then
-      export STEP='Restart files different'
+      STATUS='FAILED'; STEP='Restart files different'
       printf "$CDO diffn\n"
       $CDO diffn ${TEST_DIR}/${base_name}_$comp.nc ${REFE_DIR}/${base_name}_$comp.nc 2> /dev/null
@@ -58 +69 @@
 
     else
+   STATUS='FAILED'; STEP='No restart files found at last time step'
    printf "no restart files found for comparison\n"
     fi
+
+    [ $STATUS == 'FAILED'] && get_out $STATUS $STEP
 }
 
@@ -93 +107 @@
    # For time being, just get revision number from XIOS
    if [ $dir == ${DIR_XIOS} ]; then
-       echo 'XIOS '$( svn info $dir | awk '(NR == 9) {print $NF}' ) >> env.log
+       echo 'XIOS '$( svn info $dir | awk '(NR == 5) {print $NF}' ) >> env.log
        continue
    fi
@@ -119 +133 @@
 
   # Send mail only when FAILED
-  if [[ ! -z $MAIL && "$STATUS"=='FAILED' ]]; then
+  if [[ ! -z $MAIL && $STATUS == 'FAILED' ]]; then
       cat << END_MAIL > trusting.mail
 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
@@ -144 +158 @@
 
     # Save tested configuration if trusting failed
-    if [ "$STATUS" == 'FAILED' ]; then
+    if [ $STATUS == 'FAILED' ]; then
+   cd ${TEST_DIR}
    touch mesg_08_inputfiles_${CFG_USER}_${CFG_ARCH}.txt mesg_09_realcputime_${CFG_USER}_${CFG_ARCH}.txt \
          mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt