source: branches/2015/dev_r5092_CNRS18_TRUST/NEMOGCM/TRUST/trusting_func.sh @ 5788

#!/bin/bash


## Messenger filenames
FILE_DATE=mesg_01_date_$PATTERNAME.txt  ; FILE_TRUS_RSLT=mesg_02_result_$PATTERNAME.txt
FILE_STAT=mesg_03_state_$PATTERNAME.txt ; FILE_NEMO=mesg_04_nemo_$PATTERNAME.txt
FILE_XIOS=mesg_05_xios_$PATTERNAME.txt  ; FILE_CMPF=mesg_06_compiler_$PATTERNAME.txt
FILE_LMPI=mesg_07_mpi_$PATTERNAME.txt   ; FILE_NCDF=mesg_08_netcdf_$PATTERNAME.txt
FILE_INPT=mesg_09_inputs_$PATTERNAME.txt; FILE_TIME=mesg_10_time_$PATTERNAME.txt
FILE_MEMY=mesg_11_memory_$PATTERNAME.txt; FILE_NOTE=mesg_12_comments_$PATTERNAME.txt

## Timestamped logfile & archive filenames
FILE_TRUS=trusting_${DATE}_$PATTERNAME.txt; FILE_ARCH=trusting_${DATE}_$PATTERNAME.tgz


## Functions in order of use
print_step() {
    local char_nb=$( echo "$1" | wc -c )
    local outline=$( printf "%${char_nb}s" )

    printf "\nStep.....\n%s\n%s\n" "$1" ${outline// /-}
}

init_files() {
    echo 'Date'               > ${FILE_DATE}; echo 'Result'           > ${FILE_TRUS_RSLT}
    echo 'State'              > ${FILE_STAT}; echo 'NEMOGCM rev.'     > ${FILE_NEMO}
    echo 'XIOS rev.'          > ${FILE_XIOS}; echo 'Fortran compiler' > ${FILE_CMPF}
    echo 'MPI libs'           > ${FILE_LMPI}; echo 'NetCDF libs'      > ${FILE_NCDF}
    echo 'Input files'        > ${FILE_INPT}; echo 'Elapsed time'     > ${FILE_TIME}
    echo 'Memory usage (P/V)' > ${FILE_MEMY}; echo 'Comments'         > ${FILE_NOTE}

    ## 'Failed' status with 'Unknown error' by default
    echo $TRUS_RSLT           \
   >> ${FILE_TRUS_RSLT}
    echo 'Unknown error' \
   >> ${FILE_STAT}
}

get_date() {
    ## UTC time zone for timestamping
    local dat=$( date -ud "${DATE}" +"%F %R %Z" )

    echo $dat           \
   >> ${FILE_DATE}
}

get_nemo_rev() {
    local dir rev_loc
    local rev=0 list=( 'ARCH CONFIG NEMO EXTERNAL/AGRIF EXTERNAL/IOIPSL EXTERNAL/fcm TOOLS/COMPILE TOOLS/REBUILD_NEMO' )

    ## Loop on essential NEMO directories
    for dir in $list ${DIR_XIOS}; do

   ## For time being, just get revision from XIOS with no action on directory
   if [ $dir == ${DIR_XIOS} ]; then
       rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' )
       echo 'XIOS '${rev_loc} \
      >> model.log
       echo "<a href=\"https://forge.ipsl.jussieu.fr/ioserver/changeset/${rev_loc}\" target=\"_blank\">${rev_loc}</a>" \
      >> ${FILE_XIOS}
       continue
   fi

   echo $dir && ${SVN_CMD} ${TRUS_WKCY}/$dir
   rev_loc=$( svn info ${TRUS_WKCY}/$dir | awk '(NR == 9) {print $NF}' )

   ## Keep last rev. nb
   [ ${rev_loc} -gt $rev ] && rev=${rev_loc}
    done

    echo 'NEMOGCM '$rev \
   >> model.log
    echo "<a href=\"https://forge.ipsl.jussieu.fr/nemo/changeset/$rev\" target=\"_blank\">$rev</a>" \
   >> ${FILE_NEMO}
}

get_soft_rel() {
    local soft soft_rel

    ## Sourcing environment modulefile only if module function is set
    [[ -e ${ARCH_ENV} && $( declare -F | grep ' module' ) ]] && . ${ARCH_ENV}

    for soft in $CMPF ${STR_CDOD} ${STR_LMPI} ${STR_NCDF}; do
   soft_rel=''

   ## Software release: next word after "$soft" in $PATH (case-insensitive)
   soft_rel=$( echo $PATH | sed "s#.*$soft\([0-9.a-z_]*\).*#\1#i" )

   ## option --version would work for main compilers (gfortran, intel, pgfortran, ...)
   [ $soft == $COMPILER ] && soft_rel=$( $soft --version | grep -m1 -oe '\<[0-9. ]*\>' )

   ## Cleaning characters string to display proper soft name
   soft=$( echo $soft | sed 's#\\##g; s#[/-]$##' )

   echo $soft ${soft_rel} \
       >> model.log
    done

    sed -n 4p model.log \
   >> ${FILE_CMPF}
    sed -n 5p model.log \
   >> ${FILE_LMPI}
    sed -n 6p model.log \
   >> ${FILE_NCDF}
}

get_inputs() {
    # List archive content & extract it by default
    local cmd_iol="tar -tvf ${TRUS_FORC}/${TRUS_TARF}" cmd_iof="tar -vxf ${TRUS_FORC}/${TRUS_TARF}"

    ## List & copy files in case of personal inputs
    [ -z "${TRUS_TARF}" ] && { cmd_iol="ls ${TRUS_FORC}/*"; cmd_iof="\cp ${TRUS_FORC}/* ."; }

    ${cmd_iol} > inputs_list.txt
    ${cmd_iof} > /dev/null
}

diff_inputs() {
    local dif file
    local files_list='' mesg='Same' 

    ## Simple diff
    for file in 'inputs_list.txt' *namelist_* *.xml cpp_*; do
   dif=''

   ## Continue even if input file is not in here (see after)
   if [ -e ${TRUS_BHMK}/$file ]; then dif=$( diff -q $file ${TRUS_BHMK}/$file ); else dif=0; fi

   ## Pass over useless file omission in benckmark directory
   [[ -n "$dif" && "$dif" != '0' ]] && ( mesg='Different'; echo $dif; files_list+=$file' ' )
    done

    [ $mesg == 'Same' ] && echo $mesg
    echo $mesg          \
   >> ${FILE_INPT}

    ## List different files for web comment
    [ -n "${files_list}" ] && echo 'Inputs  : '${files_list}'differ<br>' \
   >> temp_${FILE_NOTE}
}

job_pending() {
    local outline=$( printf "%100s" ) time_elapsed=0 time_increment=30

    sleep ${time_increment}

    ## Append a log file while pending
    while [[ $( eval ${JOB_STAT} ) && ${time_elapsed} -lt $TIMEOUT ]]; do
   printf "\n%s\n" ${outline// /#} \
       >> computation.log
   eval ${JOB_INFO}                \
       >> computation.log
   sleep ${time_increment}
   time_elapsed=$(( ${time_elapsed} + ${time_increment} ))
    done

    sleep ${time_increment}

    ## Kill remaining job & stop the test if it's too long
    [ ${time_elapsed} -eq $TIMEOUT ] && { eval ${JOB_DELE} &> /dev/null; get_out 6; }
}

diff_results() {
    local file
    local files_list='' mesg='Same'

    ## Simple diff
    for file in 'ocean.output' *.stat; do
   ## Stop if no benchmark files (ocean.output, eventual stat files)
   [ ! -e ${TRUS_BHMK}/$file ] && { export TRUS_RSLT='FAILED'; get_out 7; }

   diff -q $file ${TRUS_BHMK}/$file

   ## Continue even if it differs
   [ $? -ne 0 ] && { export TRUS_RSLT='FAILED'; mesg='Different'; files_list+=$file' '; }
    done

    [ $mesg == 'Same' ] && echo $mesg

    ## List different files for web comment
    [ -n "${files_list}" ] && echo 'Results : '${files_list}'differ<br>' \
   >> temp_${FILE_NOTE}
}

diff_restart() {
    local base_name comp dif file list_comp list_tmsp nb_dom time_step tmsp
    local files_list='' dift=0

    ## Stop if no benchmark files (ie time.step)
    [ ! -e ${TRUS_BHMK}/time.step ] && { export TRUS_RSLT='FAILED'; get_out 7; }
    time_step=$( cat ${TRUS_BHMK}/time.step | tr -d [:space:] )

    ## Find all restart files to rebuild
    if [ $( find -regex ".*_restart.*[0-9]\.nc" -print -quit ) ]; then
   base_name=$( find -regex ".*_restart.*[0-9]\.nc"                       \
                | sed "s#^\./\(.*\)_[0-9]*_restart.*#\1#"       | sort -u  )
   list_comp=$( find -regex ".*_restart.*[0-9]\.nc"                       \
                | sed "s#^.*\(restart[a-z_]*\)_[0-9].*\.nc#\1#" | sort -u  )
   list_tmsp=$( find -regex ".*_restart.*[0-9]\.nc"                       \
                | sed "s#^.*\([0-9]\{8\}\)_restart.*#\1#"       | sort -u  )

   ## Loop on each time step
   for tmsp in ${list_tmsp}; do

       for comp in ${list_comp}; do
      file=${base_name}_${tmsp}_${comp}
      nb_dom=$( find -name "${file}_[0-9]*.nc" | wc -l | awk '{ print $1 }' )

      if   [ ${nb_dom} -gt 1 ]; then
          ${TRUS_WKCY}/TOOLS/REBUILD_NEMO/rebuild_nemo -t ${TRUS_NPRO} $file ${nb_dom} > /dev/null
          [ $? -eq 0 ] && rm -f ${file}_[0-9]*.nc                                > /dev/null
      elif [ ${nb_dom} -eq 0 ]; then
          export TRUS_RSLT='FAILED' && get_out 8
      fi

      ## Compare restart files at same time step
      if [ $tmsp -eq ${time_step} ]; then

                    ## Stop if no benchmark files (restart file)
          if [ -e ${TRUS_BHMK}/$file.nc ]; then

                   ## UNIX `cmp` not suitable (timestamp in .nc file)
         dif=$( $CDOD $file.nc ${TRUS_BHMK}/$file.nc 2> /dev/null          \
                | awk '/records/ {print $0}' | sed '2 s/^/,/' | tr -d '\n' )

         ## CDO can return void stdout with no difference
         if [[ -n "$dif" && $( echo $dif | awk '{print $1}' ) -ne 0 ]]; then
             export TRUS_RSLT='FAILED'
             files_list+=$comp' ' && let dif_sum+=$( echo $dif | awk '{print $1}' )
             echo $file.nc': '$dif
         fi

          else
         export TRUS_RSLT='FAILED' && get_out 7
          fi

      else
          continue
      fi

       done

   done

        ## List different files for web comment with sum of different parameters
   if [ ${dif_sum} -ne 0 ]; then
       echo 'Restarts: '${files_list}${dif_sum}' record(s) differ<br>' \
      >> temp_${FILE_NOTE}
   else
       echo 'Same'
   fi

    else
   export TRUS_RSLT='FAILED'
    fi

}

get_time() {
    ## Interest for checking unusual time computation
    local time_cpu=$( eval ${JOB_TIME} )

    printf "Elapsed time: "
    echo ${time_cpu} | tee -a ${FILE_TIME}
}

get_memy() {
    ## Interest for checking unusual memory usage
    local memory_pmax=$( eval ${JOB_PMEM} ) memory_vmax=$( eval ${JOB_VMEM} )

    printf "Memory max usage (physical/virtual): "
    echo ${memory_pmax}' / '${memory_vmax} | tee -a ${FILE_MEMY}
}

comments() {
    local opat
    local line='' state=$1

    if [ -e ocean.output ]; then
        ## 'W A R N I N G' pattern by default
   opat="-A2 \"^ $state\""
   [ "$state" == 'E R R O R' ] && opat="-A4 \"$state\""

        ## Select first occurence for web comment
   line=$( eval grep -m1 $opat ocean.output | tr -d '\n' )
    fi

    [ -n "$line" ] && ( echo $line; printf "$line<br>" \
   >> temp_${FILE_NOTE} )
}

log_make() {
    ## Format comments for web
    [ -e temp_${FILE_NOTE} ] && cat temp_${FILE_NOTE} | tr -d '\n' | sed 's/<br>$//' \
   >> ${FILE_NOTE}

    ## Construct txt file with all messenger files
    paste -d ';' mesg_*.txt | tee ${FILE_TRUS}
}

prod_publish() {
    local cmd
    local rev=$( awk '/NEMOGCM/ {print $NF}' model.log )

    ## Production mode (-p|--prod)
    if [ $PROD -eq 1 ]; then

   ## Create or append trusting logfile
   if [ -f ${TRUS_BHMK}/trusting_$PATTERNAME.txt ]; then cmd='tail -1'; else cmd='cat'; fi

   $cmd ${FILE_TRUS}                           \
       >> ${TRUS_BHMK}/trusting_$PATTERNAME.txt

        ## Send mail only when FAILED
   if [[ ! -z "$TRUS_MAIL" && $TRUS_RSLT == 'FAILED' ]]; then

       ## Content
       cat <<END_MAIL      \
      > trusting.mail
Dear all,


The trusting sequence has not completed successfully on new configuration ${TRUS_TEST} based on ${TRUS_REFE}.

Here is the model summary:
`cat model.log`

First checking would be on the trusting environment files:
${TRUS_USER}.cfg & ${TRUS_HPCC}.cfg

For more details, look into the testing directory at:
${TEST_DIR}

An archive has been created to share the questionable configuration for further studies:
${TRUS_BHMK}/${FILE_ARCH}

END_MAIL

       ## Send with detailed subject
       mail -s "[NEMO Trusting][$rev][${TRUS_WKCY}][${TRUS_REFE}] $TRUS_RSLT $ERR" $TRUS_MAIL \
      <  trusting.mail
   fi

    fi
}

get_out() {
    local time_step=0

    ERR=$1

    printf "\n\nEnd of test\n"

    ## In case of compilation error
    cd ${TEST_DIR}

    if [ $TRUS_RSLT == 'FAILED' ]; then
   echo 'Failure'

        ## Error identification
   case $ERR in
            ## Compilation
       '1') ERR='XIOS compilation failed' ;; '2') ERR='NEMO compilation failed';;
       ## Submission
       '3') ERR='Missing input files'     ;; '4') ERR='Job submission error'   ;;
       ## Computation
       '5') ERR='Crashed at time step'    ;; '6') ERR='Exceeded time limit'    ;;
       ## Results
       '7') ERR='Missing previous outputs';; '8') ERR='New outputs differ'     ;;
   esac

    else
   echo 'Success' && ERR='Code is reliable'
    fi

    ## Eventual comments from ocean.output
    if [ "$ERR" == 'Crashed at time step' ]; then
   comments 'E R R O R'
   [ -e time.step ] && time_step=$( grep -o [0-9]* time.step )
   ERR+=' '$time_step
    else
   comments 'W A R N I N G'
   [ "$ERR" == 'Exceeded time limit' ] && ERR+=' '$(( ${TIMEOUTT}/3600 ))'h'
    fi

    ## Last messenger files
    export ERR
    sed -i "2 s/.*/$TRUS_RSLT/" ${FILE_TRUS_RSLT}; sed -i "2 s/.*/$ERR/" ${FILE_STAT}

    ## Save tested configuration if trusting failed in production mode (-p|--prod)
    if [[ $TRUS_RSLT == 'FAILED' && $PROD -eq 1 ]]; then
   echo 'Creating archive '${FILE_ARCH}' under '${TRUS_BHMK}
   tar -czf ${TRUS_BHMK}/${FILE_ARCH}              *                    \
       -C ${TRUS_WKCY}/CONFIG/${TRUS_TEST}/MY_SRC .                    \
       -C ${TRUS_WKCY}/CONFIG/${TRUS_TEST}        cpp_${TRUS_TEST}.fcm
    fi

    ## Logfile construct & eventual sending of notification email
    printf "\nTrusting digest:\n----------------\n"
    log_make
    prod_publish

    exit 0
}