#!/bin/bash


## Messenger filenames
FILE_DATE=mesg_01_date_$PATTERNAME.txt    ; FILE_STAT=mesg_02_status_$PATTERNAME.txt
FILE_RESU=mesg_03_result_$PATTERNAME.txt  ; FILE_NEMO=mesg_04_nemogcm_$PATTERNAME.txt
FILE_XIOS=mesg_05_xios_$PATTERNAME.txt    ; FILE_COMP=mesg_06_compiler_$PATTERNAME.txt
FILE_MPIN=mesg_07_mpi_$PATTERNAME.txt     ; FILE_NCDF=mesg_08_netcdf_$PATTERNAME.txt
FILE_INPT=mesg_09_inputs_$PATTERNAME.txt  ; FILE_CPUT=mesg_10_cputime_$PATTERNAME.txt
FILE_COMM=mesg_11_comments_$PATTERNAME.txt;

## Trusting summary & archive timestamped filenames
FILE_TRUS=trusting_${DATE}_$PATTERNAME.txt; FILE_ARCH=trusting_${DATE}_$PATTERNAME.tar.gz

print_step() { printf "\nStep.....\n$1\n"; }

init_files() {
    echo 'Date'        	 >  ${FILE_DATE}; echo 'Status'           >  ${FILE_STAT}
    echo 'Result'      	 >  ${FILE_RESU}; echo 'NEMOGCM rev.'     >  ${FILE_NEMO}
    echo 'XIOS rev.'   	 >  ${FILE_XIOS}; echo 'Fortran compiler' >  ${FILE_COMP}
    echo 'MPI libs'    	 >  ${FILE_MPIN}; echo 'NetCDF libs'      >  ${FILE_NCDF}
    echo 'Input files' 	 >  ${FILE_INPT}; echo 'Real CPU time'    >  ${FILE_CPUT}
    echo 'Comments'    	 >  ${FILE_COMM}

    # 'Failed' status with 'Unknown error' by default
    echo $ST             >> ${FILE_STAT}; echo 'Unknown error'    >> ${FILE_RESU}
}

get_date() {
    local dat=$( date -ud ${DATE} +"%F %R %Z" )

    echo $dat >> ${FILE_DATE}
}

get_nemo_rev() {
    local rev_loc svn_cmd
    local rev=0 svn_opt='status'

    ## If -v|--version option has been set, modify svn command
    if   [ $( echo ${NEMO_VERS} | grep  "HEAD\|up\|update"                     ) ]; then
	svn_cmd='svn update -r HEAD'
    elif [ $( echo ${NEMO_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' ) ]; then
	svn_cmd='svn update -r '$( echo ${NEMO_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' )
    elif [ $( echo ${NEMO_VERS} | grep -o '[0-9]*'                             ) ]; then
	svn_cmd='svn update -r '$( echo ${NEMO_VERS} | grep -o '[0-9]*'                             )
    else
	svn_cmd='svn '${svn_opt}
    fi

    for dir in ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} \
               ${NEMO_EIOI} ${NEMO_EFCM}              \
	       ${NEMO_TCMP} ${NEMO_TRBD}              \
	       ${DIR_XIOS}                             ; do

	## For time being, just get revision number from XIOS
	if [ "$dir" == "${DIR_XIOS}" ]; then
	    rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' )
	    echo 'XIOS '${rev_loc} >> model.log
	    echo "<a href=\"https://forge.ipsl.jussieu.fr/ioserver/changeset/${rev_loc}\" target=\"_blank\">${rev_loc}</a>" \
		>> ${FILE_XIOS}
	    continue
	fi

	echo $dir
	${svn_cmd} $dir
	rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' )
	(( ${rev_loc} >= $rev )) && rev=${rev_loc}
    done

    echo 'NEMOGCM '$rev   >> model.log
    echo "<a href=\"https://forge.ipsl.jussieu.fr/nemo/changeset/$rev\" target=\"_blank\">$rev</a>" \
	>> ${FILE_NEMO}
}

get_soft_rel() {
    local soft_rel

    for soft in $CDO $COMPILER $MPI $NETCDF; do
	soft_rel=''

        if (( ${I_MODULE} == 1 )); then
	    soft_rel=$( echo $LOADEDMODULES | sed "s/.*$soft\/\([0-9.a-z_]*\).*/\1/i" )
	else
	    soft_rel=$( echo $PATH          | sed "s/.*$soft\([0-9.a-z_]*\).*/\1/i"   )
	fi

	[ "$soft" == "$COMPILER" ] && soft_rel=$( $soft --version | grep -m1 -oe '\<[0-9. ]*\>' )
	soft=$( echo $soft | sed 's#\\##g; s#[/-]$##' )	## Cleaning characters string to display proper soft name
	echo $soft ${soft_rel} >> model.log
    done

    sed -n 4p model.log >> ${FILE_COMP}; sed -n 5p model.log >> ${FILE_MPIN}; sed -n 6p model.log >> ${FILE_NCDF}
}

get_inputs() {
    local cmd_iol="tar -tvf ${NEMO_FORC}/${NEMO_TARF}" cmd_iof="tar -vxf ${NEMO_FORC}/${NEMO_TARF}"

    [ -z "${NEMO_TARF}" ] && cmd_iol="ls ${NEMO_FORC}/*" && cmd_iof="\cp ${NEMO_FORC}/* ."
    ${cmd_iol} > inputs_list.txt && ${cmd_iof} > /dev/null
}

diff_inputs() {
    local dif
    local files_list='' mesg='Same' 

    for file in inputs_list.txt namelist_* *.xml; do
	dif=''
	if [ -e ${REFE_DIR}/$file ]; then dif=$( diff -q $file ${REFE_DIR}/$file ); else dif=0; fi

	if [ -n "$dif" ]; then
	    mesg='Different'
	    [ "$dif" != '0' ] && echo $dif && files_list+=$file' '
	fi

    done

    [ "$mesg" == 'Same' ] && echo $mesg
    echo $mesg >> ${FILE_INPT}

    [ -n "${files_list}" ] && echo 'Inputs  : '${files_list}'differ<br>' >> temp_${FILE_COMM}
}

job_pending() {
    local time_elapsed=0 time_increment=30

    sleep ${time_increment}

    while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do
	printf "\n####################################################\n" >> computation.log
	${JOB_INFO} ${JOB_ID}                                             >> computation.log
	sleep ${time_increment}
	time_elapsed=$(( ${time_elapsed} + ${time_increment} ))
    done

    sleep ${time_increment}

    if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then
	${JOB_DELE} ${JOB_ID} &> /dev/null
	let TIME_LIMIT/=3600
	get_out 6
    fi
}

diff_results() {
    local files_list=''

    for file in ocean.output *.stat; do
	[ ! -e ${REFE_DIR}/$file ] && export ST='FAILED' && get_out 7
	diff -q $file ${REFE_DIR}/$file
	(( $? != 0 )) && export ST='FAILED' && files_list+=$file' '
    done

    [ -n "${files_list}" ] && echo 'Results : '${files_list}'differ<br>' >> temp_${FILE_COMM}
}

diff_restart() {
    local base_name diff_rcd file nb_dom
    local dif=0 files_list=''

    [ ! -e  ${REFE_DIR}/time.step ] && export ST='FAILED' && get_out 7

    TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] )
    echo 'Last time step of standard run: '${TIME_STEP}

    if [ $( find -regex ".*_0+$TIME_STEP_restart.*\.nc" -print -quit ) ]; then
	base_name=$( find -regex ".*_0+$TIME_STEP_restart.*\.nc" -print -quit  \
	             | cut -d/ -f2 | sed "s/^\(.*$TIME_STEP\)\_restart.*/\1\_/" )

	for comp in restart restart_ice restart_trc; do
	    file=${base_name}${comp}

	    nb_dom=$( find -name "${file}_[0-9]*.nc" | wc -l | awk '{ print $1 }' )
	    if   [ ${nb_dom} -gt 1 ]; then
		${NEMO_TRBD}/rebuild_nemo -t $NPROC $file ${nb_dom} > /dev/null
		[ $? -eq 0 ] && rm -f ${base_name}${comp}_[0-9]*.nc > /dev/null
	    elif [ ${nb_dom} -eq 0 ]; then
		export ST='FAILED' && get_out 8
	    fi

	    if [ -e ${REFE_DIR}/$file.nc ]; then
	    ## UNIX `cmp` not suitable (filename & timestamp in .nc file)
		diff_rcd=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null      \
		            | awk '/records/ {print $0}' | sed '2 s/^/,/' | tr -d '\n'  )

		if [[ ! -z ${diff_rcd} && $( echo ${diff_rcd} | awk '{ print $1 }') -ne 0 ]]; then
		    export ST='FAILED'
		    files_list+=$file' '
		    let dif+=$( echo ${diff_rcd} | awk '{print $1}' )
		    echo $file.nc': '${diff_rcd}
		else
		    echo 'identical'
		fi

	    else
		export ST='FAILED' && get_out 7
	    fi

	done

	[ -n "${files_list}" ] && echo 'Restarts: '${files_list}$dif' record(s) differ<br>' >> temp_${FILE_COMM}
    else
	export ST='FAILED'
    fi

}

get_cpu_time() {
    local real_cpu_time=$( eval ${JOB_TIME} )

    echo ${real_cpu_time} | tee -a ${FILE_CPUT}
}

comments() {
    local state=$1
    export TIME_STEP=0

    if [ "$state" == 'E R R O R' ]; then
	[ -e time.step ] && export TIME_STEP=$( cat time.step | tr -d ' ' )
	[ -e ocean.output ] && line=$( grep -m1 -A4 "$state"   ocean.output | tr -d '\n' )
    else
	[ -e ocean.output ] && line=$( grep -m1 -A2 "^ $state" ocean.output | tr -d '\n' )
    fi

    echo   $line
    printf "$line<br>" >> temp_${FILE_COMM}
}

mesg_make() {
    ## Format comments for web
    [ -e temp_${FILE_COMM} ] && cat temp_${FILE_COMM} | tr -d '\n' | sed 's/<br>$//' >> ${FILE_COMM}

    paste -d ';' mesg_*.txt | tee ${FILE_TRUS}
}

mesg_publish() {
    local cmd

    if [ $PUBLISH -eq 1 ]; then

	if [ -f ${REFE_DIR}/trusting_$PATTERNAME.txt ]; then cmd='tail -1'; else cmd='cat'; fi

	$cmd ${FILE_TRUS} >> ${REFE_DIR}/trusting_$PATTERNAME.txt

  ## Send mail only when FAILED
	if [[ ! -z $EMAIL && "$ST" == 'FAILED' ]]; then
	    cat << END_MAIL > trusting.mail
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX

Dear $USER,

The trusting sequence for ${REFE_CONF} has failed.
Directory: ${TEST_DIR}
Archive created: ${FILE_ARCH} in ${REFE_DIR}

XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX

END_MAIL
	    [ -e ${TEST_DIR}/${FILE_TRUS} ] && cat ${TEST_DIR}/${FILE_TRUS}                  >> trusting.mail
	    mail -s "[NEMO Trusting ${NEMO_HPCC} ${REFE_CONF} ${NEMO_BRAN}] $ST $ERR" $EMAIL <  trusting.mail
	fi

    fi
}

get_out() {
    ERR=$1

    cd ${TEST_DIR}

    if [ "$ST" == 'FAILED' ]; then

        ## Error identification
	case $ERR in
	    ## Compilation
	    1) ERR='XIOS compilation failed'          ;; 2) ERR='NEMO compilation failed'            ;;
	    ## Submission
	    3) ERR='Missing input files'              ;; 4) ERR='Job submission error'               ;;
	    ## Running
	    5) ERR='Crashed at time step '${TIME_STEP};; 6) ERR='Exceeded time limit '${TIME_LIMI}'h';;
	    ## Results
	    7) ERR='Missing previous outputs'         ;; 8) ERR='New outputs differ'                 ;;
	    ## Undefined
	    *) ERR='Unknown error'                    ;;
	esac

	export ERR
    fi

    sed -i "2 s/.*/$ST/"  ${FILE_STAT} && sed -i "2 s/.*/$ERR/" ${FILE_RESU}

    ## Save tested configuration if trusting failed
    if [[ "$ST" == 'FAILED' && $PUBLISH -eq 1 && $DEBUG -eq 0 ]]; then
	echo 'Creating archive '${FILE_ARCH}' under '${REFE_DIR}
	tar -czf ${REFE_DIR}/${FILE_ARCH} * ${TEST_DIR}/${NEMO_CONF}/${TEST_CONF}/cpp_*  \
                                 	    ${TEST_DIR}/${NEMO_CONF}/${TEST_CONF}/MY_SRC \
	                                    ${TEST_DIR}/${NEMO_CONF}/${TEST_CONF}/WORK
    fi

    mesg_make
    mesg_publish

    exit 0
}
