#!/bin/bash


## Messenger filenames
FILE_DATE=mesg_01_date_$PATTERNAME.txt    ; FILE_STAT=mesg_02_status_$PATTERNAME.txt
FILE_RESU=mesg_03_result_$PATTERNAME.txt  ; FILE_NEMO=mesg_04_nemogcm_$PATTERNAME.txt
FILE_XIOS=mesg_05_xios_$PATTERNAME.txt    ; FILE_COMP=mesg_06_compiler_$PATTERNAME.txt
FILE_MPIN=mesg_07_mpi_$PATTERNAME.txt     ; FILE_NCDF=mesg_08_netcdf_$PATTERNAME.txt
FILE_INPT=mesg_09_inputs_$PATTERNAME.txt  ; FILE_CPUT=mesg_10_cputime_$PATTERNAME.txt
FILE_COMM=mesg_11_comments_$PATTERNAME.txt;

## Timestamped outcome & archive filenames
FILE_TRUS=trusting_${DATE}_$PATTERNAME.txt; FILE_ARCH=trusting_${DATE}_$PATTERNAME.tar.gz


print_step() { printf "\nStep.....\n$1\n"; }


init_files() {
    echo 'Date'        	 >  ${FILE_DATE}; echo 'Status'           >  ${FILE_STAT}
    echo 'Result'      	 >  ${FILE_RESU}; echo 'NEMOGCM rev.'     >  ${FILE_NEMO}
    echo 'XIOS rev.'   	 >  ${FILE_XIOS}; echo 'Fortran compiler' >  ${FILE_COMP}
    echo 'MPI libs'    	 >  ${FILE_MPIN}; echo 'NetCDF libs'      >  ${FILE_NCDF}
    echo 'Input files' 	 >  ${FILE_INPT}; echo 'Real CPU time'    >  ${FILE_CPUT}
    echo 'Comments'    	 >  ${FILE_COMM}

    ## 'Failed' status with 'Unknown error' by default
    echo $ST             \
	>> ${FILE_STAT}
    echo 'Unknown error' \
	>> ${FILE_RESU}
}


get_date() {
    ## UTC time zone for timestamping
    local dat=$( date -ud "${DATE}" +"%F %R %Z" )

    echo $dat          \
	>> ${FILE_DATE}
}


get_nemo_rev() {
    local dir rev_loc
    local rev=0

    ## Loop on essential directories
    for dir in ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} \
               ${NEMO_EAGR} ${NEMO_EIOI} ${NEMO_EFCM} \
	       ${NEMO_TCMP} ${NEMO_TRBD}              \
	       ${DIR_XIOS}                             ; do

	## For time being, just get revision from XIOS with no action on directory
	if [ $dir == ${DIR_XIOS} ]; then
	    rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' )
	    echo 'XIOS '${rev_loc} \
		>> model.log
	    echo "<a href=\"https://forge.ipsl.jussieu.fr/ioserver/changeset/${rev_loc}\" target=\"_blank\">${rev_loc}</a>" \
		>> ${FILE_XIOS}
	    continue
	fi

	echo $dir && ${SVN_CMD} $dir
	rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' )

	## Keep last rev. nb
	[ ${rev_loc} -gt $rev ] && rev=${rev_loc}
    done

    echo 'NEMOGCM '$rev \
	>> model.log
    echo "<a href=\"https://forge.ipsl.jussieu.fr/nemo/changeset/$rev\" target=\"_blank\">$rev</a>" \
	>> ${FILE_NEMO}
}


get_soft_rel() {
    local soft soft_rel

    for soft in $CDO $COMPILER $MPI $NETCDF; do
	soft_rel=''

	## Software strings to identify depends on use (sed is case-insensitive)
        if [ $IMOD -eq 1 ]; then
	    ## Modules system: next word after "$soft/"
	    soft_rel=$( echo $LOADEDMODULES | sed "s#.*$soft/\([0-9.a-z_]*\).*#\1#i" )
	    ## LOADEDMODULES=netcdf/mpi/4.1.3:cdo/1.5.9:intel/2013.0
	    ## LOADEDMODULES=bullxmpi/1.2.8.2:netcdf/4.3.3.1_hdf5_parallel:cdo/1.6.7
	else
	    ## Hard-coded paths: next word after "$soft"
	    soft_rel=$( echo $PATH          | sed "s#.*$soft\([0-9.a-z_]*\).*#\1#i"  )
	    ## PATH=/smplocal/intel/impi/4.1.0.024/intel64/bin:/smplocal/pub/cdo/1.5.9/bin:/smplocal/pub/NetCDF/4.1.3/mpi/bin
	    ## PATH=/usr/local/netcdf-4.3.3.1_hdf5_parallel/bin:/opt/mpi/bullxmpi/1.2.8.2/bin:/usr/local/netcdf-utils-4.3.3.1_hdf5/bin
	fi

	## option --version would work for main compilers (gfortran, intel, pgfortran, ...)
	[ $soft == $COMPILER ] && soft_rel=$( $soft --version | grep -m1 -oe '\<[0-9. ]*\>' )

	## Cleaning characters string to display proper soft name
	soft=$( echo $soft | sed 's#\\##g; s#[/-]$##' )

	echo $soft ${soft_rel} \
	    >> model.log
    done

    sed -n 4p model.log \
	>> ${FILE_COMP}
    sed -n 5p model.log \
	>> ${FILE_MPIN}
    sed -n 6p model.log \
	>> ${FILE_NCDF}
}


get_inputs() {
    # List archive content & extract it by default
    local cmd_iol="tar -tvf ${NEMO_FORC}/${NEMO_TARF}" cmd_iof="tar -vxf ${NEMO_FORC}/${NEMO_TARF}"

    ## List & copy files in case of personal inputs
    [ -z "${NEMO_TARF}" ] && { cmd_iol="ls ${NEMO_FORC}/*"; cmd_iof="\cp ${NEMO_FORC}/* ."; }

    ${cmd_iol} > inputs_list.txt
    ${cmd_iof} > /dev/null
}


diff_inputs() {
    local dif file
    local files_list='' mesg='Same' 

    ## Simple diff
    for file in 'inputs_list.txt' *namelist_* *.xml cpp_*; do
	dif=''

	## Continue even if input file is not in here (see after)
	if [ -e ${REFE_DIR}/$file ]; then dif=$( diff -q $file ${REFE_DIR}/$file ); else dif=0; fi

	## Pass over useless file omission in benckmark directory
	if [[ -n "$dif" && "$dif" != '0' ]]; then
	    mesg='Different'; echo $dif; files_list+=$file' '
	fi

    done

    [ $mesg == 'Same' ] && echo $mesg
    echo $mesg         \
	>> ${FILE_INPT}

    ## List different files for web comment
    [ -n "${files_list}" ] && echo 'Inputs  : '${files_list}'differ<br>' \
	>> temp_${FILE_COMM}
}


job_pending() {
    local outline=$( printf "%100s" ) time_elapsed=0 time_increment=30

    sleep ${time_increment}

    ## Append a log file while pending
    while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do
	printf "\n%s\n" ${outline// /#} \
	    >> computation.log
	${JOB_INFO} ${JOB_ID}           \
	    >> computation.log
	sleep ${time_increment}
	time_elapsed=$(( ${time_elapsed} + ${time_increment} ))
    done

    sleep ${time_increment}

    ## Kill remaining job & stop the test if it's too long
    [ ${time_elapsed} -eq ${TIME_LIMI} ] && { ${JOB_DELE} ${JOB_ID} &> /dev/null; get_out 6; }
}


diff_results() {
    local file
    local files_list='' mesg='Same'

    ## Simple diff
    for file in 'ocean.output' *.stat; do

	## Stop if no benchmark files (ocean.output, eventual stat files)
	[ ! -e ${REFE_DIR}/$file ] && { export ST='FAILED'; get_out 7; }

	diff -q $file ${REFE_DIR}/$file

	## Continue even if it differs
	[ $? -ne 0 ] && { export ST='FAILED'; mesg='Different'; files_list+=$file' '; }
    done

    [ $mesg == 'Same' ] && echo $mesg

    ## List different files for web comment
    [ -n "${files_list}" ] && echo 'Results : '${files_list}'differ<br>' \
	>> temp_${FILE_COMM}
}


diff_restart() {
    local base_name comp dif file list_comp list_tmsp nb_dom time_step tmsp
    local files_list='' dift=0

    ## Stop if no benchmark files (ie time.step)
    [ ! -e ${REFE_DIR}/time.step ] && { export ST='FAILED'; get_out 7; }
    time_step=$( cat ${REFE_DIR}/time.step | tr -d [:space:] )

    ## Find all restart files to rebuild
    if [ $( find -regex ".*_restart.*[0-9]\.nc" -print -quit ) ]; then
	base_name=$( find -regex ".*_restart.*[0-9]\.nc"                      \
	             | sed "s#^\./\(.*\)_[0-9]*_restart.*#\1#"       | sort -u )
	list_comp=$( find -regex ".*_restart.*[0-9]\.nc"                      \
	             | sed "s#^.*\(restart[a-z_]*\)_[0-9].*\.nc#\1#" | sort -u )
	list_tmsp=$( find -regex ".*_restart.*[0-9]\.nc"                      \
	             | sed "s#^.*\([0-9]\{8\}\)_restart.*#\1#"       | sort -u )

	## Loop on each time step
	for tmsp in ${list_tmsp}; do
	    for comp in ${list_comp}; do
		file=${base_name}_${tmsp}_${comp}
		nb_dom=$( find -name "${file}_[0-9]*.nc" | wc -l | awk '{ print $1 }' )

		if   [ ${nb_dom} -gt 1 ]; then
		    ${NEMO_TRBD}/rebuild_nemo -t $NPROC $file ${nb_dom} > /dev/null
		    [ $? -eq 0 ] && rm -f ${file}_[0-9]*.nc             > /dev/null
		elif [ ${nb_dom} -eq 0 ]; then
		    export ST='FAILED' && get_out 8
		fi

		## Compare restart files at same time step
		if [ $tmsp -eq ${time_step} ]; then

                    ## Stop if no benchmark files (restart file)
		    if [ -e ${REFE_DIR}/$file.nc ]; then

	                ## UNIX `cmp` not suitable (timestamp in .nc file)
			dif=$( $CDOD $file.nc ${REFE_DIR}/$file.nc 2> /dev/null          \
			       | awk '/records/ {print $0}' | sed '2 s/^/,/' | tr -d '\n' )

			## CDO can return void stdout with no difference
			if [[ -n "$dif" && $( echo $dif | awk '{print $1}' ) -ne 0 ]]; then
			    export ST='FAILED'
			    files_list+=$file' ' && let dift+=$( echo $dif | awk '{print $1}' )
			    echo $file.nc': '$dif
			fi

		    else
			export ST='FAILED' && get_out 7
		    fi

		else
		    continue
		fi

	    done
	done

        ## List different files for web comment with sum of different parameters
	if [ $dift -ne 0 ]; then
	    echo 'Restarts: '${files_list}$dift' record(s) differ<br>' \
		>> temp_${FILE_COMM}
	else
	    echo 'Same'
	fi

    else
	export ST='FAILED'
    fi

}


get_cpu_time() {
    ## Interest for checking unusual time computing
    local real_cpu_time=$( eval ${JOB_TIME} )

    echo ${real_cpu_time} | tee -a ${FILE_CPUT}
}


comments() {
    local opat
    local line='' state=$1

    if [ -e ocean.output ]; then
        ## 'W A R N I N G' by default
	opat="-A2 \"^ $state\""
	[ "$state" == 'E R R O R' ] && opat="-A4 \"$state\""

        ## Select first occurence for web comment
	line=$( eval grep -m1 $opat ocean.output | tr -d '\n' )
	echo $line
    fi

    printf "$line<br>"      \
	>> temp_${FILE_COMM}
}


mesg_make() {
    ## Format comments for web
    [ -e temp_${FILE_COMM} ] && cat temp_${FILE_COMM} | tr -d '\n' | sed 's/<br>$//' \
	>> ${FILE_COMM}

    ## Construct txt file for web with all messenger files
    paste -d ';' mesg_*.txt | tee ${FILE_TRUS}
}


mesg_publish() {
    local cmd

    ## Production mode (-p|--publish)
    if [ $PUBLISH -eq 1 ]; then

	## Create or append historical trusting file
	if [ -f ${REFE_DIR}/trusting_$PATTERNAME.txt ]; then cmd='tail -1'; else cmd='cat'; fi

	$cmd ${FILE_TRUS}                           \
	    >> ${REFE_DIR}/trusting_$PATTERNAME.txt

        ## Send mail only when FAILED
	if [[ ! -z "$EMAIL" && $ST == 'FAILED' ]]; then

	    ## Content
	    cat <<END_MAIL \
		> trusting.mail
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX

Dear $USER,

The trusting sequence for ${REFE_CONF} has failed.
Directory: ${TEST_DIR}
Archive created: ${FILE_ARCH} in ${REFE_DIR}

XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX

`cat ${TEST_DIR}/${FILE_TRUS}`
END_MAIL

	    ## Send with detailed subject
	    mail -s "[NEMO Trusting ${NEMO_HPCC} ${REFE_CONF} ${NEMO_BRAN}] $ST $ERR" $EMAIL \
		<  trusting.mail
	fi

    fi
}


get_out() {
    local time_step=0

    ERR=$1

    print_step 'End of trusting test'

    ## In case of compilation error
    cd ${TEST_DIR}

    if [ $ST == 'FAILED' ]; then
	echo 'Failure'

        ## Error identification
	case $ERR in
            ## Compilation
	    '1') ERR='XIOS compilation failed' ;; '2') ERR='NEMO compilation failed';;
	    ## Submission
	    '3') ERR='Missing input files'     ;; '4') ERR='Job submission error'   ;;
	    ## Running
	    '5') ERR='Crashed at time step'    ;; '6') ERR='Exceeded time limit'    ;;
	    ## Results
	    '7') ERR='Missing previous outputs';; '8') ERR='New outputs differ'     ;;
	    ## Undefined
	    "*") ERR='Unknown error'           ;;
	esac

    else
	echo 'Success' && ERR='Code is reliable'
    fi

    ## Eventual comments from ocean.output
    if [ "$ERR" == 'Crashed at time step' ]; then
	comments 'E R R O R'
	[ -e time.step ] && time_step=$( cat time.step | tr -d ' ' )
	ERR+=' '$time_step
    else
	comments 'W A R N I N G'
	[ "$ERR" == 'Exceeded time limit' ] && ERR+=' '$(( ${TIME_LIMIT}/3600 ))'h'
    fi

    ## Last messenger files
    export ERR
    sed -i "2 s/.*/$ST/" ${FILE_STAT}; sed -i "2 s/.*/$ERR/" ${FILE_RESU}

    ## Save tested configuration if trusting failed in production mode (-p|--publish)
    if [[ $ST == 'FAILED' && $PUBLISH -eq 1 ]]; then
	echo 'Creating archive '${FILE_ARCH}' under '${REFE_DIR}
	tar -czf ${REFE_DIR}/${FILE_ARCH} * ${NEMO_HOME}/CONFIG/${TEST_CONF}/MY_SRC \
	                                    ${NEMO_HOME}/CONFIG/${TEST_CONF}/WORK
    fi

    ## Share trusting result (.txt file/mail)
    print_step 'Trusting digest'
    mesg_make
    mesg_publish

    exit 0
}
