#!/bin/ksh
# $Id$
######################################################
# Original : C. Talandier for ESOPA
# Contact  : opatlod@locean-ipsl.upmc.fr
#
# It is automatically launched (by the cron_jobs.ksh 
# script) when all runs (mon, mpi, (omp)) for a given 
# configuration are finished 
#
# INPUT ARGS: 1 + 1 optional
#     - CONF: configuration name (ORCA2_LIM, GYRE ..)
#     - TODO: (optional) Arg=yes or empty to build 
#             output files including timing results. 
#             Specific to Zahir
#
# WORK: It aims to (for each standard configuration) :
#
#    I. Check Executable Memory size
#       - from memory_size.txt output file
#      *- compare it to a reference version 
#
#   II. Check CPU time used (if specified)
#       - from timing output files
#      *- compare it to a reference version 
#
#  III. Check the reproductibility between mon & mpi runs
#       - making diff of solver.stat output files
#
#   IV. Check the restartability for mon & mpi runs
#       - making diff of solver.stat output files
#
#    V.*Compare current results to a reference one
#        
#   VI. Send the assessment file "endjob.txt" using 
#       user's e-mail
#        
# * this step is performed only if a reference tag 
#   is passing through the specifs.txt file
#   See the Makefile under ./config/NVTK directory
#
# It is based on the directories architecture below :
#
#         NEMO_VALID/WNAME_CONFIG/
#                    |-- mon            --> MONO run
#                    |   |-- 1_SHORT
#                    |   |-- 2_SHORT
#                    |   |-- LONG
#                    |   `-- GTIME
#                    |-- mpi            --> MPI run
#                    |   |-- 1_SHORT
#                    |   |-- 2_SHORT
#                    |   |-- LONG
#                    |   `-- GTIME
#                    `-- omp            --> Open-MP run
#                        |-- 1_SHORT
#                        |-- 2_SHORT
#                        |-- LONG
#                        `-- GTIME
#
###########################################################
#set -xv 
###########################################################
CONF=$1
TODO=$2
TARGET=osxxlf
VERSION=$( ( awk '/test=/ { print $2 }' specifs.txt ) )
LRTYPE="$( ( awk '/runs=/ { printf  "%s %s %s", $2 , $3 , $4}' specifs.txt ) )"
VTIME=$( ( awk '/time=/ { print $2 }' specifs.txt ) )
REF_TAG=$( (awk '/tagname=/ {print $2}' specifs.txt) )
TITLE="NEMO VALIDATION ${VERSION} : $CONF on $TARGET"
EMAIL='ctlod@locean-ipsl.upmc.fr'

\rm endjob.txt 2> /dev/null
touch endjob.txt

###########################################################
# checkfile() is a local function to check if a file exist
# 2 arguments:
#    - zf2check : file name to check
#    - zoutfile : output file name to print results
#    - zwork    : kind of prints to perform
#    - zreprint : (optional) to avoid printing twice a message
###########################################################
################ Begin checkfile function #################
function checkfile
{
# name of the file to check
zf2check=$1
# print output file name
zoutfile=$2
# white space prints
zworkt=$3
# Print a message only once
zreprint=$4

#set -x 

test -n "${zreprint}"
zrep=$?


genf=$( basename ${zf2check} )

case "${zworkt}" in
      'NORM' )  blnk='       ' ;;
      'DIFF' )  blnk=' '  ;;
esac
fmes4=" - file ${zf2check} doesn't exist "

# Check file exists
if [ -a ${zf2check} ] ; then
    fmes1=" List ${genf} files and check date creation:   "
    fmes2=" `ls -l ${zf2check}`           "
    fmes3=" - file ${zf2check} is empty  "
    [ ${zrep} == 0 ] && echo "${blnk}${fmes1}"  >> ${zoutfile}

   # Check file is not empty
   if [ -s ${zf2check} ] ; then
       [ ${zoutfile} ] && echo "${blnk}${fmes2}" >> ${zoutfile}
       xist=0 ; return ${xist}
   else
       [ ${zoutfile} ] &&  echo "${blnk}${fmes3}" >> ${zoutfile}
       xist=1 ; return ${xist}
   fi
else
   [ ${zoutfile} ] && echo "${blnk}${fmes4}" >> ${zoutfile}
   xist=1 ; return ${xist}
fi
return
}
############### End checkfile function ###############
######################################################

# Get the name of the present script
nscript=$( (basename $0) )

###########################################################
# mak_diff_file() is a local function which performs diff
# on some files and print messages in the endjob.txt file
# 7 input arguments:
#    - zfex    : (0/1) (make/do not make) diff (files doesn't exist)
#    - zfile1  : first input file name
#    - zfile2  : second input file name
#    - zruntype: run type (mon, mpi, omp)
#    - zstream : stream of the run (long, stream_1, stream_2)
#    - ztag    : the tag name to which are compared files
#    - zkind   : kind of the check (repro/resta)
###############################################################
################ Begin mak_diff_file function #################
function mak_diff_file
{
#
# help_diff() To print help
#
function help_diff
{
  echo "Usage: ${nscript} [-x] [-a] [-b] [-r] [-s] [-t] [-k]"
  echo "Options: These are optional argument"
  echo " -x : (0/1) (make/do not make) diff "
  echo " -a : first input file name"
  echo " -b : second input file name"
  echo " -r : run type (mon, mpi, omp)"
  echo " -s : stream of the run (long, stream_1, stream_2)"
  echo " -t : tag name to which are compared files"
  echo " -k : kind of the check (repro/resta)" 
  echo " their values are not taken)"
  exit 1
}

##set -xv

# Initialization
zfex=1
zfile1=
zfile2=
zruntype=
zstream=
ztag=
zkind=none

# Get arguments
while getopts x:a:b:r:s:t:k: opt
do
  case "$opt" in
    x) zfex="$OPTARG";;
    a) zfile1="$OPTARG";;
    b) zfile2="$OPTARG";;
    r) zruntype="$OPTARG";;
    s) zstream="$OPTARG";;
    t) ztag="$OPTARG";;
    k) zkind="$OPTARG";;
    *) help_diff;;
  esac
done

zgenf=$( basename ${zfile1} )
zgenftrc=$( basename ${zfile1%.*} )

case "$zkind" in
  none ) 
      zmesg1="                   Current run                                                     ${ztag} run   "
      zmesg2="                   -----------                                                     --------------"
      zmesg_NOK="        ${zstream} ${zgenf} is NOT identical to the version ${ztag} one"
      zmesg_OK="        ${zstream} ${zgenf} IS IDENTICAL to the version ${ztag} one  "
      zmesg_wht="        NO comparison between current ${zstream} and the version ${ztag} ${zgenf} "
      bnam='cvref_'${zgenftrc}_${zstream}_${zruntype} ;;
  repro ) 
      zmesg1="                   mon run                                                         ${zruntype} run      "
      zmesg2="                   -------                                                         -------      "
      zmesg_NOK="        NO NO NO NO NO NO for the current version"
      zmesg_OK="        YES YES YES YES YES YES YES YES for the current version "
      zmesg_wht="        NO CONCLUSION !!!!!  "
      bnam=${zkind}_${zgenftrc} ;;
  resta ) 
      zmesg1="                   LONG Stream                                                     SHORT Streams"
      zmesg2="                   -----------                                                     -------------"
      zmesg_NOK="        NO NO NO NO NO NO for the current version i.e. LONG stream != ( 1_SHORT + 2_SHORT ) streams "
      zmesg_OK="        YES YES YES YES YES YES YES YES for the current version i.e. LONG stream = ( 1_SHORT + 2_SHORT ) streams "
      zmesg_wht="        NO CONCLUSION !!!!!! "
      bnam=${zkind}_${zgenftrc} ;;
esac


if [ ${zfex} == 0 ] ;  then
    \rm ${bnam}.txt 2> /dev/null
    echo "                                    Make difference between ${zgenf} files                       "  > ${bnam}.txt
    echo "                                            ${zstream} ${zruntype} run                           " >> ${bnam}.txt
    echo "${zmesg1}"                                                                                         >> ${bnam}.txt
    echo "${zmesg2}"                                                                                         >> ${bnam}.txt
    sdiff    ${zfile1} ${zfile2} >>  ${bnam}.txt
    sdiff -s ${zfile1} ${zfile2} >>  ${bnam}_dd.txt

    #---------
    if [ ${zgenf} = 'ocean.output' ] ; then
        echo "                                                                                                 " >> ${zstream}_head.txt
        echo "                                    Make difference between ${zgenf} files                       "  > ${zstream}_head.txt
        echo "                                            ${zstream} ${zruntype} run                           " >> ${zstream}_head.txt
        echo "                   Current run                                                     ${ztag} run   " >> ${zstream}_head.txt
        echo "                   -----------                                                     --------------" >> ${zstream}_head.txt
        echo "                                                                                                 " >> ${zstream}_head.txt
        zfout=${bnam}_dd.txt
        zspmes=" See ${bnam}.txt file for full difference"
    else
        touch ${zstream}_head.txt
        zfout=${bnam}.txt
        zspmes=""
    fi
    #---------
    if [ -s ${bnam}_dd.txt ] ; then
    # Differences exist between solver.stat or ocean.output files
       echo "                             " >> endjob.txt
       echo "${zmesg_NOK}"                  >> endjob.txt
       echo "           ${zspmes}         " >> endjob.txt
       echo "                             " >> endjob.txt
       cat ${zstream}_head.txt ${zfout}     >> endjob.txt
       echo "                             " >> endjob.txt
       echo "                             " >> endjob.txt
    else
       # No differences exist between solver.stat or ocean.output files
       echo "                             " >> endjob.txt
       echo "${zmesg_OK}"                   >> endjob.txt
       echo "                             " >> endjob.txt
    fi

    \rm *_dd* *_head.txt 2> /dev/null

else
       # It is not possible to conclude because files are missing or empty
       echo "                             " >> endjob.txt
       echo "${zmesg_wht}"                  >> endjob.txt
       echo "                             " >> endjob.txt
fi

return

}
############### End mak_diff_file function ####################
###############################################################

###########################################################
# concat_perf() is a local function to gather performances
# files (perfs_runtype.txt_xxxx) build with the hpmcount 
# command on Zahir and standard(s) error output file(s)
# runtypeCONFNAME_err
# 1 argument:
#    - zconf : configuration name
#############################################################
################ Begin concat_perf function #################
function concat_perf
{

#set -x

# name of the configuration
zconf=$1

# List of files to treat
zcatfe[0]="mon/LONG/mon${zconf}_err"
zcatfe[1]="mon/GTIME/mon${zconf}_err"
zcatfe[2]="mpi/GTIME/mpi${zconf}_err"

ind=0
while [ ${ind} -lt 3 ] ; do

       zpath=$( ( dirname ${zcatfe[${ind}]} ) )
       zname=$( ( basename ${zcatfe[${ind}]} ) )
       cd ${zpath}
       cp ${zname} ${zname}_save
       for kf in `ls perf*` ; do
           cat ${kf} >> ${zname}
       done
       cd ../..
       let ind=${ind}+1
done

return
}
############### End concat_perf function ###############
########################################################


# ###############################
# 0. CURRENT VERSION NAME
# ###############################
echo " ############################                           "  > endjob.txt
echo " CURRENT VERSION: $VERSION                              " >> endjob.txt
echo " ############################                           " >> endjob.txt
echo "                                                        " >> endjob.txt
echo "                                                        " >> endjob.txt

test -n "${TODO}"
id2do=$?
if [ ${id2do} == 0 -a ${TARGET} = 'aix' ] ; then concat_perf ${CONF} ; fi

#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# ###############################
# I. CHECK EXECUTABLE MEMORY SIZE
# ###############################
filecs='mon/LONG/memory_size.txt' ;  filers='mon/LONG/REF/memory_size.txt'
filecm="mon/LONG/mon${CONF}_err"  ;  filerm="mon/LONG/REF/mon${CONF}_err"
xfilecs=0  ; xfilecm=0 ; xfilers=0 ; xfilerm=0
echo " ############################                           " >> endjob.txt
echo " CHECK EXECUTABLE MEMORY SIZE                           " >> endjob.txt
echo " ############################                           " >> endjob.txt
echo "                                                        " >> endjob.txt
#--------
checkfile ${filecs} 'endjob.txt' DIFF yes ; xfilecs=$?
checkfile ${filers} 'endjob.txt' DIFF     ; xfilers=$?
checkfile ${filecm} 'endjob.txt' DIFF     ; xfilecm=$?
checkfile ${filerm} 'endjob.txt' DIFF     ; xfilerm=$?

# If current version files exist
if [ ${xfilecs} == 0 ] ;  then

    if [ ${REF_TAG} ] ; then 
        ./MEM_size_${TARGET}.ksh -a ${xfilecm} -b ${xfilers} -d ${xfilerm} -c ${CONF} -o endjob.txt -t ${REF_TAG} 
    else
        ./MEM_size_${TARGET}.ksh -a ${xfilecm} -b ${xfilers} -d ${xfilerm} -c ${CONF} -o endjob.txt
    fi

else
    echo "                                                    " >> endjob.txt
    echo "   ---> NO memory size check                        " >> endjob.txt
    echo "                                                    " >> endjob.txt
fi
#
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# #######################
# II. CHECK CPU TIME USED
# #######################
if [ ${VTIME} = 'timing' ] ; then

    echo "                                                        " >> endjob.txt
    echo " ######################                                 " >> endjob.txt
    echo " CHECK CPU TIME USED                                    " >> endjob.txt
    echo " ######################                                 " >> endjob.txt
    echo "                                                        " >> endjob.txt
    #------

    for runt in ${LRTYPE} ; do

        echo "                                                    " >> endjob.txt
        echo "   ---> Timing for the ${runt} run :                " >> endjob.txt
        echo "        ----------------------------                " >> endjob.txt
        echo "                                                    " >> endjob.txt

        xfilecv=0 ; xfilerv=0

        # Check that timing file for the ${runt} current version exist
        filecv="${runt}/GTIME/${runt}${CONF}_err" 
        checkfile ${filecv} 'endjob.txt' NORM yes ; xfilecv=$?

        # Check that timing file for the ${runt} reference version exist
        filerv="${runt}/GTIME/REF/${runt}${CONF}_err" 
        checkfile ${filerv} 'endjob.txt' NORM ; xfilerv=$?
        echo "                                                             " >> endjob.txt

        if [ ${xfilecv} == 0 ] ; then 
            if [ ${REF_TAG} ] ; then 
                ./CPU_time_${TARGET}.ksh -r ${runt} -a ${filecv} -b ${filerv} -c ${CONF} -o endjob.txt -x ${xfilerv} -t ${REF_TAG} 
            else
                ./CPU_time_${TARGET}.ksh -r ${runt} -a ${filecv} -b ${filerv} -c ${CONF} -o endjob.txt -x ${xfilerv}
            fi
        fi

    done

fi

#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# ########################################################
# III. CHECK REPRODUCTIBILITY: mon .vs. mpi & mon .vs .omp
# ########################################################
echo "                                                        " >> endjob.txt
echo " ######################                                 " >> endjob.txt
echo " CHECK REPRODUCTIBILITY                                 " >> endjob.txt
echo " ######################                                 " >> endjob.txt
echo "                                                        " >> endjob.txt
#------
for runt in ${LRTYPE} ; do

    if [ ${runt} != 'mon' ] ; then
        echo "                                                    " >> endjob.txt
        echo "   ---> Reproductibility mon .vs. ${runt} ? :       " >> endjob.txt
        echo "        -------------------------------------       " >> endjob.txt

        filemon="mon/LONG/solver.stat" ;  filexxx="${runt}/LONG/solver.stat"
        xfilemon=0

        checkfile ${filemon} 'endjob.txt' NORM yes ; xfilemon=$?
        checkfile ${filexxx} 'endjob.txt' NORM ; xfilemon=$?
        
        mak_diff_file -x ${xfilemon} -a ${filemon} -b ${filexxx} -r ${runt} -s LONG -k repro
        
        if [ ${CONF} = 'GYRE_LOBSTER' ] ; then
            echo "                                                    " >> endjob.txt
            echo "   ---> BIO Reproductibility mon .vs. ${runt} ? :   " >> endjob.txt
            echo "        -----------------------------------------   " >> endjob.txt

            # check that ocean.output files exist
            filemon="mon/LONG/ocean.output" ;  filexxx="${runt}/LONG/ocean.output"
            xfilemon=0

            checkfile ${filemon} 'endjob.txt' NORM yes ; xfilemon=$?
            checkfile ${filexxx} 'endjob.txt' NORM ; xfilemon=$?
            
            # check that tracer.stat files have been built based on ocean.output ones
            filemon="mon/LONG/tracer.stat" ;  filexxx="${runt}/LONG/tracer.stat"
            xfilemon=0
            
            ./BIO_${TARGET}.ksh -a ${filemon} -b ${filexxx} -c ocean.output

            checkfile ${filemon} 'endjob.txt' NORM yes ; xfilemon=$?
            checkfile ${filexxx} 'endjob.txt' NORM ; xfilemon=$?
            
            mak_diff_file -x ${xfilemon} -a ${filemon} -b ${filexxx} -r ${runt} -s LONG -k repro
        fi
    fi
done

#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# ##########################################
# IV. CHECK RESTARTABILITY: mon mpi omp runs
# ##########################################
echo "                                                        " >> endjob.txt
echo " ####################                                   " >> endjob.txt
echo " CHECK RESTARTABILITY                                   " >> endjob.txt
echo " ####################                                   " >> endjob.txt
echo "                                                        " >> endjob.txt

for runt in ${LRTYPE} ; do

    echo "                                                    " >> endjob.txt
    echo "   ---> ${runt} restartability ? :                  " >> endjob.txt
    echo "        --------------------------                  " >> endjob.txt

    file1="${runt}/1_SHORT/solver.stat" ;  file2="${runt}/2_SHORT/solver.stat" ; file3="${runt}/LONG/solver.stat"
    xfile1=0
    #-------
    checkfile ${file1} 'endjob.txt' NORM yes ; xfile1=$?
    checkfile ${file2} 'endjob.txt' NORM ; xfile1=$?
    #
    cat ${file1} ${file2}  > short_solver_${runt}.stat
    mak_diff_file -x ${xfile1} -a ${file3} -b short_solver_${runt}.stat -r ${runt} -k resta
    \rm short_solver_${runt}.stat
    
    #-------

    if [ ${CONF} = 'GYRE_LOBSTER' ] ; then
        echo "                                                    " >> endjob.txt
        echo "   ---> ${runt} BIO restartability ? :              " >> endjob.txt
        echo "        ------------------------------              " >> endjob.txt

       # check that ocean.output files exist
        file1="${runt}/2_SHORT/ocean.output" ;  file2="${runt}/LONG/ocean.output"
        xfile1=0

        checkfile ${file1} 'endjob.txt' NORM yes ; xfile1=$?
        checkfile ${file2} 'endjob.txt' NORM     ; xfile1=$?
            
        # check that tracer.stat files have been built based on ocean.output ones
        file1="${runt}/2_SHORT/tracer.stat" ;  file2="${runt}/LONG/tracer.stat"
        xfile1=0

        ./BIO_${TARGET}.ksh -a ${file1} -b ${file2} -c ocean.output

        checkfile ${file1} 'endjob.txt' NORM yes ; xfile1=$?
        checkfile ${file2} 'endjob.txt' NORM     ; xfile1=$?
        #
        mak_diff_file -x ${xfile1} -a ${file2} -b ${file1} -r ${runt} -k resta

    fi
    
done

#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# #######################################################
# V. COMPARE CURRENT ${runt} RESULTS TO THE $REF_TAG ONES
# #######################################################

# This step is realised only if a reference tag ${REF_TAG} is specified
if [ ${REF_TAG} ] ; then

    echo "                                                        " >> endjob.txt
    echo " ####################################################   " >> endjob.txt
    echo " COMPARE CURRENT VERSION RESULTS TO THE $REF_TAG ONES   " >> endjob.txt
    echo " ####################################################   " >> endjob.txt
    echo "                                                        " >> endjob.txt

    for runt in ${LRTYPE} ; do

        echo "   -----------------------------------------------------------------------------" >> endjob.txt
        echo "   ---> ${runt} results :  current version  |   ${REF_TAG} version  STREAM 1 & 2" >> endjob.txt
        echo '                        solver.stat    .vs.    solver.stat         ' >> endjob.txt
        echo '                        ocean.output   .vs.    ocean.output        ' >> endjob.txt

        file1="${runt}/1_SHORT/solver.stat"          ;  file2="${runt}/2_SHORT/solver.stat"
        filev1="${runt}/1_SHORT/REF/solver.stat"     ;  filev2="${runt}/2_SHORT/REF/solver.stat"
        file_oc1="${runt}/1_SHORT/ocean.output"      ;  file_oc2="${runt}/2_SHORT/ocean.output"
        file_roc1="${runt}/1_SHORT/REF/ocean.output" ;  file_roc2="${runt}/2_SHORT/REF/ocean.output"
        xfilev1=0 ; xfilev2=0 ; xfile_oc1=0 ; xfile_oc2=0

        echo "                                           " >> endjob.txt
        echo "        STREAM 1 results :                 " >> endjob.txt
        echo "        -----------------                  " >> endjob.txt
        #
        #-------
        # solver.stat STREAM 1
        checkfile ${file1}  'endjob.txt' NORM yes ; xfilev1=$?
        checkfile ${filev1} 'endjob.txt' NORM ; xfilev1=$?
    
        mak_diff_file -x ${xfilev1} -a ${file1} -b ${filev1} -r ${runt} -s STREAM_1 -t ${REF_TAG}
    
        #-------
        # ocean.output STREAM 1
        checkfile ${file_oc1}  'endjob.txt' NORM yes ; xfile_oc1=$?
        checkfile ${file_roc1} 'endjob.txt' NORM ; xfile_oc1=$?
    
        mak_diff_file -x ${xfile_oc1} -a ${file_oc1} -b ${file_roc1} -r ${runt} -s STREAM_1 -t ${REF_TAG}
    
        echo "                                           " >> endjob.txt
        echo "        STREAM 2 results :                 " >> endjob.txt
        echo "        -----------------                  " >> endjob.txt
        #
        #-------
        # solver.stat STREAM 2
        checkfile ${file2}  'endjob.txt' NORM yes ; xfilev2=$?
        checkfile ${filev2} 'endjob.txt' NORM ; xfilev2=$?
    
        mak_diff_file -x ${xfilev2} -a ${file2} -b ${filev2} -r ${runt} -s STREAM_2 -t ${REF_TAG}
    
        #-------
        # ocean.output STREAM 2
        checkfile ${file_oc2}  'endjob.txt' NORM yes ; xfile_oc2=$?
        checkfile ${file_roc2} 'endjob.txt' NORM ; xfile_oc2=$?
    
        mak_diff_file -x ${xfile_oc2} -a ${file_oc2} -b ${file_roc2} -r ${runt} -s STREAM_2 -t ${REF_TAG}
    
    done
    #
    #-------
fi

#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# ###############################################
# VI. SEND THE ASSESSMENT FILE REPORT "endjob.txt"
# ###############################################
mail -s "$TITLE" ${EMAIL} < endjob.txt