#!/bin/ksh
# $Id$
######################################################
# Original : C. Talandier for NEMO team
# Contact  : nemo_st@locean-ipsl.upmc.fr
#
# It is automatically launched (by the cron_jobs.ksh 
# script) when all runs (mon, mpi, (omp)) for a given 
# configuration are finished 
#
# INPUT ARGS: 1 
#    - EXPER: name of the experience for which the 
#             assessment will be done
#
# WORK: It aims to (for each standard configuration) :
#
#    I. Check Executable Memory size
#       - from memory_size.txt output file
#      *- compare it to a reference version 
#
#   II. Check CPU time used (if specified)
#       - from timing output files
#      *- compare it to a reference version 
#
#  III. Check the reproductibility between mon & mpi runs
#       - making diff of solver.stat output files
#
#   IV. Check the restartability for mon & mpi runs
#       - making diff of solver.stat output files
#
#    V.*Compare current results to a reference one
#        
#   VI. Send the assessment file "endjob_EXPER.txt" using 
#       user's e-mail
#        
# * this step is performed only if a reference tag 
#   is passing through the specifs_EXPER.txt file
#   See the Makefile under ./config/NVTK directory
#
# It is based on the directories architecture below :
#
#         NEMO_VALID/WNAME_CONFIG/
#                    |-- mon            --> MONO run
#                    |   |-- 1_SHORT
#                    |   |-- 2_SHORT
#                    |   |-- LONG
#                    |   `-- GTIME
#                    |-- mpi            --> MPI run
#                    |   |-- 1_SHORT
#                    |   |-- 2_SHORT
#                    |   |-- LONG
#                    |   `-- GTIME
#                    `-- omp            --> Open-MP run
#                        |-- 1_SHORT
#                        |-- 2_SHORT
#                        |-- LONG
#                        `-- GTIME
#
###########################################################
#set -xv 
###########################################################
CONF=`basename $( pwd ) | cut -c2-`
EXPER=$1
TARGET=sx8brodie
INFOF="./infos/specifs_${EXPER}.txt"
LRTYPE="$( ( awk '/runs=/ { printf  "%s %s %s", $2 , $3 , $4}' ${INFOF} ) )"
MPIDCI="$( ( awk '/mpis=/ { printf  "%s ", $2}' ${INFOF} ) )"
MPIDCJ="$( ( awk '/mpis=/ { printf  "%s ", $3}' ${INFOF} ) )"
MPIDCIJ="$( ( awk '/mpis=/ { printf  "%s ", $4}' ${INFOF} ) )"
VTIME=$( ( awk '/time=/ { print $2 }' ${INFOF} ) )
REF_TAG=$( (awk '/tagname=/ {print $2}' ${INFOF}) )
URL_USE=$( (awk '/URL:/ {print $0}' ${INFOF} ) )
REV_USE=$( (awk '/Revision:/ {print $2}' ${INFOF} ) )
TITLE="NEMO VALIDATION ${EXPER} : $CONF on $TARGET"
EMAIL='ctlod@locean-ipsl.upmc.fr'

OUTPUTF="endjob_${EXPER}.txt"

\rm ${OUTPUTF} 2> /dev/null
touch ${OUTPUTF}

if [ ${CONF} == 'ORCA2_LIM' ] ; then
    CONFS=ORCA2
elif [ ${CONF} == 'ORCA2_LIM3' ] ; then
    CONFS=OR2L3
elif [ ${CONF} == 'GYRE' ] ; then
    CONFS=GYREO
elif [ ${CONF} == 'GYRE_LOBSTER' ] ; then
    CONFS=GYREL
elif [ ${CONF} == 'ORCA2_LIM_PISCES' ] ; then
    CONFS=OR2LP
elif [ ${CONF} == 'ORCA2_OFF_PISCES' ] ; then
    CONFS=OR2OP
elif [ ${CONF} == 'ZAGRIF' ] ; then
    CONFS=ZAGRI
else
    CONFS=${CONF}
fi

###########################################################
# checkfile() is a local function to check if a file exist
# 5 arguments:
#    - zf2check : file name to check
#    - zoutfile : output file name to print results
#    - zwork    : kind of prints to perform
#    - zreprint : print a message once (yes/no)
#    - ziter    : (optional) print time steps number (yes/no)
###########################################################
################ Begin checkfile function #################
function checkfile
{
# name of the file to check
zf2check=$1
# print output file name
zoutfile=$2
# white space prints
zworkt=$3
# Print a message only once
zreprint=$4
# Print iterations number
ziter=$5

#set -x 

test -n "${ziter}"
zrepiter=$?


genf=$( basename ${zf2check} )

case "${zworkt}" in
      'NORM' )  blnk='       ' ;;
      'DIFF' )  blnk=' '  ;;
esac
fmes4=" - file ${zf2check} doesn't exist "

# Check file exists
if [ -a ${zf2check} ] ; then
    fmes1=" List ${genf} files and check date creation:   "
    fmes2=" `ls -o ${zf2check}`           "
    fmes3=" - file ${zf2check} is empty  "
    fmes4="                  Num. time steps:   done  /  expected "
    if [ ${zreprint} = 'yes' ] ; then 
        if [ ${zrepiter} == 0 ] ; then 
            echo "${blnk}${fmes1}" "${fmes4}" >> ${zoutfile}
        else
            echo "${blnk}${fmes1}"          >> ${zoutfile}
        fi
    fi

   # Check file is not empty
   if [ -s ${zf2check} ] ; then
       if [ ${zrepiter} == 0 ] ; then
           # Check run's iterations number from solver.stat
           basedir="`dirname ${zf2check}`"
           echo "`tail -1 ${zf2check}`" > temp1
           echo "`grep nn_itend ${basedir}/namelist`" > temp2
           typeset -Z4 ziter_run
           typeset -Z4 ziter_ask
           ziter_run="`awk '{print $2}' temp1`"
           ziter_ask="`awk '{print $3}' temp2`"
           \rm temp1 temp2
           [ ${zoutfile} ] && echo "${blnk}${fmes2}" "   ${ziter_run}  /   ${ziter_ask}" >> ${zoutfile}
           xist=0 ; return ${xist}
       else
           [ ${zoutfile} ] && echo "${blnk}${fmes2}" >> ${zoutfile}
           xist=0 ; return ${xist}
       fi
   else
       [ ${zoutfile} ] &&  echo "${blnk}${fmes3}" >> ${zoutfile}
       xist=1 ; return ${xist}
   fi
else
   [ ${zoutfile} ] && echo "${blnk}${fmes4}" >> ${zoutfile}
   xist=1 ; return ${xist}
fi
return
}
############### End checkfile function ###############
######################################################

# Get the name of the present script
nscript=$( (basename $0) )

###########################################################
# mak_diff_file() is a local function which performs diff
# on some files and print messages in the endjob_EXPER.txt file
# 7 input arguments:
#    - zfex    : (0/1) (make/do not make) diff (files doesn't exist)
#    - zfile1  : first input file name
#    - zfile2  : second input file name
#    - zruntype: run type (mon, mpi, omp)
#    - zstream : stream of the run (long, stream_1, stream_2)
#    - ztag    : the tag name to which are compared files
#    - zkind   : kind of the check (repro/resta)
###############################################################
################ Begin mak_diff_file function #################
function mak_diff_file
{
#
# help_diff() To print help
#
function help_diff
{
  echo "Usage: ${nscript} [-x] [-a] [-b] [-r] [-s] [-t] [-k]"
  echo "Options: These are optional argument"
  echo " -x : (0/1) (make/do not make) diff "
  echo " -a : first input file name"
  echo " -b : second input file name"
  echo " -r : run type (mon, mpi, omp)"
  echo " -s : stream of the run (long, stream_1, stream_2)"
  echo " -t : tag name to which are compared files"
  echo " -k : kind of the check (repro/resta)" 
  echo " their values are not taken)"
  exit 1
}

##set -xv

# Initialization
zfex=1
zfile1=
zfile2=
zruntype=
zstream=
ztag=
zkind=none
outdir='./infos/inter/'

# Get arguments
while getopts x:a:b:r:s:t:k: opt
do
  case "$opt" in
    x) zfex="$OPTARG";;
    a) zfile1="$OPTARG";;
    b) zfile2="$OPTARG";;
    r) zruntype="$OPTARG";;
    s) zstream="$OPTARG";;
    t) ztag="$OPTARG";;
    k) zkind="$OPTARG";;
    *) help_diff;;
  esac
done

zgenf=$( basename ${zfile1} )
zgenftrc=$( basename ${zfile1%.*} )

case "$zkind" in
  none ) 
      zmesg1="                   Current run                                                     ${ztag} run   "
      zmesg2="                   -----------                                                     --------------"
      zmesg_NOK="        ${zstream} ${zgenf} is NOT identical to the version ${ztag} one"
      zmesg_OK="        ${zstream} ${zgenf} IS IDENTICAL to the version ${ztag} one  "
      zmesg_wht="        NO comparison between current ${zstream} and the version ${ztag} ${zgenf} "
      bnam=${outdir}'cvref_'${zgenftrc}_${zstream}_${zruntype}_${EXPER} ;;
  repro ) 
      zmesg1="                   mon run                                                         ${zruntype} run      "
      zmesg2="                   -------                                                         -------      "
      zmesg_NOK="        NO NO NO NO NO NO for the current version"
      zmesg_OK="        YES YES YES YES YES YES YES YES for the current version "
      zmesg_wht="        NO CONCLUSION !!!!!  "
      bnam=${outdir}${zkind}_${zgenftrc}_${EXPER} ;;
  resta ) 
      zmesg1="                   LONG Stream                                                     SHORT Streams"
      zmesg2="                   -----------                                                     -------------"
      zmesg_NOK="        NO NO NO NO NO NO for the current version i.e. LONG stream != ( 1_SHORT + 2_SHORT ) streams "
      zmesg_OK="        YES YES YES YES YES YES YES YES for the current version i.e. LONG stream = ( 1_SHORT + 2_SHORT ) streams "
      zmesg_wht="        NO CONCLUSION !!!!!! "
      bnam=${outdir}${zkind}_${zgenftrc}_${EXPER} ;;
esac


if [ ${zfex} == 0 ] ;  then
    \rm ${bnam}.txt 2> /dev/null
    echo "                                    Make difference between ${zgenf} files                       "  > ${bnam}.txt
    echo "                                            ${zstream} ${zruntype} run                           " >> ${bnam}.txt
    echo "${zmesg1}"                                                                                         >> ${bnam}.txt
    echo "${zmesg2}"                                                                                         >> ${bnam}.txt
    sdiff    ${zfile1} ${zfile2} >>  ${bnam}.txt
    sdiff -s ${zfile1} ${zfile2} >>  ${bnam}_dd.txt

    #---------
    if [ ${zgenf} = 'ocean.output' ] ; then
        echo "                                                                                                 " >> ${outdir}${zstream}_head.txt
        echo "                                    Make difference between ${zgenf} files                       "  > ${outdir}${zstream}_head.txt
        echo "                                            ${zstream} ${zruntype} run                           " >> ${outdir}${zstream}_head.txt
        echo "                   Current run                                                     ${ztag} run   " >> ${outdir}${zstream}_head.txt
        echo "                   -----------                                                     --------------" >> ${outdir}${zstream}_head.txt
        echo "                                                                                                 " >> ${outdir}${zstream}_head.txt
        zfout=${bnam}_dd.txt
        zspmes=" See ${bnam}.txt file for full difference"
    else
        touch ${outdir}${zstream}_head.txt
        zfout=${bnam}.txt
        zspmes=""
    fi
    #---------
    if [ -s ${bnam}_dd.txt ] ; then
    # Differences exist between solver.stat or ocean.output files
       echo "                                           " >> ${OUTPUTF}
       echo "${zmesg_NOK}"                                >> ${OUTPUTF}
       echo "           ${zspmes}                       " >> ${OUTPUTF}
       echo "                                           " >> ${OUTPUTF}
       cat ${outdir}${zstream}_head.txt ${zfout}     >> ${OUTPUTF}
       echo "                                           " >> ${OUTPUTF}
       echo "                                           " >> ${OUTPUTF}
    else
       # No differences exist between solver.stat or ocean.output files
       echo "                             " >> ${OUTPUTF}
       echo "${zmesg_OK}"                   >> ${OUTPUTF}
       echo "                             " >> ${OUTPUTF}
    fi

    rm -f ${outdir}*_dd* ${outdir}*_head.txt 2> /dev/null

else
       # It is not possible to conclude because files are missing or empty
       echo "                             " >> ${OUTPUTF}
       echo "${zmesg_wht}"                  >> ${OUTPUTF}
       echo "                             " >> ${OUTPUTF}
fi

return

}
############### End mak_diff_file function ####################
###############################################################

###########################################################
# concat_perf() is a local function to gather performances
# files (perfs_runtype.txt_xxxx) build with the hpmcount 
# command on Zahir and standard(s) error output file(s)
# runtypeCONFNAME_err
# 1 argument:
#    - zconf : configuration name
#############################################################
################ Begin concat_perf function #################
function concat_perf
{

#set -x

# name of the configuration
zconf=$1

# List of files to treat
zcatfe[0]="mon/LONG/${EXPER}/mon${zconf}_err"
zcatfe[1]="mon/GTIME/${EXPER}/mon${zconf}_err"
zcatfe[2]="mpi/GTIME/${EXPER}/mpi${zconf}_err"

ind=0
while [ ${ind} -lt 3 ] ; do

       zpath=$( ( dirname ${zcatfe[${ind}]} ) )
       zname=$( ( basename ${zcatfe[${ind}]} ) )
       cd ${zpath}
       cp ${zname} ${zname}_save
       for kf in `ls perf*` ; do
           cat ${kf} >> ${zname}
       done
       cd ../../..
       let ind=${ind}+1
done

return
}
############### End concat_perf function ###############
########################################################


# ###############################
# 0. EXPERIENCE NAME
# ###############################
echo " ############################                           "  > ${OUTPUTF}
echo " EXPERIENCE NAME: $EXPER                                " >> ${OUTPUTF}
echo " ############################                           " >> ${OUTPUTF}
echo "                                                        " >> ${OUTPUTF}

# ###############################
# 0.I LIST OF FILES TESTED, i.e. stored under NVTK/2TEST directory
# ###############################
echo " #################################################      " >> ${OUTPUTF}
echo " FILES STORED UNDER NVTK/${EXPER} DIRECTORY & TESTED    " >> ${OUTPUTF}
echo " #################################################      " >> ${OUTPUTF}
echo "                                                        " >> ${OUTPUTF}
LSFILES=`tail -2 ${INFOF} | head -1 | wc -w`
if [ $LSFILES != 0 ] ; then
    for file in `tail -2 ${INFOF} | head -1` ; do 
        echo "   -" $file                                       >> ${OUTPUTF}
    done
else
    echo "                                                    " >> ${OUTPUTF}
    echo "   ---> NO files tested                             " >> ${OUTPUTF}
    echo "                                                    " >> ${OUTPUTF}
fi
echo "                                                        " >> ${OUTPUTF}

# ###############################
# 0.II LIST OF CPP KEYS USED
# ###############################
echo " #################################################      " >> ${OUTPUTF}
echo " CPP KEYS USED FOR ${EXPER} EXPERIENCE                  " >> ${OUTPUTF}
echo " #################################################      " >> ${OUTPUTF}
echo "                                                        " >> ${OUTPUTF}
for file in `tail -2 ${INFOF} | tail -1` ; do 
    echo "   -" $file                                           >> ${OUTPUTF}
done
echo "                                                        " >> ${OUTPUTF}


# ###############################
# ###############################
# 0.III NEMO MPI DECOMPOSITION USED
# ###############################
for runt in ${LRTYPE} ; do
    if [ ${runt} = 'mpi' ] ; then
        echo " #############################                               " >> ${OUTPUTF}
        echo " NEMO MPI DECOMPOSITION USED                                 " >> ${OUTPUTF}
        echo " #############################                               " >> ${OUTPUTF}
        echo "                                                             " >> ${OUTPUTF}
        echo "   jpni = ${MPIDCI} ; jpnj = ${MPIDCJ} ; jpnij = ${MPIDCIJ}  " >> ${OUTPUTF}
        echo "                                                             " >> ${OUTPUTF}
    fi
done

# ###############################
# ###############################
# 0.IV NEMO SOURCE FILES INFORMATION, i.e. revision & URL downloaded
# ###############################
echo " #############################                          " >> ${OUTPUTF}
echo " NEMO SOURCE FILES INFORMATION                          " >> ${OUTPUTF}
echo " #############################                          " >> ${OUTPUTF}
echo "                                                        " >> ${OUTPUTF}
echo "   NEMO downloaded from server: ${URL_USE}              " >> ${OUTPUTF}
echo "   NEMO revision : ${REV_USE}                           " >> ${OUTPUTF}
echo "                                                        " >> ${OUTPUTF}

if [ ${TARGET} = 'aix6' -a ${VTIME} = 'timing' ] ; then concat_perf ${CONFS} ; fi

#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# ###############################
# I. CHECK EXECUTABLE MEMORY SIZE
# ###############################
filecs="mon/LONG/${EXPER}/memory_size.txt"  ;  filers="mon/LONG/${REF_TAG}/memory_size.txt"
filecm="mon/LONG/${EXPER}/mon${CONFS}_err"  ;  filerm="mon/LONG/${REF_TAG}/mon${CONFS}_err"
xfilecs=0  ; xfilecm=0 ; xfilers=0 ; xfilerm=0
echo " ############################                           " >> ${OUTPUTF}
echo " CHECK EXECUTABLE MEMORY SIZE                           " >> ${OUTPUTF}
echo " ############################                           " >> ${OUTPUTF}
echo "                                                        " >> ${OUTPUTF}
#--------
                  checkfile ${filecs} ${OUTPUTF} DIFF yes ; xfilecs=$?
[ ${REF_TAG} ] && checkfile ${filers} ${OUTPUTF} DIFF no  ; xfilers=$?
                  checkfile ${filecm} ${OUTPUTF} DIFF no  ; xfilecm=$?
[ ${REF_TAG} ] && checkfile ${filerm} ${OUTPUTF} DIFF no  ; xfilerm=$?

# If current version files exist
if [ ${xfilecs} == 0 ] ;  then

    if [ ${REF_TAG} ] ; then 
        ./MEM_size_${TARGET}.ksh -a ${xfilecm} -b ${xfilers} -d ${xfilerm} -e ${EXPER} -c ${CONFS} -o ${OUTPUTF} -t ${REF_TAG} 
    else
        ./MEM_size_${TARGET}.ksh -a ${xfilecm} -b ${xfilers} -d ${xfilerm} -e ${EXPER} -c ${CONFS} -o ${OUTPUTF}
    fi

else
    echo "                                                    " >> ${OUTPUTF}
    echo "   ---> NO memory size check                        " >> ${OUTPUTF}
    echo "                                                    " >> ${OUTPUTF}
fi
#
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# #######################
# II. CHECK CPU TIME USED
# #######################
if [ ${VTIME} = 'timing' ] ; then

    echo "                                                        " >> ${OUTPUTF}
    echo " ######################                                 " >> ${OUTPUTF}
    echo " CHECK CPU TIME USED                                    " >> ${OUTPUTF}
    echo " ######################                                 " >> ${OUTPUTF}
    echo "                                                        " >> ${OUTPUTF}
    #------

    for runt in ${LRTYPE} ; do

        echo "                                                    " >> ${OUTPUTF}
        echo "   ---> Timing for the ${runt} run :                " >> ${OUTPUTF}
        echo "        ----------------------------                " >> ${OUTPUTF}
        echo "                                                    " >> ${OUTPUTF}

        xfilecv=0 ; xfilerv=0

        # Check that timing file for the ${runt} current version exist
        filecv="${runt}/GTIME/${EXPER}/${runt}${CONFS}_err" 
        checkfile ${filecv} ${OUTPUTF} NORM yes ; xfilecv=$?

        # Check that timing file for the ${runt} reference version exist
        if [ ${REF_TAG} ] ; then 
            filerv="${runt}/GTIME/${REF_TAG}/${runt}${CONFS}_err" 
            checkfile ${filerv} ${OUTPUTF} NORM no ; xfilerv=$?
        else 
            filerv='NoFile'
            xfilerv=1
        fi
        echo "                                                             " >> ${OUTPUTF}

        if [ ${xfilecv} == 0 ] ; then 
            if [ ${REF_TAG} ] ; then 
                ./CPU_time_${TARGET}.ksh -r ${runt} -a ${filecv} -b ${filerv} -o ${OUTPUTF} -x ${xfilerv} -t ${REF_TAG} 
            else
                ./CPU_time_${TARGET}.ksh -r ${runt} -a ${filecv} -b ${filerv} -o ${OUTPUTF} -x ${xfilerv}
            fi
        fi

    done

fi

#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# ########################################################
# III. CHECK REPRODUCTIBILITY: mon .vs. mpi & mon .vs .omp
# ########################################################
echo "                                                        " >> ${OUTPUTF}
echo " ######################                                 " >> ${OUTPUTF}
echo " CHECK REPRODUCTIBILITY                                 " >> ${OUTPUTF}
echo " ######################                                 " >> ${OUTPUTF}
echo "                                                        " >> ${OUTPUTF}
#------
for runt in ${LRTYPE} ; do

    if [ ${runt} != 'mon' ] ; then

        if  [ ${CONF} != 'ORCA2_OFF_PISCES' ] ; then
           echo "                                                          " >> ${OUTPUTF}
           echo "   ---> Ocean Reproductibility mon .vs. ${runt} ? :       " >> ${OUTPUTF}
           echo "        -------------------------------------------       " >> ${OUTPUTF}

           filemon="mon/LONG/${EXPER}/solver.stat" ;  filexxx="${runt}/LONG/${EXPER}/solver.stat"
           xfilemon=0

           checkfile ${filemon} ${OUTPUTF} NORM yes yes ; xfilemon=$?
           checkfile ${filexxx} ${OUTPUTF} NORM no  yes ; xfilemon=$?
        
           mak_diff_file -x ${xfilemon} -a ${filemon} -b ${filexxx} -r ${runt} -s LONG -k repro
        fi
        
        if [ ${CONF} = 'GYRE_LOBSTER' -o ${CONF} = 'ORCA2_LIM_PISCES' -o ${CONF} = 'ORCA2_OFF_PISCES' ] ; then
            # check that ocean.output files exist
            filemon="mon/LONG/${EXPER}/ocean.output" ;  filexxx="${runt}/LONG/${EXPER}/ocean.output"
            xfilemon=0

            checkfile ${filemon} ${OUTPUTF} NORM yes ; xfilemon=$?
            checkfile ${filexxx} ${OUTPUTF} NORM no  ; xfilemon=$?
            
            # check that tracer.stat files have been built based on ocean.output ones
            filemon="mon/LONG/${EXPER}/tracer.stat" ;  filexxx="${runt}/LONG/${EXPER}/tracer.stat"
            xfilemon=0
            if [ ${CONF} = 'GYRE_LOBSTER' ] ; then
               echo "                                                        " >> ${OUTPUTF}
               echo "   ---> LOBSTER Reproductibility mon .vs. ${runt} ? :   " >> ${OUTPUTF}
               echo "        ---------------------------------------------   " >> ${OUTPUTF}

               ./LOBSTER_stat.ksh -a ${filemon} -b ${filexxx} -c ocean.output
            else
               echo "                                                        " >> ${OUTPUTF}
               echo "   ---> PISCES  Reproductibility mon .vs. ${runt} ? :   " >> ${OUTPUTF}
               echo "        ---------------------------------------------   " >> ${OUTPUTF}

               ./PISCES_stat.ksh -a ${filemon} -b ${filexxx} -c ocean.output
            fi

            checkfile ${filemon} ${OUTPUTF} NORM yes ; xfilemon=$?
            checkfile ${filexxx} ${OUTPUTF} NORM no  ; xfilemon=$?
            
            mak_diff_file -x ${xfilemon} -a ${filemon} -b ${filexxx} -r ${runt} -s LONG -k repro
        fi
    fi
done

#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# ##########################################
# IV. CHECK RESTARTABILITY: mon mpi omp runs
# ##########################################
echo "                                                        " >> ${OUTPUTF}
echo " ####################                                   " >> ${OUTPUTF}
echo " CHECK RESTARTABILITY                                   " >> ${OUTPUTF}
echo " ####################                                   " >> ${OUTPUTF}
echo "                                                        " >> ${OUTPUTF}

for runt in ${LRTYPE} ; do
    if  [ ${CONF} != 'ORCA2_OFF_PISCES' ] ; then
       echo "                                                          " >> ${OUTPUTF}
       echo "   ---> ${runt} ocean restartability ? :                  " >> ${OUTPUTF}
       echo "        --------------------------------                  " >> ${OUTPUTF}

       file1="${runt}/1_SHORT/${EXPER}/solver.stat" ;  file2="${runt}/2_SHORT/${EXPER}/solver.stat" ; file3="${runt}/LONG/${EXPER}/solver.stat"
       xfile1=0
       #-------
       checkfile ${file1} ${OUTPUTF} NORM yes yes ; xfile1=$?
       checkfile ${file2} ${OUTPUTF} NORM no  yes ; xfile1=$?
       #
       cat ${file1} ${file2}  > short_solver_${runt}.stat
       mak_diff_file -x ${xfile1} -a ${file3} -b short_solver_${runt}.stat -r ${runt} -k resta
       \rm short_solver_${runt}.stat
    fi 
    #-------

    if [ ${CONF} = 'GYRE_LOBSTER' -o ${CONF} = 'ORCA2_LIM_PISCES' -o ${CONF} = 'ORCA2_OFF_PISCES' ] ; then

       # check that ocean.output files exist
        file1="${runt}/2_SHORT/${EXPER}/ocean.output" ;  file2="${runt}/LONG/${EXPER}/ocean.output"
        xfile1=0

        checkfile ${file1} ${OUTPUTF} NORM yes ; xfile1=$?
        checkfile ${file2} ${OUTPUTF} NORM no  ; xfile1=$?
            
        # check that tracer.stat files have been built based on ocean.output ones
        file1="${runt}/2_SHORT/${EXPER}/tracer.stat" ;  file2="${runt}/LONG/${EXPER}/tracer.stat"
        xfile1=0

        if [ ${CONF} = 'GYRE_LOBSTER' ] ; then
           echo "                                                        " >> ${OUTPUTF}
           echo "   ---> ${runt} LOBSTER restartability ? :              " >> ${OUTPUTF}
           echo "        ----------------------------------              " >> ${OUTPUTF}

           ./LOBSTER_stat.ksh -a ${file1} -b ${file2} -c ocean.output
        else
           echo "                                                        " >> ${OUTPUTF}
           echo "   ---> ${runt} PISCES  restartability ? :              " >> ${OUTPUTF}
           echo "        ---------------------------------------------   " >> ${OUTPUTF}

           ./PISCES_stat.ksh -a ${file1} -b ${file2} -c ocean.output
        fi


        checkfile ${file1} ${OUTPUTF} NORM yes ; xfile1=$?
        checkfile ${file2} ${OUTPUTF} NORM no  ; xfile1=$?
        #
        mak_diff_file -x ${xfile1} -a ${file2} -b ${file1} -r ${runt} -k resta

    fi
    
done

#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# #######################################################
# V. COMPARE CURRENT ${runt} RESULTS TO THE $REF_TAG ONES
# #######################################################

# This step is realised only if a reference tag ${REF_TAG} is specified
if [ ${REF_TAG} ] ; then

    echo "                                                        " >> ${OUTPUTF}
    echo " ####################################################   " >> ${OUTPUTF}
    echo " COMPARE CURRENT VERSION RESULTS TO THE $REF_TAG ONES   " >> ${OUTPUTF}
    echo " ####################################################   " >> ${OUTPUTF}
    echo "                                                        " >> ${OUTPUTF}

    for runt in ${LRTYPE} ; do

        echo "   -----------------------------------------------------------------------------" >> ${OUTPUTF}
        echo "   ---> ${runt} results :  current version  |   ${REF_TAG} version  STREAM 1 & 2" >> ${OUTPUTF}
        echo '                        solver.stat    .vs.    solver.stat         '              >> ${OUTPUTF}
        echo '                        ocean.output   .vs.    ocean.output        '              >> ${OUTPUTF}

        file1="${runt}/1_SHORT/${EXPER}/solver.stat"          ;  file2="${runt}/2_SHORT/${EXPER}/solver.stat"
        filev1="${runt}/1_SHORT/${REF_TAG}/solver.stat"              ;  filev2="${runt}/2_SHORT/${REF_TAG}/solver.stat"
        file_oc1="${runt}/1_SHORT/${EXPER}/ocean.output"      ;  file_oc2="${runt}/2_SHORT/${EXPER}/ocean.output"
        file_roc1="${runt}/1_SHORT/${REF_TAG}/ocean.output"          ;  file_roc2="${runt}/2_SHORT/${REF_TAG}/ocean.output"
        xfilev1=0 ; xfilev2=0 ; xfile_oc1=0 ; xfile_oc2=0

        echo "                                           " >> ${OUTPUTF}
        echo "        STREAM 1 results :                 " >> ${OUTPUTF}
        echo "        -----------------                  " >> ${OUTPUTF}
        #
        #-------
        # solver.stat STREAM 1
        checkfile ${file1}  ${OUTPUTF} NORM yes ; xfilev1=$?
        checkfile ${filev1} ${OUTPUTF} NORM no  ; xfilev1=$?
    
        mak_diff_file -x ${xfilev1} -a ${file1} -b ${filev1} -r ${runt} -s STREAM_1 -t ${REF_TAG}
    
        #-------
        # ocean.output STREAM 1
        checkfile ${file_oc1}  ${OUTPUTF} NORM yes ; xfile_oc1=$?
        checkfile ${file_roc1} ${OUTPUTF} NORM no  ; xfile_oc1=$?
    
        mak_diff_file -x ${xfile_oc1} -a ${file_oc1} -b ${file_roc1} -r ${runt} -s STREAM_1 -t ${REF_TAG}
    
        echo "                                           " >> ${OUTPUTF}
        echo "        STREAM 2 results :                 " >> ${OUTPUTF}
        echo "        -----------------                  " >> ${OUTPUTF}
        #
        #-------
        # solver.stat STREAM 2
        checkfile ${file2}  ${OUTPUTF} NORM yes ; xfilev2=$?
        checkfile ${filev2} ${OUTPUTF} NORM no  ; xfilev2=$?
    
        mak_diff_file -x ${xfilev2} -a ${file2} -b ${filev2} -r ${runt} -s STREAM_2 -t ${REF_TAG}
    
        #-------
        # ocean.output STREAM 2
        checkfile ${file_oc2}  ${OUTPUTF} NORM yes ; xfile_oc2=$?
        checkfile ${file_roc2} ${OUTPUTF} NORM no  ; xfile_oc2=$?
    
        mak_diff_file -x ${xfile_oc2} -a ${file_oc2} -b ${file_roc2} -r ${runt} -s STREAM_2 -t ${REF_TAG}
    
    done
    #
    #-------
fi

#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
#@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@@
# ###############################################
# VI. SEND THE ASSESSMENT FILE REPORT "endjob_EXPER.txt"
# ###############################################
mail -s "$TITLE" ${EMAIL} < ${OUTPUTF}