source: branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/SETTE/trusting.sh @ 5286

#!/bin/bash
#
# Author: Patrick Brockmann
# Contact: Patrick.Brockmann@cea.fr
# History:
# Modification: Nicolas-thierry.Martin@locean-ipsl.upmc.fr
#


# Set defaults
DEBUG=0; PUBLISH=0; status='OK'

# Get options && replace some initials settings
while [ $# -ne 0 ]; do
    case $1 in
        -b|--branch ) NEMO_BRAN=$2         ; shift 2;; -d|--debug  ) set -vx; DEBUG=1     ; shift  ;;
   -e|--email  ) MAIL=$2              ; shift 2;; -j|--job    ) NPROC=$2             ; shift 2;;
   -h|--help   ) cat trusting_help.txt; exit   ;; -m|--machine) ARCH=$2              ; shift 2;;
   -n|--newconf) CONF_TEST=$2         ; shift 2;; -r|--refconf) CONF_REFE=$2         ; shift 2;;
   -t|--time   ) TIME_LIMI=$2         ; shift 2;; -p|--publish) PUBLISH=1            ; shift  ;;
                   *) cat trusting_help.txt; exit   ;;
    esac
done

# Set super-computer & user environment
[[ ! -z "$ARCH" && . trusting.env ]] || { ./trusting.sh -h && exit; }
export DEBUG PUBLISH

# Initialization
#=============================================

echo ${DIR_TEST}
mkdir -p ${DIR_TEST} ${DIR_REFE}
cd ${DIR_TEST}
datestr1=$( LC_ALL=C date -d"$DATE_STR" +%Y-%m-%dT%H:%M )
printf "Date\n%s\n"       $datestr1 > mesg_01_date.txt

printf "Status\nFAILED\n"           > mesg_02_status.txt # Only at end status will change to OK

. trusting_func.sh
#=============================================


# 
#=============================================
for rel in $CDO $COMPILER $MPI $NETCDF; do
    printf "Step......................\nGet $rel release\n" > mesg_03_step.txt
                             arch_rel=$( echo $LOADEDMODULES | sed "s#.*:$rel/\([^:]*\).*#\1#" )
    [ -z "${arch_rel}"  ] && arch_rel=$( echo $PATH          | sed "s#.*$rel/\([^/]*\).*#\1#"  )
    [ $( echo $rel | grep "$CDO\|$COMPILER" ) ] && arch_rel=$( $rel --version | grep -m1 -o ' [0-9.]* ' )
    echo $rel ${arch_rel} >> model.log
done

sed -n 1p model.log | sed 'i\Fortran compiler' > mesg_04_curiec.txt
sed -n 2p model.log | sed 'i\MPI libraries   ' > mesg_05_mpicurie.txt
sed -n 3p model.log | sed 'i\NetCDF libraries' > mesg_06_netcdf_curie.txt


#---------------------------------------------
printf "Step......................\nCheckout XIOS, FCM-make(/EXTERNAL/fcm), NEMO engines (/NEMO) & configuration (/CONFIG)\n" > mesg_03_step.txt
rev=0
for dir in ${DIR_XIOS} ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_EFCM} ${NEMO_ENGI} ${NEMO_IPSL} ${NEMO_SETT} ${NEMO_TCMP}; do
    svn up $dir
    rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' )
    [ $dir == ${DIR_XIOS} ] && echo 'XIOS '${rev_loc} >> model.log && continue
    (( ${rev_loc} >= $rev )) && rev=${rev_loc}
done
echo 'NEMO '$rev >> model.log


#---------------------------------------------
printf "Step......................\nCompilation NEMO configuration %s\n" ${CONF_REF} > mesg_03_step.txt
cd ${NEMO_CONF}
. ${ARCH_ENV}     # Sourcing architecture environment
#[ -d ${CONF_TEST} ] && ./makenemo -n ${CONF_TEST} clean
./makenemo -n ${CONF_TEST} -r ${CONF_REFE} -m $ARCH -j $NPROC #> /dev/null 2>&1
[ ! -e ${CONF_TEST}/BLD/bin/nemo.exe ] && getout


#---------------------------------------------
printf "Step......................\nPreparing job\n" > mesg_03_step.txt
cd ${DIR_TEST}
find ${NEMO_CONF}/${CONF_TEST}/EXP00 -regex '.*\(cfg\|opa\|ref\|xml\)' -exec cp {} . \;

if [ ! -z "${NEMO_FORC}" ]; then
    cp ${NEMO_FORC}/${FORC_TAR} .
    tar -tvf ${FORC_TAR} > inputs_list.txt && tar -vxf ${FORC_TAR}
    [ $? -ne 0 ] && getout
    gunzip *.gz
else
    touch inputs_list.txt
fi


#---------------------------------------------
printf "Step......................\nRunning\n"       > mesg_03_step.txt
#${JOB_SUB} ${DIR_TEST}/../../batch.sh
job_num=$( ${JOB_SUB} ${DIR_TEST}/../../batch.sh | awk '{print $NF}')
[ $? -ne 0 ] && getout

time_elapsed=0; time_increment=300
sleep ${time_increment}
while [[ $( ${JOB_LIST} | grep $job_num ) && ${time_elapsed} -lt ${TIME_LIMI} ]] ; do
    printf "\n####################################################" >> computation.log
    ${JOB_VIEW} $job_num                                            >> computation.log
    sleep ${time_increment}
    let time_elapsed+=${time_increment}
done


#---------------------------------------------
printf "Step......................\nTest if ASCII output files exist\n"   > mesg_03_step.txt
if      [ ! -f ${output_test} ]; then

    getout
else if [  ]

    grep 'E R R O R' ocean.output    >> mesg_11_comments.txt
    sed 's/^/Time step :/' time.step >> mesg_11_comments.txt
fi


#---------------------------------------------
printf "Step......................\nTest ASCII output files difference\n" > mesg_03_step.txt
for file in ocean.output solver.stat tracer.stat; do
    diff $file ${DIR_REFE}/$file
    [ $? -ne 0 ] && status='FAILED'
done


#---------------------------------------------
printf "Step......................\nTest last restart files difference\n" > mesg_03_step.txt
diff_trusting 2>/dev/null
[ $? -ne 0 ] && status='FAILED'


#---------------------------------------------
echo 'Comments'                    >  mesg_11_comments.txt
grep 'W A R N I N G:' ocean.output >> mesg_11_comments.txt


#---------------------------------------------
if [ $status != 'FAILED' ]; then
    printf "Step......................\nGet Real CPU time\n" > mesg_03_step.txt
    real_cpu_time=$( ${JOB_VIEW} $job_num | awk 'END{print $9}')
    printf "Real CPU time\n%s\n" ${real_cpu_time} > mesg_08_realcputime.txt
else
    printf "Step......................\nTest InputFiles difference\n" > mesg_03_step.txt
    diff inputs_list.txt ${DIR_REFE}/inputs_list.txt
    [ $? -ne 0 ] && getout
    [[ $? -ne 0 && printf "InputFiles\n%s\n" "Different" > mesg_09_inputfiles.txt ]] || printf "InputFiles\n%s\n" "Same" > mesg_09_inputfiles.txt
fi


#---------------------------------------------
printf "Status\nOK\n" > mesg_02_status.txt
printf "Step......................\nCode is reliable\n" > mesg_03_step.txt

#---------------------------------------------
getout