New URL for NEMO forge!   http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.
sette_eval.sh in utils/CI/sette – NEMO

source: utils/CI/sette/sette_eval.sh @ 15065

Last change on this file since 15065 was 14981, checked in by acc, 3 years ago

#2673 . Reintegrate sette developments back onto main sette branch. This action closes #2673
  • Property svn:executable set to *
File size: 9.3 KB
Line 
1#!/bin/bash -f
2# set -vx
3# simple SETTE result evaluator
4#
5# This version should be run in the SETTE directory.
6# The machine name will be picked up from the sette.sh script but the location of the
7# validation directory needs to be set here (currently assumed to reside in the ../cfgs directory)
8#
9#########################################################################################
10######################### Start of function definitions #################################
11##
12errcnt=0
13nmatch=0
14nrmiss=0
15nvmiss=0
16
17function get_dorv() {
18  if [ $lastchange == 'old' ] ; then
19    dorv=`ls -1rt $vdir/$mach/ | tail -1l `
20    dorv=`echo $dorv | sed -e 's:.*/::'`
21    dorv2=`ls -1rt $vdir/$mach/ 2>/dev/null | tail -1l `
22    dorv2=`echo $dorv2 | sed -e 's:.*/::'`
23  else
24    dorv=$lastchange
25    dorv2=$lastchange
26  fi
27}
28
29function get_ktdiff() {
30  ktdiff=`diff ${1} ${2} | head -2 | grep it | awk '{ print $4 }'`
31}
32
33function get_ktdiff2() {
34  ktdiff=`diff ${1} ${2} |  head -2 | tail -1l | awk '{print $2}'`
35}
36
37function runcmpres(){
38#
39# compare *.stat file with reference file from a previous sette test or previous version
40# store in NEMO_VALID_REF at revision NEMO_REV_REF
41# Compares end of stat files from each
42#
43  vdir=$1
44  nam=$2
45  vdirref=$3
46  dorvref=$4
47  silent=$5
48#
49# get $dorv
50  get_dorv
51#
52# check if reference directory is present
53  if [ ! -d $vdirref/$mach/$dorvref/$nam ] || [ ! -d $vdir/$mach/$dorv/$nam ] ; then
54   if [ ! -d $vdirref/$mach/$dorvref/$nam ]; then
55     if [ $silent -eq 0 ] ; then
56      printf "%-27s %s\n" $nam " REFERENCE directory at $dorvref is MISSING"
57     else
58      nrmiss=$(( $nrmiss + 1 ))
59     fi
60   fi
61   if [ ! -d $vdir/$mach/$dorv/$nam ]; then
62     if [ $silent -eq 0 ] ; then
63      printf "%-27s %s\n" $nam " VALID     directory at $dorv is MISSING"
64     else
65      nvmiss=$(( $nvmiss + 1 ))
66     fi
67   fi
68   return
69  fi
70  nmatch=$(( $nmatch + 1 ))
71
72#
73  if [ -d $vdir/$mach/$dorv/$nam ]; then
74    f1s=$vdir/$mach/$dorv/$nam/LONG/run.stat
75    f1t=$vdir/$mach/$dorv/$nam/LONG/tracer.stat
76    f2s=$vdirref/$mach/$dorvref/$nam/LONG/run.stat
77    f2t=$vdirref/$mach/$dorvref/$nam/LONG/tracer.stat
78    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
79      printf "%-20s %s\n" $nam " incomplete test";
80      return;
81    fi
82    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
83      printf "%-20s %s\n" $nam " incomplete test";
84      return;
85    fi
86#
87    done_oce=0
88
89    if  [ -f $f1s ] && [ -f $f2s ] ; then
90      cmp -s $f1s $f2s
91      if [ $? == 0 ]; then
92        if [ $silent == 0 ]; then
93          printf "%-20s %s %s\n" $nam  " run.stat    files are identical "
94        fi
95      else
96        get_ktdiff $f1s $f2s
97        if [ $silent == 0 ]; then
98         printf "%-20s %s %s %-5s %s\n" $nam  " run.stat    files are DIFFERENT (results are different after " $ktdiff " time steps)"
99        else
100         errcnt=$(( $errcnt + 1 ))
101        fi
102#
103      fi
104    fi
105    # Check tracer.stat files (if they exist)
106#
107    if  [ -f $f1t ] && [ -f $f2t ] ; then
108      cmp -s $f1t $f2t
109      if [ $? == 0 ]; then
110        if [ $silent == 0 ]; then         
111          printf "%-20s %s %s\n" $nam  " tracer.stat files are identical "
112        fi
113      else
114        get_ktdiff2 $f1t $f2t
115        if [ $silent == 0 ]; then         
116         printf "%-20s %s %s %-5s %s\n" $nam  " tracer.stat files are DIFFERENT (results are different after " $ktdiff " time steps) "
117        else
118         errcnt=$(( $errcnt + 1 ))
119        fi
120      fi
121#
122    fi
123  fi
124}
125
126########################### END of function definitions #################################
127##                                                                                     ##
128##    Main script                                                                      ##
129##                                                                                     ##
130#########################################################################################
131#
132# LOAD param variable (COMPILER, NEMO_VALIDATION_DIR, SVN_CMD)
133  SETTE_DIR=$(cd $(dirname "$0"); pwd)
134  MAIN_DIR=$(dirname $SETTE_DIR)
135  quiet=0
136  . ./param.cfg
137
138  mach=${COMPILER}
139# overwrite revision (later) or compiler
140  if [ $# -gt 0 ]; then
141    while getopts r:R:c:v:V:qh option; do
142       case $option in
143          c) mach=$OPTARG;;
144          r) rev=$OPTARG;;
145          R) refrev=$OPTARG;;
146          q) quiet=1;;
147          v) SETTE_SUB_VAL=$OPTARG;;
148          V) SETTE_SUB_VAL2=$OPTARG
149             if [ -d ${NEMO_VALIDATION_DIR}/${SETTE_SUB_VAL2} ] ; then
150               export NEMO_VALIDATION_REF=${NEMO_VALIDATION_DIR}/${SETTE_SUB_VAL2}
151             else
152               echo "Requested comparison subdirectory: ${NEMO_VALIDATION_DIR}/${SETTE_SUB_VAL2} does not exist"
153             fi
154             ;;
155          h | *) echo ''
156                 echo 'sette_eval.sh : ' 
157                 echo '     display result for the latest change'
158                 echo ' -c COMPILER_name :'
159                 echo '     display result for the specified compiler'
160                 echo ' -r REVISION_number :'
161                 echo '     display sette results for the specified revision (set old for the latest revision available for each config)'
162                 echo ' -R REFERENCE REVISION_number :'
163                 echo '     compare sette results against the specified revision (use to over-ride value set in param.cfg)'
164                 echo ' -v sub_dir :'
165                 echo '     validation sub-directory below NEMO_VALIDATION_DIR'
166                 echo ' -V sub_dir2 :'
167                 echo '     2nd validation sub-directory below NEMO_VALIDATION_DIR'
168                 echo '     if set the comparison is between two subdirectory trees beneath NEMO_VALIDATION_DIR'
169                 echo ' -q : Activate quiet mode - only the number of differing results is returned'
170                 echo ''
171                 exit 42;;
172       esac
173    done
174    shift $((OPTIND - 1))
175  fi
176# if $1 (remaining arguments)
177  if [[ ! -z $1 ]] ; then rev=$1 ; fi
178
179  if [ ! -z $SETTE_SUB_VAL ] ; then
180   export NEMO_VALIDATION_DIR=$NEMO_VALIDATION_DIR/$SETTE_SUB_VAL
181   if [ -d $NEMO_VALIDATION_REF/$SETTE_SUB_VAL ] && [ -z $SETTE_SUB_VAL2 ] && [ ${quiet} -eq 0 ] ; then
182    while true; do
183        read -p "$NEMO_VALIDATION_REF/$SETTE_SUB_VAL exists. Do you wish to use it as a reference? " yn
184        case $yn in
185            [Yy]* ) export $NEMO_VALIDATION_REF/$SETTE_SUB_VAL; break;;
186            [Nn]* ) echo "Ok, continuing with ${NEMO_VALIDATION_REF}/MAIN as the reference directory"
187                    export NEMO_VALIDATION_REF=${NEMO_VALIDATION_REF}/MAIN
188                    break
189                    ;;
190            * ) echo "Please answer yes or no.";;
191        esac
192    done
193   fi
194  else
195   export NEMO_VALIDATION_DIR=${NEMO_VALIDATION_DIR}/MAIN
196   export NEMO_VALIDATION_REF=${NEMO_VALIDATION_REF}/MAIN
197  fi
198  NEMO_VALID=${NEMO_VALIDATION_DIR}
199  NEMO_VALID_REF=${NEMO_VALIDATION_REF}
200  if [ ! -z $refrev ] ; then
201    NEMO_REV_REF=${refrev}
202  fi
203#
204  if [ ! -d $NEMO_VALID ]; then
205    echo "$NEMO_VALID validation directory not found"
206    exit
207  fi
208#
209#
210# Show current revision tag and branch name
211#
212if [ ${quiet} -eq 0 ] ; then echo "" ; fi
213lastchange=`${SVN_CMD} info ${MAIN_DIR} | grep 'Last Changed Rev' | awk '{print $NF}'`
214revision=`${SVN_CMD} info ${MAIN_DIR} | grep 'Revision' | awk '{print $NF}'`
215branchname=`${SVN_CMD} info ${MAIN_DIR} | grep ^URL | awk -F ipsl/forge/projets/nemo/svn/ '{print $NF}'`
216if [ ${quiet} -eq 0 ] ; then echo "Current code is : $branchname @ r$revision  ( last change @ r$lastchange )" ; fi
217[ `${SVN_CMD} status -q ${MAIN_DIR}/{cfgs,tests,src} | wc -l` -ge 1 ] && lastchange=${lastchange}+
218
219# by default use the current lastchanged revision
220lastchange=${rev:-$lastchange}
221
222if [ ${quiet} -eq 0 ] ; then
223 echo ""
224 echo "SETTE evaluation for : "
225 echo ""
226 echo "       $branchname @ r$lastchange (last changed revision)"
227 echo ""
228 echo "       on $COMPILER arch file"
229 echo ""
230fi
231
232#
233# The script also needs the date or revision tag. Currently this is taken from the latest sub-directory found in each directory
234# 
235# before/after tests
236 if [ $lastchange == 'old' ] ; then
237    echo ""
238    echo "   !---- 'old' specified as revision => no comparison with reference results ----!   "
239    echo ""
240 else
241   if [ ${quiet} -eq 0 ] ; then
242    echo ""
243    echo "   !----result comparison check----!   "
244   fi
245   if [ $NEMO_VALID_REF != "/path/to/reference/sette/results" ]; then
246     if [ ${quiet} -eq 0 ] ; then
247      echo ''
248      echo 'check result differences between :'
249      echo "VALID directory : $NEMO_VALID at rev $lastchange"
250      echo 'and'
251      echo "REFERENCE directory : $NEMO_VALID_REF at rev $NEMO_REV_REF"
252      echo ''
253     fi
254     checklist=(GYRE_PISCES ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 ORCA2_SAS_ICE AGRIF_DEMO WED025 ISOMIP+ VORTEX ICE_AGRIF OVERFLOW LOCK_EXCHANGE SWG)
255     for repro_test in ${checklist[@]}
256     do
257        runcmpres $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $quiet
258     done
259     if [ ${quiet} -eq 0 ] ; then
260      echo ''
261     else
262      if [ $(( $nrmiss + $nvmiss )) -gt 0 ] ; then
263       echo $errcnt " differences from "$nmatch" matches. "$nrmiss" missing from REFERENCE "$nvmiss" missing from VALID"
264      else
265       echo $errcnt " differences from "$nmatch" matches. "
266      fi
267     fi
268   else
269     echo ''
270     echo ' No path for comparison specified. Result are not compare with any other revision. '
271     echo ' To do it please fill NEMO_VALID_REF and NEMO_REV_REF in param.cfg. '
272     echo ''
273   fi
274 fi
275#
276exit
Note: See TracBrowser for help on using the repository browser.