[15159] | 1 | #!/bin/bash |
---|
[14897] | 2 | # set -vx |
---|
| 3 | # simple SETTE result evaluator |
---|
| 4 | # |
---|
| 5 | # This version should be run in the SETTE directory. |
---|
| 6 | # The machine name will be picked up from the sette.sh script but the location of the |
---|
| 7 | # validation directory needs to be set here (currently assumed to reside in the ../cfgs directory) |
---|
| 8 | # |
---|
| 9 | ######################################################################################### |
---|
| 10 | ######################### Start of function definitions ################################# |
---|
| 11 | ## |
---|
| 12 | errcnt=0 |
---|
| 13 | nmatch=0 |
---|
| 14 | nrmiss=0 |
---|
| 15 | nvmiss=0 |
---|
[15159] | 16 | TESTD_ROOT=LONG |
---|
[14897] | 17 | |
---|
[15159] | 18 | function get_testd() { |
---|
| 19 | sroot=$1/${TESTD_ROOT} |
---|
| 20 | ls -1d "$sroot"* &> /dev/null |
---|
| 21 | if [ $? -eq 0 ] ; then |
---|
| 22 | TESTD=`ls -1d "$sroot"* | head -1l` |
---|
| 23 | TESTD=`basename $TESTD` |
---|
| 24 | if [ ! -d $1/$TESTD ] ; then |
---|
| 25 | echo "TEST directory not found with rootname: "$TESTD_ROOT |
---|
| 26 | exit |
---|
| 27 | fi |
---|
| 28 | else |
---|
| 29 | TESTD=$TESTD_ROOT |
---|
| 30 | fi |
---|
| 31 | } |
---|
| 32 | |
---|
[14897] | 33 | function get_dorv() { |
---|
| 34 | if [ $lastchange == 'old' ] ; then |
---|
| 35 | dorv=`ls -1rt $vdir/$mach/ | tail -1l ` |
---|
| 36 | dorv=`echo $dorv | sed -e 's:.*/::'` |
---|
| 37 | dorv2=`ls -1rt $vdir/$mach/ 2>/dev/null | tail -1l ` |
---|
| 38 | dorv2=`echo $dorv2 | sed -e 's:.*/::'` |
---|
| 39 | else |
---|
| 40 | dorv=$lastchange |
---|
| 41 | dorv2=$lastchange |
---|
| 42 | fi |
---|
| 43 | } |
---|
| 44 | |
---|
| 45 | function get_ktdiff() { |
---|
| 46 | ktdiff=`diff ${1} ${2} | head -2 | grep it | awk '{ print $4 }'` |
---|
| 47 | } |
---|
| 48 | |
---|
| 49 | function get_ktdiff2() { |
---|
| 50 | ktdiff=`diff ${1} ${2} | head -2 | tail -1l | awk '{print $2}'` |
---|
| 51 | } |
---|
| 52 | |
---|
| 53 | function runcmpres(){ |
---|
| 54 | # |
---|
| 55 | # compare *.stat file with reference file from a previous sette test or previous version |
---|
| 56 | # store in NEMO_VALID_REF at revision NEMO_REV_REF |
---|
| 57 | # Compares end of stat files from each |
---|
| 58 | # |
---|
| 59 | vdir=$1 |
---|
| 60 | nam=$2 |
---|
| 61 | vdirref=$3 |
---|
| 62 | dorvref=$4 |
---|
| 63 | silent=$5 |
---|
| 64 | # |
---|
| 65 | # get $dorv |
---|
| 66 | get_dorv |
---|
| 67 | # |
---|
| 68 | # check if reference directory is present |
---|
| 69 | if [ ! -d $vdirref/$mach/$dorvref/$nam ] || [ ! -d $vdir/$mach/$dorv/$nam ] ; then |
---|
| 70 | if [ ! -d $vdirref/$mach/$dorvref/$nam ]; then |
---|
| 71 | if [ $silent -eq 0 ] ; then |
---|
| 72 | printf "%-27s %s\n" $nam " REFERENCE directory at $dorvref is MISSING" |
---|
| 73 | else |
---|
| 74 | nrmiss=$(( $nrmiss + 1 )) |
---|
| 75 | fi |
---|
| 76 | fi |
---|
| 77 | if [ ! -d $vdir/$mach/$dorv/$nam ]; then |
---|
| 78 | if [ $silent -eq 0 ] ; then |
---|
| 79 | printf "%-27s %s\n" $nam " VALID directory at $dorv is MISSING" |
---|
| 80 | else |
---|
| 81 | nvmiss=$(( $nvmiss + 1 )) |
---|
| 82 | fi |
---|
| 83 | fi |
---|
| 84 | return |
---|
| 85 | fi |
---|
| 86 | nmatch=$(( $nmatch + 1 )) |
---|
| 87 | |
---|
| 88 | # |
---|
| 89 | if [ -d $vdir/$mach/$dorv/$nam ]; then |
---|
[15159] | 90 | get_testd $vdir/$mach/$dorv/$nam |
---|
| 91 | if [ ! -d $vdir/$mach/$dorv/$nam/$TESTD ] ; then |
---|
| 92 | printf "%-20s %s (%s)\n" $nam " not tested" $TESTD; |
---|
| 93 | return; |
---|
| 94 | fi |
---|
| 95 | f1s=$vdir/$mach/$dorv/$nam/$TESTD/run.stat |
---|
| 96 | f1t=$vdir/$mach/$dorv/$nam/$TESTD/tracer.stat |
---|
| 97 | f2s=$vdirref/$mach/$dorvref/$nam/$TESTD/run.stat |
---|
| 98 | f2t=$vdirref/$mach/$dorvref/$nam/$TESTD/tracer.stat |
---|
[14897] | 99 | if [ ! -f $f1s ] && [ ! -f $f1t ] ; then |
---|
| 100 | printf "%-20s %s\n" $nam " incomplete test"; |
---|
| 101 | return; |
---|
| 102 | fi |
---|
| 103 | if [ ! -f $f2s ] && [ ! -f $f2t ] ; then |
---|
| 104 | printf "%-20s %s\n" $nam " incomplete test"; |
---|
| 105 | return; |
---|
| 106 | fi |
---|
| 107 | # |
---|
| 108 | done_oce=0 |
---|
| 109 | |
---|
| 110 | if [ -f $f1s ] && [ -f $f2s ] ; then |
---|
| 111 | cmp -s $f1s $f2s |
---|
| 112 | if [ $? == 0 ]; then |
---|
| 113 | if [ $silent == 0 ]; then |
---|
[15159] | 114 | printf "%-20s %s (%s)\n" $nam " run.stat files are identical " $TESTD |
---|
[14897] | 115 | fi |
---|
| 116 | else |
---|
| 117 | get_ktdiff $f1s $f2s |
---|
| 118 | if [ $silent == 0 ]; then |
---|
[15159] | 119 | printf "%-20s %s %s %-5s (%s)\n" $nam " run.stat files are DIFFERENT (results are different after " $ktdiff " time steps) " $TESTD |
---|
[14897] | 120 | else |
---|
| 121 | errcnt=$(( $errcnt + 1 )) |
---|
| 122 | fi |
---|
| 123 | # |
---|
| 124 | fi |
---|
| 125 | fi |
---|
| 126 | # Check tracer.stat files (if they exist) |
---|
| 127 | # |
---|
| 128 | if [ -f $f1t ] && [ -f $f2t ] ; then |
---|
| 129 | cmp -s $f1t $f2t |
---|
| 130 | if [ $? == 0 ]; then |
---|
| 131 | if [ $silent == 0 ]; then |
---|
[15159] | 132 | printf "%-20s %s (%s)\n" $nam " tracer.stat files are identical " $TESTD |
---|
[14897] | 133 | fi |
---|
| 134 | else |
---|
| 135 | get_ktdiff2 $f1t $f2t |
---|
| 136 | if [ $silent == 0 ]; then |
---|
[15159] | 137 | printf "%-20s %s %s %-5s (%s)\n" $nam " tracer.stat files are DIFFERENT (results are different after " $ktdiff " time steps) " $TESTD |
---|
[14897] | 138 | else |
---|
| 139 | errcnt=$(( $errcnt + 1 )) |
---|
| 140 | fi |
---|
| 141 | fi |
---|
| 142 | # |
---|
| 143 | fi |
---|
| 144 | fi |
---|
| 145 | } |
---|
| 146 | |
---|
| 147 | ########################### END of function definitions ################################# |
---|
| 148 | ## ## |
---|
| 149 | ## Main script ## |
---|
| 150 | ## ## |
---|
| 151 | ######################################################################################### |
---|
| 152 | # |
---|
| 153 | # LOAD param variable (COMPILER, NEMO_VALIDATION_DIR, SVN_CMD) |
---|
| 154 | SETTE_DIR=$(cd $(dirname "$0"); pwd) |
---|
| 155 | MAIN_DIR=$(dirname $SETTE_DIR) |
---|
| 156 | quiet=0 |
---|
| 157 | . ./param.cfg |
---|
| 158 | |
---|
| 159 | mach=${COMPILER} |
---|
| 160 | # overwrite revision (later) or compiler |
---|
| 161 | if [ $# -gt 0 ]; then |
---|
[15159] | 162 | while getopts r:R:c:v:V:T:qh option; do |
---|
[14897] | 163 | case $option in |
---|
| 164 | c) mach=$OPTARG;; |
---|
| 165 | r) rev=$OPTARG;; |
---|
| 166 | R) refrev=$OPTARG;; |
---|
| 167 | q) quiet=1;; |
---|
| 168 | v) SETTE_SUB_VAL=$OPTARG;; |
---|
| 169 | V) SETTE_SUB_VAL2=$OPTARG |
---|
| 170 | if [ -d ${NEMO_VALIDATION_DIR}/${SETTE_SUB_VAL2} ] ; then |
---|
| 171 | export NEMO_VALIDATION_REF=${NEMO_VALIDATION_DIR}/${SETTE_SUB_VAL2} |
---|
| 172 | else |
---|
| 173 | echo "Requested comparison subdirectory: ${NEMO_VALIDATION_DIR}/${SETTE_SUB_VAL2} does not exist" |
---|
| 174 | fi |
---|
| 175 | ;; |
---|
[15159] | 176 | T) TESTD_ROOT=$OPTARG;; |
---|
[14897] | 177 | h | *) echo '' |
---|
| 178 | echo 'sette_eval.sh : ' |
---|
| 179 | echo ' display result for the latest change' |
---|
| 180 | echo ' -c COMPILER_name :' |
---|
| 181 | echo ' display result for the specified compiler' |
---|
| 182 | echo ' -r REVISION_number :' |
---|
| 183 | echo ' display sette results for the specified revision (set old for the latest revision available for each config)' |
---|
| 184 | echo ' -R REFERENCE REVISION_number :' |
---|
| 185 | echo ' compare sette results against the specified revision (use to over-ride value set in param.cfg)' |
---|
[15159] | 186 | echo ' -T test_rootname :' |
---|
| 187 | echo ' root of test name to be checked. Valid choices ares: LONG, SHORT, REPRO. [default: LONG]' |
---|
[14897] | 188 | echo ' -v sub_dir :' |
---|
| 189 | echo ' validation sub-directory below NEMO_VALIDATION_DIR' |
---|
| 190 | echo ' -V sub_dir2 :' |
---|
| 191 | echo ' 2nd validation sub-directory below NEMO_VALIDATION_DIR' |
---|
| 192 | echo ' if set the comparison is between two subdirectory trees beneath NEMO_VALIDATION_DIR' |
---|
| 193 | echo ' -q : Activate quiet mode - only the number of differing results is returned' |
---|
| 194 | echo '' |
---|
| 195 | exit 42;; |
---|
| 196 | esac |
---|
| 197 | done |
---|
| 198 | shift $((OPTIND - 1)) |
---|
| 199 | fi |
---|
| 200 | # if $1 (remaining arguments) |
---|
| 201 | if [[ ! -z $1 ]] ; then rev=$1 ; fi |
---|
| 202 | |
---|
| 203 | if [ ! -z $SETTE_SUB_VAL ] ; then |
---|
| 204 | export NEMO_VALIDATION_DIR=$NEMO_VALIDATION_DIR/$SETTE_SUB_VAL |
---|
| 205 | if [ -d $NEMO_VALIDATION_REF/$SETTE_SUB_VAL ] && [ -z $SETTE_SUB_VAL2 ] && [ ${quiet} -eq 0 ] ; then |
---|
| 206 | while true; do |
---|
| 207 | read -p "$NEMO_VALIDATION_REF/$SETTE_SUB_VAL exists. Do you wish to use it as a reference? " yn |
---|
| 208 | case $yn in |
---|
| 209 | [Yy]* ) export $NEMO_VALIDATION_REF/$SETTE_SUB_VAL; break;; |
---|
| 210 | [Nn]* ) echo "Ok, continuing with ${NEMO_VALIDATION_REF}/MAIN as the reference directory" |
---|
| 211 | export NEMO_VALIDATION_REF=${NEMO_VALIDATION_REF}/MAIN |
---|
| 212 | break |
---|
| 213 | ;; |
---|
| 214 | * ) echo "Please answer yes or no.";; |
---|
| 215 | esac |
---|
| 216 | done |
---|
| 217 | fi |
---|
| 218 | else |
---|
| 219 | export NEMO_VALIDATION_DIR=${NEMO_VALIDATION_DIR}/MAIN |
---|
| 220 | export NEMO_VALIDATION_REF=${NEMO_VALIDATION_REF}/MAIN |
---|
| 221 | fi |
---|
| 222 | NEMO_VALID=${NEMO_VALIDATION_DIR} |
---|
| 223 | NEMO_VALID_REF=${NEMO_VALIDATION_REF} |
---|
| 224 | if [ ! -z $refrev ] ; then |
---|
| 225 | NEMO_REV_REF=${refrev} |
---|
| 226 | fi |
---|
| 227 | # |
---|
| 228 | if [ ! -d $NEMO_VALID ]; then |
---|
| 229 | echo "$NEMO_VALID validation directory not found" |
---|
| 230 | exit |
---|
| 231 | fi |
---|
| 232 | # |
---|
| 233 | # |
---|
| 234 | # Show current revision tag and branch name |
---|
| 235 | # |
---|
| 236 | if [ ${quiet} -eq 0 ] ; then echo "" ; fi |
---|
| 237 | lastchange=`${SVN_CMD} info ${MAIN_DIR} | grep 'Last Changed Rev' | awk '{print $NF}'` |
---|
| 238 | revision=`${SVN_CMD} info ${MAIN_DIR} | grep 'Revision' | awk '{print $NF}'` |
---|
| 239 | branchname=`${SVN_CMD} info ${MAIN_DIR} | grep ^URL | awk -F ipsl/forge/projets/nemo/svn/ '{print $NF}'` |
---|
| 240 | if [ ${quiet} -eq 0 ] ; then echo "Current code is : $branchname @ r$revision ( last change @ r$lastchange )" ; fi |
---|
| 241 | [ `${SVN_CMD} status -q ${MAIN_DIR}/{cfgs,tests,src} | wc -l` -ge 1 ] && lastchange=${lastchange}+ |
---|
| 242 | |
---|
| 243 | # by default use the current lastchanged revision |
---|
| 244 | lastchange=${rev:-$lastchange} |
---|
| 245 | |
---|
| 246 | if [ ${quiet} -eq 0 ] ; then |
---|
| 247 | echo "" |
---|
| 248 | echo "SETTE evaluation for : " |
---|
| 249 | echo "" |
---|
| 250 | echo " $branchname @ r$lastchange (last changed revision)" |
---|
| 251 | echo "" |
---|
| 252 | echo " on $COMPILER arch file" |
---|
| 253 | echo "" |
---|
| 254 | fi |
---|
| 255 | |
---|
| 256 | # |
---|
| 257 | # The script also needs the date or revision tag. Currently this is taken from the latest sub-directory found in each directory |
---|
| 258 | # |
---|
| 259 | # before/after tests |
---|
| 260 | if [ $lastchange == 'old' ] ; then |
---|
| 261 | echo "" |
---|
| 262 | echo " !---- 'old' specified as revision => no comparison with reference results ----! " |
---|
| 263 | echo "" |
---|
| 264 | else |
---|
| 265 | if [ ${quiet} -eq 0 ] ; then |
---|
| 266 | echo "" |
---|
| 267 | echo " !----result comparison check----! " |
---|
| 268 | fi |
---|
| 269 | if [ $NEMO_VALID_REF != "/path/to/reference/sette/results" ]; then |
---|
| 270 | if [ ${quiet} -eq 0 ] ; then |
---|
| 271 | echo '' |
---|
| 272 | echo 'check result differences between :' |
---|
| 273 | echo "VALID directory : $NEMO_VALID at rev $lastchange" |
---|
| 274 | echo 'and' |
---|
| 275 | echo "REFERENCE directory : $NEMO_VALID_REF at rev $NEMO_REV_REF" |
---|
| 276 | echo '' |
---|
| 277 | fi |
---|
[15159] | 278 | checklist=(GYRE_PISCES ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 ORCA2_SAS_ICE ORCA2_ICE_OBS AGRIF_DEMO WED025 ISOMIP+ VORTEX ICE_AGRIF OVERFLOW LOCK_EXCHANGE SWG) |
---|
[14897] | 279 | for repro_test in ${checklist[@]} |
---|
| 280 | do |
---|
| 281 | runcmpres $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $quiet |
---|
| 282 | done |
---|
| 283 | if [ ${quiet} -eq 0 ] ; then |
---|
| 284 | echo '' |
---|
| 285 | else |
---|
| 286 | if [ $(( $nrmiss + $nvmiss )) -gt 0 ] ; then |
---|
| 287 | echo $errcnt " differences from "$nmatch" matches. "$nrmiss" missing from REFERENCE "$nvmiss" missing from VALID" |
---|
| 288 | else |
---|
| 289 | echo $errcnt " differences from "$nmatch" matches. " |
---|
| 290 | fi |
---|
| 291 | fi |
---|
| 292 | else |
---|
| 293 | echo '' |
---|
| 294 | echo ' No path for comparison specified. Result are not compare with any other revision. ' |
---|
| 295 | echo ' To do it please fill NEMO_VALID_REF and NEMO_REV_REF in param.cfg. ' |
---|
| 296 | echo '' |
---|
| 297 | fi |
---|
| 298 | fi |
---|
| 299 | # |
---|
| 300 | exit |
---|