New URL for NEMO forge!   http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.
sette_rpt.sh in utils/CI/sette – NEMO

source: utils/CI/sette/sette_rpt.sh @ 14825

Last change on this file since 14825 was 14825, checked in by acc, 3 years ago

Update sette for ARCHER2 and remove support for the de-commissioned ARCHER platform. Also fixed reporting for tracer.stat files in sette_rpt.sh and added reporting of average cpu timings from timing.output (if available). Uses red/green highlighting for increases/decreases from the reference.

  • Property svn:executable set to *
File size: 17.1 KB
RevLine 
[6124]1#!/bin/bash -f
[9019]2# set -vx
[6124]3# simple SETTE report generator.
4#
5# This version should be run in the SETTE directory.
6# The machine name will be picked up from the sette.sh script but the location of the
[9602]7# validation directory needs to be set here (currently assumed to reside in the ../cfgs directory)
[6124]8#
9#########################################################################################
10######################### Start of function definitions #################################
11##
[8468]12
[11161]13function get_dorv() {
14  if [ $lastchange == 'old' ] ; then
[8468]15    dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l `
16    dorv=`echo $dorv | sed -e 's:.*/::'`
[12569]17    dorv2=`ls -1rt $vdir/$nam2/$mach/ 2>/dev/null | tail -1l `
[11497]18    dorv2=`echo $dorv2 | sed -e 's:.*/::'`
[9019]19  else
[11161]20    dorv=$lastchange
[11497]21    dorv2=$lastchange
[9019]22  fi
[8468]23}
24
[11161]25function get_ktdiff() {
26  ktdiff=`diff ${1} ${2} | head -2 | grep it | awk '{ print $4 }'`
27}
28
[14825]29function get_ktdiff2() {
30  ktdiff=`diff ${1} ${2} |  head -2 | tail -1l | awk '{print $2}'`
31}
32
[6124]33function resttest() { 
34#
35# Restartability checks. Expects LONG and SHORT run directories
36# Compares end of LONG stat files with equivalent entries from the SHORT stat files.
37#
38  vdir=$1
39  nam=$2
40  pass=$3
41#
[11161]42# get $dorv
43  get_dorv
44#
45# check if directory is here
46  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
47    printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv
48    return
49  fi
50
51  if [ -d $vdir/$nam/$mach/$dorv ]; then
52    # check ocean output
53    runtest $vdir $nam $pass RST
54    #
55    # run restartibility test
[6124]56    f1o=$vdir/$nam/$mach/$dorv/LONG/ocean.output
[9019]57    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
[6124]58    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
59    f2o=$vdir/$nam/$mach/$dorv/SHORT/ocean.output
[9019]60    f2s=$vdir/$nam/$mach/$dorv/SHORT/run.stat
[6124]61    f2t=$vdir/$nam/$mach/$dorv/SHORT/tracer.stat
62
[6152]63    if  [ ! -f $f1s ] &&  [ ! -f $f1t ] ; then
[9525]64      printf "%-27s %s\n" $nam " incomplete test";
[6124]65      return;
66    fi
[6152]67    if  [ ! -f $f2s ] &&  [ ! -f $f2t ] ; then
[9525]68      printf "%-27s %s\n" $nam " incomplete test";
[6124]69      return;
70    fi
71#
72    done_oce=0
73
[6152]74    if  [  -f $f1s ] && [  -f $f2s ]; then
75      nl=(`wc -l $f2s`)
76      tail -${nl[0]} $f1s > f1.tmp$$
77      cmp -s f1.tmp$$ $f2s
78      if [ $? == 0 ]; then
79        if [ $pass == 0 ]; then
[9525]80          printf "%-27s %s %s\n" $nam  " run.stat    restartability  passed : " $dorv
[6152]81        fi
82      else
[11161]83        get_ktdiff f1.tmp$$ $f2s
84        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " run.stat    restartability  FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
[6124]85#
86# Offer view of differences on the second pass
87#
[6152]88        if [ $pass == 1 ]; then
[9019]89          echo "<return> to view run.stat differences"
[6152]90          read y
91          sdiff f1.tmp$$ $f2s
92          echo "<return> to view ocean.output differences"
93          read y
94          sdiff $f1o $f2o | grep "|"
95          done_oce=1
96          echo "<return> to continue"
97          read y
98        fi
[6124]99      fi
100    fi
101#
102# Check tracer.stat files (if they exist)
103#
[6152]104    if  [  -f $f1t ] && [  -f $f2t ]; then
[6124]105      nl=(`wc -l $f2t`)
106      tail -${nl[0]} $f1t > f1.tmp$$
107      cmp -s f1.tmp$$ $f2t
108      if [ $? == 0 ]; then
[6152]109        if [ $pass == 0 ]; then
[9525]110          printf "%-27s %s %s\n" $nam  " tracer.stat restartability  passed : " $dorv
[6152]111        fi
[6124]112      else
[14825]113        get_ktdiff2 f1.tmp$$ $f2t
[11161]114        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " tracer.stat    restartability  FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
[6124]115#
116# Offer view of differences on the second pass
117#
118        if [ $pass == 1 ]; then
119          echo "<return> to view tracer.stat differences"
120          read y
121          sdiff f1.tmp$$ $f2t
122#
123# Only offer ocean.output view if it has not been viewed previously
124#
125          if [ $done_oce == 0 ]; then
126            echo "<return> to view ocean.output differences"
127            read y
128            sdiff $f1o $f2o | grep "|"
129          fi
130          echo "<return> to continue"
131          read y
132        fi
133      fi
134    fi
135    rm f1.tmp$$
136  fi
137}
138
139function reprotest(){
140#
141# Reproducibility checks. Expects REPRO_N_M and REPRO_I_J run directories
142# Compares end of stat files from each
143#
144  vdir=$1
145  nam=$2
146  pass=$3
147#
[11161]148# get $dorv
149  get_dorv
150#
151# check if directory is here
152  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
153    printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv
154    return
155  fi
156#
157  if [ -d $vdir/$nam/$mach/$dorv ]; then
158    # check ocean output
159    runtest $vdir $nam $pass REPRO
160    #
161    # check reproducibility
[9019]162    rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -2l | head -1 `
163    rep2=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -1l`
[13785]164    if [ $rep1 == $rep2 ]; then
165       rep2=''
166    fi
[6124]167    f1o=$vdir/$nam/$mach/$dorv/$rep1/ocean.output
[9019]168    f1s=$vdir/$nam/$mach/$dorv/$rep1/run.stat
[6124]169    f1t=$vdir/$nam/$mach/$dorv/$rep1/tracer.stat
170    f2o=$vdir/$nam/$mach/$dorv/$rep2/ocean.output
[9019]171    f2s=$vdir/$nam/$mach/$dorv/$rep2/run.stat
[6124]172    f2t=$vdir/$nam/$mach/$dorv/$rep2/tracer.stat
173
[6152]174    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
[9525]175      printf "%-27s %s\n" $nam " incomplete test";
[6124]176      return;
177    fi
[6152]178    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
[9525]179      printf "%-27s %s\n" $nam " incomplete test";
[6124]180      return;
181    fi
182#
183    done_oce=0
184
[6152]185    if  [ -f $f1s ] && [ -f $f2s ] ; then
186      cmp -s $f1s $f2s
187      if [ $? == 0 ]; then
188        if [ $pass == 0 ]; then
[9525]189          printf "%-27s %s %s\n" $nam  " run.stat    reproducibility passed : " $dorv
[6152]190        fi
191      else
[11161]192        get_ktdiff $f1s $f2s
193        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " run.stat    reproducibility FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
[6124]194#
195# Offer view of differences on the second pass
196#
[6152]197        if [ $pass == 1 ]; then
[9019]198          echo "<return> to view run.stat differences"
[6152]199          read y
[9911]200          sdiff $f1s $f2s
[6152]201          echo "<return> to view ocean.output differences"
202          read y
203          sdiff $f1o $f2o | grep "|"
204          done_oce=1
205          echo "<return> to continue"
206          read y
207        fi
[6124]208      fi
209    fi
210#
211# Check tracer.stat files (if they exist)
212#
[6152]213    if  [ -f $f1t ] && [ -f $f2t ] ; then
[6124]214      cmp -s $f1t $f2t
215      if [ $? == 0 ]; then
[9525]216        if [ $pass == 0 ]; then           printf "%-27s %s %s\n" $nam  " tracer.stat reproducibility passed : " $dorv
[6152]217        fi
[6124]218      else
[14825]219        get_ktdiff2 $f1t $f2t
[12569]220        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " tracer.stat reproducibility FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
[6124]221#
222# Offer view of differences on the second pass
223#
224        if [ $pass == 1 ]; then
225          echo "<return> to view tracer.stat differences"
226          read y
227          sdiff $f1t $f2t
228#
229# Only offer ocean.output view if it has not been viewed previously
230#
231          if [ $done_oce == 0 ]; then
232            echo "<return> to view ocean.output differences"
233            read y
234            sdiff $f1o $f2o | grep "|"
235          fi
236          echo "<return> to continue"
237          read y
238        fi
239      fi
240    fi
[9223]241  fi
242}
[11161]243function runcmpres(){
[9223]244#
[11161]245# compare *.stat file with reference file from a previous sette test or previous version
246# store in NEMO_VALID_REF at revision NEMO_REV_REF
247# Compares end of stat files from each
[9223]248#
249  vdir=$1
250  nam=$2
[11161]251  vdirref=$3
252  dorvref=$4
253  pass=$5
[9223]254#
[11161]255# get $dorv
256  get_dorv
257#
258# check if reference directory is present
259  if [ ! -d $vdirref/$nam/$mach/$dorvref ]; then
260    printf "%-27s %s\n" $nam " REFERENCE directory at $dorvref is MISSING"
261    return
262  fi
263  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
264    printf "%-27s %s\n" $nam " VALID     directory at $dorv is MISSING"
265    return
266  fi
267
268#
269  if [ -d $vdir/$nam/$mach/$dorv ]; then
270    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
271    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
[14825]272    f1a=$vdir/$nam/$mach/$dorv/LONG/timing.output
[11161]273    f2s=$vdirref/$nam/$mach/$dorvref/LONG/run.stat
274    f2t=$vdirref/$nam/$mach/$dorvref/LONG/tracer.stat
[14825]275    f2a=$vdirref/$nam/$mach/$dorvref/LONG/timing.output
[11161]276    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
277      printf "%-20s %s\n" $nam " incomplete test";
278      return;
[9911]279    fi
[11161]280    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
281      printf "%-20s %s\n" $nam " incomplete test";
[9223]282      return;
[11161]283    fi
284#
285    done_oce=0
286
287    if  [ -f $f1s ] && [ -f $f2s ] ; then
288      cmp -s $f1s $f2s
289      if [ $? == 0 ]; then
290        if [ $pass == 0 ]; then
291          printf "%-20s %s %s\n" $nam  " run.stat    files are identical "
292        fi
293      else
[12569]294        get_ktdiff $f1s $f2s
295        printf "%-20s %s %s %-5s %s\n" $nam  " run.stat    files are DIFFERENT (results are different after " $ktdiff " time steps)"
[11161]296#
297# Offer view of differences on the second pass
298#
[9223]299        if [ $pass == 1 ]; then
[11161]300          echo "<return> to view run.stat differences"
[9223]301          read y
[11161]302          sdiff $f1s $f2s
303          done_oce=1
304          echo "<return> to continue"
305          read y
[9223]306        fi
307      fi
308    fi
[11161]309    # Check tracer.stat files (if they exist)
310#
311    if  [ -f $f1t ] && [ -f $f2t ] ; then
312      cmp -s $f1t $f2t
313      if [ $? == 0 ]; then
314        if [ $pass == 0 ]; then         
315          printf "%-20s %s %s\n" $nam  " tracer.stat files are identical "
316        fi
317      else
[14825]318        get_ktdiff2 $f1t $f2t
[12569]319        printf "%-20s %s %s %-5s %s\n" $nam  " tracer.stat files are DIFFERENT (results are different after " $ktdiff " time steps) "
[11161]320#
321# Offer view of differences on the second pass
322#
323        if [ $pass == 1 ]; then
324          echo "<return> to view tracer.stat differences"
325          read y
326          sdiff $f1t $f2t
327        fi
328      fi
329    fi
[14825]330#
331# Report average CPU time differences (if available)
332#
333    if  [ -f $f1a ] && [ -f $f2a ] ; then
334      tnew=$(grep 'Average ' $f1a  | awk '{print $5}')
335      tref=$(grep 'Average ' $f2a  | awk '{print $5}')
336      if [ $? == 0 ]; then
337        if [ $pass == 0 ]; then
338          tdif=$( echo ${tnew} ${tref} | awk '{print $1 - $2}')
339          if (( $(echo "$tnew > $tref" |bc -l) )); then
340            printf "%-20s %14s %10s %14s %10s %14s \\e[41;33;196m%10s\\e[0m\n" $nam  " ref. time:" $tref "cur. time:" $tnew "diff.:" $tdif
341          else
342            printf "%-20s %14s %10s %14s %10s %14s \\e[42;01;196m%10s\\e[0m\n" $nam  " ref. time:" $tref "cur. time:" $tnew "diff.:" $tdif
343          fi
344        fi
345      fi
346    fi
[11161]347  fi
348}
349
350function runtest(){
351#
352# Run checks.
353# Check presence of E R R O R in ocean.output from each
354#
355  vdir=$1
356  nam=$2
357  pass=$3
358  ttype=$4
359  [[ $ttype == 'RST' ]] && ttype="LONG|SHORT"
360#
361# get $dorv
362  get_dorv
363#
364# no print needed if the repository is not here (already catch before)
365#
366  if [ -d $vdir/$nam/$mach/$dorv/ ]; then
367    #
368    # apply check for all ttype directory
369    rep1=$(ls -rt $vdir/$nam/$mach/$dorv/ | grep -E $ttype)
370    for tdir in $rep1 ; do
371       f1o=$vdir/$nam/$mach/$dorv/$tdir/ocean.output
372       if  [ ! -f $f1o ] ; then
373          if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam " ocean.output               MISSING : " $dorv ; fi
374          return;
375       else
376          nerr=`grep 'E R R O R' $f1o | wc -l`
377          if [[ $nerr > 0 ]]; then
378             printf "\e[38;5;196m%-27s %s %s %s\e[0m\n" $nam " run                         FAILED : " $dorv " ( E R R O R in ocean.output) " 
379             if [ $pass == 1 ]; then
380                echo "<return> to view end of ocean.output"
381                read y
382                tail -100 $f1o
383                echo ''
384                echo "full ocean.output available here: $f1o"
385             fi
386             return;
387          fi
388       fi
389    done
[9221]390  else
[11161]391    if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv ; fi
[6124]392  fi
393}
[9019]394
395function identictest(){
396#
[11497]397# Checks AGRIF does not corrupt results with no AGRIF zoom by comparing run.stat files
[9019]398#
399  vdir=$1
[11497]400  nam=$2
401  nam2=$3
[9019]402  pass=$4
403#
[11497]404  get_dorv
[9019]405#
[11497]406  rep=`ls -1rt $vdir/$nam/$mach/$dorv/ |  tail -1l`
407  f1s=${vdir}/${nam}/${mach}/${dorv}/${rep}/run.stat
408  f2s=${vdir}/${nam2}/${mach}/${dorv2}/${rep}/run.stat
409#
410  if  [ -f $f1s ] && [ -f $f2s ] ; then
[9019]411      cmp -s $f1s $f2s
412      if [ $? == 0 ]; then
[11497]413          if [ $pass == 0 ]; then
414         printf "%-5s %s %-5s %s %s %s\n" $rep "AGRIF vs" $rep "NOAGRIF run.stat    unchanged  -    passed : " $dorv $dorv2
415          fi
[9019]416      else
[11497]417          get_ktdiff $f1s $f2s
418          printf "\e[38;5;196m%-5s %s %-5s %s %s %s %s %-5s %s\e[0m\n" $rep "AGRIF vs" $rep "NOAGRIF run.stat    changed  -     FAILED : " $dorv $dorv2 " (results are different after " $ktdiff " time steps)"
[9019]419#
420# Offer view of differences on the second pass
421#
[11497]422          if [ $pass == 1 ]; then
423         echo "<return> to view run.stat differences"
424         read y
425         sdiff $f1s $f2s
426         echo "<return> to continue"
427         read y
428          fi
[9019]429      fi
430  else
[11497]431      printf "%-27s %-27s %s\n" $nam $nam2 " incomplete test"
[9019]432  fi
433}
[6124]434########################### END of function definitions #################################
435##                                                                                     ##
436##    Main script                                                                      ##
437##                                                                                     ##
438#########################################################################################
439#
[11161]440# LOAD param variable (COMPILER, NEMO_VALIDATION_DIR, SVN_CMD)
441  SETTE_DIR=$(cd $(dirname "$0"); pwd)
442  MAIN_DIR=$(dirname $SETTE_DIR)
443  . ./param.cfg
444
445  mach=${COMPILER}
446  NEMO_VALID=${NEMO_VALIDATION_DIR}
447  NEMO_VALID_REF=${NEMO_VALIDATION_REF}
[6124]448#
449  if [ ! -d $NEMO_VALID ]; then
450    echo "$NEMO_VALID validation directory not found"
451    exit
452  fi
453#
[9019]454#
455# Show current revision tag and branch name
456#
[11161]457echo ""
[12569]458lastchange=`${SVN_CMD} info ${MAIN_DIR} | grep 'Last Changed Rev' | awk '{print $NF}'`
459revision=`${SVN_CMD} info ${MAIN_DIR} | grep 'Revision' | awk '{print $NF}'`
460branchname=`${SVN_CMD} info ${MAIN_DIR} | grep ^URL | awk -F ipsl/forge/projets/nemo/svn/ '{print $NF}'`
[11161]461echo "Current code is : $branchname @ r$revision  ( last change @ r$lastchange )"
[13795]462[ `${SVN_CMD} status -q ${MAIN_DIR}/{cfgs,tests,src} | wc -l` -ge 1 ] && lastchange=${lastchange}+
[11161]463
[12569]464# overwrite revision or compiler
465  if [ $# -gt 0 ]; then
466    while getopts r:c:h option; do
467       case $option in
468          c) mach=$OPTARG;;
469          r) rev=$OPTARG;;
470          h | *) echo ''
471                 echo 'sette_rpt.sh : ' 
472                 echo '     display result for the latest change'
473                 echo ' -c COMPILER_name :'
474                 echo '     display result for the specified compiler'
475                 echo ' -r REVISION_number :'
476                 echo '     display sette results for the specified revision (set old for the latest revision available for each config)'
477                 echo ''
478                 exit 42;;
479       esac
480    done
481    shift $((OPTIND - 1))
482fi
[11161]483
[12569]484# if $1 (remaining arguments)
485if [[ ! -z $1 ]] ; then rev=$1 ; fi
486
487# by default use the current lastchanged revision
488lastchange=${rev:-$lastchange}
489
[11161]490echo ""
491echo "SETTE validation report generated for : "
492echo ""
493echo "       $branchname @ r$lastchange (last changed revision)"
494echo ""
495echo "       on $COMPILER arch file"
496echo ""
497
[9019]498#
[6124]499# The script also needs the date or revision tag. Currently this is taken from the latest sub-directory found in each directory
500
[12569]501for pass in  $RPT_PASSES 
[6124]502do
503#
[9525]504 if [ $pass == 0 ]; then
505   echo "" 
506   echo "!!---------------1st pass------------------!!"
507 fi
508 if [ $pass == 1 ]; then
509    echo ""
510    echo "!!---------------2nd pass------------------!!"
511 fi
[6124]512#
[8468]513
[6124]514# Restartability test
[9525]515 echo ""
[9518]516 echo "   !----restart----!   "
[14244]517 for restart_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WWED025_ST WISOMIP+_ST WOVERFLOW_ST WLOCK_EXCHANGE_ST WVORTEX_ST WICE_AGRIF_ST WSWG_ST
[6124]518 do
519   resttest $NEMO_VALID $restart_test $pass
520 done
521#
522# Reproducibility tests
[9525]523 echo ""
[9518]524 echo "   !----repro----!   "
[14244]525 for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WORCA2_ICE_OBS_ST WAGRIF_DEMO_ST WWED025_ST WISOMIP+_ST WVORTEX_ST WICE_AGRIF_ST WSWG_ST
[6124]526 do
527   reprotest $NEMO_VALID $repro_test $pass
528 done
[9019]529
[9518]530# AGRIF special check to ensure results are unchanged with and without key_agrif
[9525]531 echo ""
[9518]532 echo "   !----agrif check----!   "
[9776]533 dir1=WAGRIF_DEMO_NOAGRIF_ST
534 dir2=WAGRIF_DEMO_ST
[9019]535 identictest $NEMO_VALID $dir1 $dir2 $pass 
[11161]536#
537# before/after tests
538 if [ $lastchange == 'old' ] ; then
539    echo ""
540    echo "   !---- 'old' specified as revision => no comparison with reference results ----!   "
541    echo ""
542 else
543   echo ""
544   echo "   !----result comparison check----!   "
545   if [ $NEMO_VALID_REF != "/path/to/reference/sette/results" ]; then
546     echo ''
547     echo 'check result differences between :'
548     echo "VALID directory : $NEMO_VALID at rev $lastchange"
549     echo 'and'
550     echo "REFERENCE directory : $NEMO_VALID_REF at rev $NEMO_REV_REF"
551     echo ''
[14244]552     for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WWED025_ST WISOMIP+_ST WVORTEX_ST WICE_AGRIF_ST WOVERFLOW_ST WLOCK_EXCHANGE_ST WSWG_ST
[11161]553     do
554       runcmpres $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $pass
555     done
556   else
557     echo ''
558     echo ' No path for comparison specified. Result are not compare with any other revision. '
559     echo ' To do it please fill NEMO_VALID_REF and NEMO_REV_REF in param.cfg. '
560     echo ''
561   fi
562 fi
[9019]563done
[6124]564#
565exit
Note: See TracBrowser for help on using the repository browser.