source: utils/CI/sette/sette_rpt.sh @ 11161

Last change on this file since 11161 was 11161, checked in by mathiot, 15 months ago

merge fix_sette_ticket2239 : major simplification/improvement of SETTE (see README.rst for all the details), tested with the latest version of the trunk (r11160) (ticket #2239)

  • Property svn:executable set to *
File size: 15.9 KB
Line 
1#!/bin/bash -f
2# set -vx
3# simple SETTE report generator.
4#
5# This version should be run in the SETTE directory.
6# The machine name will be picked up from the sette.sh script but the location of the
7# validation directory needs to be set here (currently assumed to reside in the ../cfgs directory)
8#
9#########################################################################################
10######################### Start of function definitions #################################
11##
12
13function get_dorv() {
14  if [ $lastchange == 'old' ] ; then
15    dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l `
16    dorv=`echo $dorv | sed -e 's:.*/::'`
17  else
18    dorv=$lastchange
19  fi
20}
21
22function get_ktdiff() {
23  ktdiff=`diff ${1} ${2} | head -2 | grep it | awk '{ print $4 }'`
24}
25
26function resttest() { 
27#
28# Restartability checks. Expects LONG and SHORT run directories
29# Compares end of LONG stat files with equivalent entries from the SHORT stat files.
30#
31  vdir=$1
32  nam=$2
33  pass=$3
34#
35# get $dorv
36  get_dorv
37#
38# check if directory is here
39  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
40    printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv
41    return
42  fi
43
44  if [ -d $vdir/$nam/$mach/$dorv ]; then
45    # check ocean output
46    runtest $vdir $nam $pass RST
47    #
48    # run restartibility test
49    f1o=$vdir/$nam/$mach/$dorv/LONG/ocean.output
50    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
51    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
52    f2o=$vdir/$nam/$mach/$dorv/SHORT/ocean.output
53    f2s=$vdir/$nam/$mach/$dorv/SHORT/run.stat
54    f2t=$vdir/$nam/$mach/$dorv/SHORT/tracer.stat
55
56    if  [ ! -f $f1s ] &&  [ ! -f $f1t ] ; then
57      printf "%-27s %s\n" $nam " incomplete test";
58      return;
59    fi
60    if  [ ! -f $f2s ] &&  [ ! -f $f2t ] ; then
61      printf "%-27s %s\n" $nam " incomplete test";
62      return;
63    fi
64#
65    done_oce=0
66
67    if  [  -f $f1s ] && [  -f $f2s ]; then
68      nl=(`wc -l $f2s`)
69      tail -${nl[0]} $f1s > f1.tmp$$
70      cmp -s f1.tmp$$ $f2s
71      if [ $? == 0 ]; then
72        if [ $pass == 0 ]; then
73          printf "%-27s %s %s\n" $nam  " run.stat    restartability  passed : " $dorv
74        fi
75      else
76        get_ktdiff f1.tmp$$ $f2s
77        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " run.stat    restartability  FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
78#
79# Offer view of differences on the second pass
80#
81        if [ $pass == 1 ]; then
82          echo "<return> to view run.stat differences"
83          read y
84          sdiff f1.tmp$$ $f2s
85          echo "<return> to view ocean.output differences"
86          read y
87          sdiff $f1o $f2o | grep "|"
88          done_oce=1
89          echo "<return> to continue"
90          read y
91        fi
92      fi
93    fi
94#
95# Check tracer.stat files (if they exist)
96#
97    if  [  -f $f1t ] && [  -f $f2t ]; then
98      nl=(`wc -l $f2t`)
99      tail -${nl[0]} $f1t > f1.tmp$$
100      cmp -s f1.tmp$$ $f2t
101      if [ $? == 0 ]; then
102        if [ $pass == 0 ]; then
103          printf "%-27s %s %s\n" $nam  " tracer.stat restartability  passed : " $dorv
104        fi
105      else
106        get_ktdiff f1.tmp$$ $f2t
107        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " tracer.stat    restartability  FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
108#
109# Offer view of differences on the second pass
110#
111        if [ $pass == 1 ]; then
112          echo "<return> to view tracer.stat differences"
113          read y
114          sdiff f1.tmp$$ $f2t
115#
116# Only offer ocean.output view if it has not been viewed previously
117#
118          if [ $done_oce == 0 ]; then
119            echo "<return> to view ocean.output differences"
120            read y
121            sdiff $f1o $f2o | grep "|"
122          fi
123          echo "<return> to continue"
124          read y
125        fi
126      fi
127    fi
128    rm f1.tmp$$
129  fi
130}
131
132function reprotest(){
133#
134# Reproducibility checks. Expects REPRO_N_M and REPRO_I_J run directories
135# Compares end of stat files from each
136#
137  vdir=$1
138  nam=$2
139  pass=$3
140#
141# get $dorv
142  get_dorv
143#
144# check if directory is here
145  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
146    printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv
147    return
148  fi
149#
150  if [ -d $vdir/$nam/$mach/$dorv ]; then
151    # check ocean output
152    runtest $vdir $nam $pass REPRO
153    #
154    # check reproducibility
155    rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -2l | head -1 `
156    rep2=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -1l`
157    f1o=$vdir/$nam/$mach/$dorv/$rep1/ocean.output
158    f1s=$vdir/$nam/$mach/$dorv/$rep1/run.stat
159    f1t=$vdir/$nam/$mach/$dorv/$rep1/tracer.stat
160    f2o=$vdir/$nam/$mach/$dorv/$rep2/ocean.output
161    f2s=$vdir/$nam/$mach/$dorv/$rep2/run.stat
162    f2t=$vdir/$nam/$mach/$dorv/$rep2/tracer.stat
163
164    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
165      printf "%-27s %s\n" $nam " incomplete test";
166      return;
167    fi
168    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
169      printf "%-27s %s\n" $nam " incomplete test";
170      return;
171    fi
172#
173    done_oce=0
174
175    if  [ -f $f1s ] && [ -f $f2s ] ; then
176      cmp -s $f1s $f2s
177      if [ $? == 0 ]; then
178        if [ $pass == 0 ]; then
179          printf "%-27s %s %s\n" $nam  " run.stat    reproducibility passed : " $dorv
180        fi
181      else
182        get_ktdiff $f1s $f2s
183        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " run.stat    reproducibility FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
184#
185# Offer view of differences on the second pass
186#
187        if [ $pass == 1 ]; then
188          echo "<return> to view run.stat differences"
189          read y
190          sdiff $f1s $f2s
191          echo "<return> to view ocean.output differences"
192          read y
193          sdiff $f1o $f2o | grep "|"
194          done_oce=1
195          echo "<return> to continue"
196          read y
197        fi
198      fi
199    fi
200#
201# Check tracer.stat files (if they exist)
202#
203    if  [ -f $f1t ] && [ -f $f2t ] ; then
204      cmp -s $f1t $f2t
205      if [ $? == 0 ]; then
206        if [ $pass == 0 ]; then           printf "%-27s %s %s\n" $nam  " tracer.stat reproducibility passed : " $dorv
207        fi
208      else
209        get_ktdiff $f1t $f2t
210        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " tracer.stat    reproducibility FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
211#
212# Offer view of differences on the second pass
213#
214        if [ $pass == 1 ]; then
215          echo "<return> to view tracer.stat differences"
216          read y
217          sdiff $f1t $f2t
218#
219# Only offer ocean.output view if it has not been viewed previously
220#
221          if [ $done_oce == 0 ]; then
222            echo "<return> to view ocean.output differences"
223            read y
224            sdiff $f1o $f2o | grep "|"
225          fi
226          echo "<return> to continue"
227          read y
228        fi
229      fi
230    fi
231  fi
232}
233function runcmpres(){
234#
235# compare *.stat file with reference file from a previous sette test or previous version
236# store in NEMO_VALID_REF at revision NEMO_REV_REF
237# Compares end of stat files from each
238#
239  vdir=$1
240  nam=$2
241  vdirref=$3
242  dorvref=$4
243  pass=$5
244#
245# get $dorv
246  get_dorv
247#
248# check if reference directory is present
249  if [ ! -d $vdirref/$nam/$mach/$dorvref ]; then
250    printf "%-27s %s\n" $nam " REFERENCE directory at $dorvref is MISSING"
251    return
252  fi
253  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
254    printf "%-27s %s\n" $nam " VALID     directory at $dorv is MISSING"
255    return
256  fi
257
258#
259  if [ -d $vdir/$nam/$mach/$dorv ]; then
260    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
261    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
262    f2s=$vdirref/$nam/$mach/$dorvref/LONG/run.stat
263    f2t=$vdirref/$nam/$mach/$dorvref/LONG/tracer.stat
264    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
265      printf "%-20s %s\n" $nam " incomplete test";
266      return;
267    fi
268    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
269      printf "%-20s %s\n" $nam " incomplete test";
270      return;
271    fi
272#
273    done_oce=0
274
275    if  [ -f $f1s ] && [ -f $f2s ] ; then
276      cmp -s $f1s $f2s
277      if [ $? == 0 ]; then
278        if [ $pass == 0 ]; then
279          printf "%-20s %s %s\n" $nam  " run.stat    files are identical "
280        fi
281      else
282        printf "%-20s %s %s\n" $nam  " run.stat    files are DIFFERENT "
283#
284# Offer view of differences on the second pass
285#
286        if [ $pass == 1 ]; then
287          echo "<return> to view run.stat differences"
288          read y
289          sdiff $f1s $f2s
290          done_oce=1
291          echo "<return> to continue"
292          read y
293        fi
294      fi
295    fi
296    # Check tracer.stat files (if they exist)
297#
298    if  [ -f $f1t ] && [ -f $f2t ] ; then
299      cmp -s $f1t $f2t
300      if [ $? == 0 ]; then
301        if [ $pass == 0 ]; then         
302          printf "%-20s %s %s\n" $nam  " tracer.stat files are identical "
303        fi
304      else
305        printf "%-20s %s %s\n" $nam  " tracer.stat files are DIFFERENT "
306#
307# Offer view of differences on the second pass
308#
309        if [ $pass == 1 ]; then
310          echo "<return> to view tracer.stat differences"
311          read y
312          sdiff $f1t $f2t
313        fi
314      fi
315    fi
316  fi
317}
318
319function runtest(){
320#
321# Run checks.
322# Check presence of E R R O R in ocean.output from each
323#
324  vdir=$1
325  nam=$2
326  pass=$3
327  ttype=$4
328  [[ $ttype == 'RST' ]] && ttype="LONG|SHORT"
329#
330# get $dorv
331  get_dorv
332#
333# no print needed if the repository is not here (already catch before)
334#
335  if [ -d $vdir/$nam/$mach/$dorv/ ]; then
336    #
337    # apply check for all ttype directory
338    rep1=$(ls -rt $vdir/$nam/$mach/$dorv/ | grep -E $ttype)
339    for tdir in $rep1 ; do
340       f1o=$vdir/$nam/$mach/$dorv/$tdir/ocean.output
341       if  [ ! -f $f1o ] ; then
342          if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam " ocean.output               MISSING : " $dorv ; fi
343          return;
344       else
345          nerr=`grep 'E R R O R' $f1o | wc -l`
346          if [[ $nerr > 0 ]]; then
347             printf "\e[38;5;196m%-27s %s %s %s\e[0m\n" $nam " run                         FAILED : " $dorv " ( E R R O R in ocean.output) " 
348             if [ $pass == 1 ]; then
349                echo "<return> to view end of ocean.output"
350                read y
351                tail -100 $f1o
352                echo ''
353                echo "full ocean.output available here: $f1o"
354             fi
355             return;
356          fi
357       fi
358    done
359  else
360    if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv ; fi
361  fi
362}
363
364function identictest(){
365#
366#  checks AGRIF does not corrupe results with no AGRIF zoom. Expects ORCA2AGUL/AGRIFNOZ and ORCA2AGUL_NAGR/AGRIFNO  run directories
367# Compares solver.stat files for each
368#
369  vdir=$1
370  dir1=$2
371  dir2=$3
372  pass=$4
373#
374  if [ -d $vdir/$dir1 ] && [ -d $vdir/$dir2 ]; then
375    dorv1=`ls -1rt $vdir/$dir1/$mach/ | tail -1l `
376    dorv1=`echo $dorv1 | sed -e 's:.*/::'`
377    dorv2=`ls -1rt $vdir/$dir2/$mach/ | tail -1l `
378    dorv2=`echo $dorv2 | sed -e 's:.*/::'`
379
380    rep1=`ls -1rt $vdir/$dir1/$mach/$dorv1/ |  tail -1l`
381#clem    rep2=`ls -1rt $vdir/$dir2/$mach/$dorv2/ |  tail -1l`
382    rep2=`ls -1rt $vdir/$dir1/$mach/$dorv1/ |  tail -1l`
383    f1s=$vdir/$dir1/$mach/$dorv1/$rep1/run.stat
384    f2s=$vdir/$dir2/$mach/$dorv2/$rep2/run.stat
385
386    if  [ ! -f $f1s ] && [ ! -f $f2s ] ; then
387      printf "%-27s %s\n" $dir1 $dir2 " incomplete test";
388      return;
389    fi
390#
391    done_oce=0
392
393    if  [ -f $f1s ] && [ -f $f2s ] ; then
394      cmp -s $f1s $f2s
395      if [ $? == 0 ]; then
396        if [ $pass == 0 ]; then
397          printf "%-5s %s %-5s %s %s %s\n" $rep1 "AGRIF vs" $rep2 "NOAGRIF run.stat    unchanged  -    passed : " $dorv1 $dorv2
398        fi
399      else
400        get_ktdiff $f1s $f2s
401        printf "\e[38;5;196m%-5s %s %-5s %s %s %s %s %-5s %s\e[0m\n" $rep1 "AGRIF vs" $rep2 "NOAGRIF run.stat    changed  -     FAILED : " $dorv1 $dorv2 " (results are different after " $ktdiff " time steps)"
402#
403# Offer view of differences on the second pass
404#
405        if [ $pass == 1 ]; then
406          echo "<return> to view run.stat differences"
407          read y
408          sdiff $f1s $f2s
409          done_oce=1
410          echo "<return> to continue"
411          read y
412        fi
413      fi
414    fi
415  else
416    printf "%-27s %s\n" $dir1 $dir2 " incomplete test";
417  fi
418}
419########################### END of function definitions #################################
420##                                                                                     ##
421##    Main script                                                                      ##
422##                                                                                     ##
423#########################################################################################
424#
425# LOAD param variable (COMPILER, NEMO_VALIDATION_DIR, SVN_CMD)
426  SETTE_DIR=$(cd $(dirname "$0"); pwd)
427  MAIN_DIR=$(dirname $SETTE_DIR)
428  . ./param.cfg
429
430  mach=${COMPILER}
431  NEMO_VALID=${NEMO_VALIDATION_DIR}
432  NEMO_VALID_REF=${NEMO_VALIDATION_REF}
433  LANG=en_US
434#
435  if [ ! -d $NEMO_VALID ]; then
436    echo "$NEMO_VALID validation directory not found"
437    exit
438  fi
439#
440# Directory to run the tests
441  CONFIG_DIR0=${MAIN_DIR}/cfgs
442  TOOLS_DIR=${MAIN_DIR}/tools
443  NPROC=32
444  SAS_RESTART_DIR=${CONFIG_DIR0}/ORCA2_SAS_ICE_ST
445#
446# Show current revision tag and branch name
447#
448echo ""
449lastchange=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep 'Last Changed Rev' | awk '{print $NF}'`
450revision=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep 'Revision' | awk '{print $NF}'`
451branchname=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep ^URL | awk -F ipsl/forge/projets/nemo/svn/ '{print $NF}'`
452echo "Current code is : $branchname @ r$revision  ( last change @ r$lastchange )"
453[ `${SVN_CMD} status -q ${MAIN_DIR}/{cfg,tests,src} | wc -l` -ge 1 ] && lastchange=${lastchange}+
454
455# overwrite revision is argument 1 present
456[[ $# -eq 1 ]] && lastchange=$1
457
458echo ""
459echo "SETTE validation report generated for : "
460echo ""
461echo "       $branchname @ r$lastchange (last changed revision)"
462echo ""
463echo "       on $COMPILER arch file"
464echo ""
465
466#
467# The script also needs the date or revision tag. Currently this is taken from the latest sub-directory found in each directory
468
469for pass in  0 1
470do
471#
472 if [ $pass == 0 ]; then
473   echo "" 
474   echo "!!---------------1st pass------------------!!"
475 fi
476 if [ $pass == 1 ]; then
477    echo ""
478    echo "!!---------------2nd pass------------------!!"
479 fi
480#
481
482# Restartability test
483 echo ""
484 echo "   !----restart----!   "
485 for restart_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WOVERFLOW_ST WLOCK_EXCHANGE_ST WVORTEX_ST WICE_AGRIF_ST
486 do
487   resttest $NEMO_VALID $restart_test $pass
488 done
489#
490# Reproducibility tests
491 echo ""
492 echo "   !----repro----!   "
493 for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WORCA2_ICE_OBS_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WVORTEX_ST WICE_AGRIF_ST
494 do
495   reprotest $NEMO_VALID $repro_test $pass
496 done
497
498# AGRIF special check to ensure results are unchanged with and without key_agrif
499 echo ""
500 echo "   !----agrif check----!   "
501 dir1=WAGRIF_DEMO_NOAGRIF_ST
502 dir2=WAGRIF_DEMO_ST
503 identictest $NEMO_VALID $dir1 $dir2 $pass 
504#
505# before/after tests
506 if [ $lastchange == 'old' ] ; then
507    echo ""
508    echo "   !---- 'old' specified as revision => no comparison with reference results ----!   "
509    echo ""
510 else
511   echo ""
512   echo "   !----result comparison check----!   "
513   if [ $NEMO_VALID_REF != "/path/to/reference/sette/results" ]; then
514     echo ''
515     echo 'check result differences between :'
516     echo "VALID directory : $NEMO_VALID at rev $lastchange"
517     echo 'and'
518     echo "REFERENCE directory : $NEMO_VALID_REF at rev $NEMO_REV_REF"
519     echo ''
520     for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WISOMIP_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WVORTEX_ST WICE_AGRIF_ST
521     do
522       runcmpres $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $pass
523     done
524   else
525     echo ''
526     echo ' No path for comparison specified. Result are not compare with any other revision. '
527     echo ' To do it please fill NEMO_VALID_REF and NEMO_REV_REF in param.cfg. '
528     echo ''
529   fi
530 fi
531done
532#
533exit
Note: See TracBrowser for help on using the repository browser.