New URL for NEMO forge!   http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.
sette_rpt.sh in utils/CI/fix_ticket2302_sette – NEMO

source: utils/CI/fix_ticket2302_sette/sette_rpt.sh @ 11490

Last change on this file since 11490 was 11490, checked in by smueller, 3 years ago

Ensuring of output from the test for all variants of incomplete tests (ticket #2302)

  • Property svn:executable set to *
File size: 15.8 KB
Line 
1#!/bin/bash -f
2# set -vx
3# simple SETTE report generator.
4#
5# This version should be run in the SETTE directory.
6# The machine name will be picked up from the sette.sh script but the location of the
7# validation directory needs to be set here (currently assumed to reside in the ../cfgs directory)
8#
9#########################################################################################
10######################### Start of function definitions #################################
11##
12
13function get_dorv() {
14  if [ $lastchange == 'old' ] ; then
15    dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l `
16    dorv=`echo $dorv | sed -e 's:.*/::'`
17  else
18    dorv=$lastchange
19  fi
20}
21
22function get_ktdiff() {
23  ktdiff=`diff ${1} ${2} | head -2 | grep it | awk '{ print $4 }'`
24}
25
26function resttest() { 
27#
28# Restartability checks. Expects LONG and SHORT run directories
29# Compares end of LONG stat files with equivalent entries from the SHORT stat files.
30#
31  vdir=$1
32  nam=$2
33  pass=$3
34#
35# get $dorv
36  get_dorv
37#
38# check if directory is here
39  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
40    printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv
41    return
42  fi
43
44  if [ -d $vdir/$nam/$mach/$dorv ]; then
45    # check ocean output
46    runtest $vdir $nam $pass RST
47    #
48    # run restartibility test
49    f1o=$vdir/$nam/$mach/$dorv/LONG/ocean.output
50    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
51    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
52    f2o=$vdir/$nam/$mach/$dorv/SHORT/ocean.output
53    f2s=$vdir/$nam/$mach/$dorv/SHORT/run.stat
54    f2t=$vdir/$nam/$mach/$dorv/SHORT/tracer.stat
55
56    if  [ ! -f $f1s ] &&  [ ! -f $f1t ] ; then
57      printf "%-27s %s\n" $nam " incomplete test";
58      return;
59    fi
60    if  [ ! -f $f2s ] &&  [ ! -f $f2t ] ; then
61      printf "%-27s %s\n" $nam " incomplete test";
62      return;
63    fi
64#
65    done_oce=0
66
67    if  [  -f $f1s ] && [  -f $f2s ]; then
68      nl=(`wc -l $f2s`)
69      tail -${nl[0]} $f1s > f1.tmp$$
70      cmp -s f1.tmp$$ $f2s
71      if [ $? == 0 ]; then
72        if [ $pass == 0 ]; then
73          printf "%-27s %s %s\n" $nam  " run.stat    restartability  passed : " $dorv
74        fi
75      else
76        get_ktdiff f1.tmp$$ $f2s
77        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " run.stat    restartability  FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
78#
79# Offer view of differences on the second pass
80#
81        if [ $pass == 1 ]; then
82          echo "<return> to view run.stat differences"
83          read y
84          sdiff f1.tmp$$ $f2s
85          echo "<return> to view ocean.output differences"
86          read y
87          sdiff $f1o $f2o | grep "|"
88          done_oce=1
89          echo "<return> to continue"
90          read y
91        fi
92      fi
93    fi
94#
95# Check tracer.stat files (if they exist)
96#
97    if  [  -f $f1t ] && [  -f $f2t ]; then
98      nl=(`wc -l $f2t`)
99      tail -${nl[0]} $f1t > f1.tmp$$
100      cmp -s f1.tmp$$ $f2t
101      if [ $? == 0 ]; then
102        if [ $pass == 0 ]; then
103          printf "%-27s %s %s\n" $nam  " tracer.stat restartability  passed : " $dorv
104        fi
105      else
106        get_ktdiff f1.tmp$$ $f2t
107        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " tracer.stat    restartability  FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
108#
109# Offer view of differences on the second pass
110#
111        if [ $pass == 1 ]; then
112          echo "<return> to view tracer.stat differences"
113          read y
114          sdiff f1.tmp$$ $f2t
115#
116# Only offer ocean.output view if it has not been viewed previously
117#
118          if [ $done_oce == 0 ]; then
119            echo "<return> to view ocean.output differences"
120            read y
121            sdiff $f1o $f2o | grep "|"
122          fi
123          echo "<return> to continue"
124          read y
125        fi
126      fi
127    fi
128    rm f1.tmp$$
129  fi
130}
131
132function reprotest(){
133#
134# Reproducibility checks. Expects REPRO_N_M and REPRO_I_J run directories
135# Compares end of stat files from each
136#
137  vdir=$1
138  nam=$2
139  pass=$3
140#
141# get $dorv
142  get_dorv
143#
144# check if directory is here
145  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
146    printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv
147    return
148  fi
149#
150  if [ -d $vdir/$nam/$mach/$dorv ]; then
151    # check ocean output
152    runtest $vdir $nam $pass REPRO
153    #
154    # check reproducibility
155    rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -2l | head -1 `
156    rep2=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -1l`
157    f1o=$vdir/$nam/$mach/$dorv/$rep1/ocean.output
158    f1s=$vdir/$nam/$mach/$dorv/$rep1/run.stat
159    f1t=$vdir/$nam/$mach/$dorv/$rep1/tracer.stat
160    f2o=$vdir/$nam/$mach/$dorv/$rep2/ocean.output
161    f2s=$vdir/$nam/$mach/$dorv/$rep2/run.stat
162    f2t=$vdir/$nam/$mach/$dorv/$rep2/tracer.stat
163
164    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
165      printf "%-27s %s\n" $nam " incomplete test";
166      return;
167    fi
168    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
169      printf "%-27s %s\n" $nam " incomplete test";
170      return;
171    fi
172#
173    done_oce=0
174
175    if  [ -f $f1s ] && [ -f $f2s ] ; then
176      cmp -s $f1s $f2s
177      if [ $? == 0 ]; then
178        if [ $pass == 0 ]; then
179          printf "%-27s %s %s\n" $nam  " run.stat    reproducibility passed : " $dorv
180        fi
181      else
182        get_ktdiff $f1s $f2s
183        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " run.stat    reproducibility FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
184#
185# Offer view of differences on the second pass
186#
187        if [ $pass == 1 ]; then
188          echo "<return> to view run.stat differences"
189          read y
190          sdiff $f1s $f2s
191          echo "<return> to view ocean.output differences"
192          read y
193          sdiff $f1o $f2o | grep "|"
194          done_oce=1
195          echo "<return> to continue"
196          read y
197        fi
198      fi
199    fi
200#
201# Check tracer.stat files (if they exist)
202#
203    if  [ -f $f1t ] && [ -f $f2t ] ; then
204      cmp -s $f1t $f2t
205      if [ $? == 0 ]; then
206        if [ $pass == 0 ]; then           printf "%-27s %s %s\n" $nam  " tracer.stat reproducibility passed : " $dorv
207        fi
208      else
209        get_ktdiff $f1t $f2t
210        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " tracer.stat    reproducibility FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
211#
212# Offer view of differences on the second pass
213#
214        if [ $pass == 1 ]; then
215          echo "<return> to view tracer.stat differences"
216          read y
217          sdiff $f1t $f2t
218#
219# Only offer ocean.output view if it has not been viewed previously
220#
221          if [ $done_oce == 0 ]; then
222            echo "<return> to view ocean.output differences"
223            read y
224            sdiff $f1o $f2o | grep "|"
225          fi
226          echo "<return> to continue"
227          read y
228        fi
229      fi
230    fi
231  fi
232}
233function runcmpres(){
234#
235# compare *.stat file with reference file from a previous sette test or previous version
236# store in NEMO_VALID_REF at revision NEMO_REV_REF
237# Compares end of stat files from each
238#
239  vdir=$1
240  nam=$2
241  vdirref=$3
242  dorvref=$4
243  pass=$5
244#
245# get $dorv
246  get_dorv
247#
248# check if reference directory is present
249  if [ ! -d $vdirref/$nam/$mach/$dorvref ]; then
250    printf "%-27s %s\n" $nam " REFERENCE directory at $dorvref is MISSING"
251    return
252  fi
253  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
254    printf "%-27s %s\n" $nam " VALID     directory at $dorv is MISSING"
255    return
256  fi
257
258#
259  if [ -d $vdir/$nam/$mach/$dorv ]; then
260    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
261    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
262    f2s=$vdirref/$nam/$mach/$dorvref/LONG/run.stat
263    f2t=$vdirref/$nam/$mach/$dorvref/LONG/tracer.stat
264    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
265      printf "%-20s %s\n" $nam " incomplete test";
266      return;
267    fi
268    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
269      printf "%-20s %s\n" $nam " incomplete test";
270      return;
271    fi
272#
273    done_oce=0
274
275    if  [ -f $f1s ] && [ -f $f2s ] ; then
276      cmp -s $f1s $f2s
277      if [ $? == 0 ]; then
278        if [ $pass == 0 ]; then
279          printf "%-20s %s %s\n" $nam  " run.stat    files are identical "
280        fi
281      else
282        printf "%-20s %s %s\n" $nam  " run.stat    files are DIFFERENT "
283#
284# Offer view of differences on the second pass
285#
286        if [ $pass == 1 ]; then
287          echo "<return> to view run.stat differences"
288          read y
289          sdiff $f1s $f2s
290          done_oce=1
291          echo "<return> to continue"
292          read y
293        fi
294      fi
295    fi
296    # Check tracer.stat files (if they exist)
297#
298    if  [ -f $f1t ] && [ -f $f2t ] ; then
299      cmp -s $f1t $f2t
300      if [ $? == 0 ]; then
301        if [ $pass == 0 ]; then         
302          printf "%-20s %s %s\n" $nam  " tracer.stat files are identical "
303        fi
304      else
305        printf "%-20s %s %s\n" $nam  " tracer.stat files are DIFFERENT "
306#
307# Offer view of differences on the second pass
308#
309        if [ $pass == 1 ]; then
310          echo "<return> to view tracer.stat differences"
311          read y
312          sdiff $f1t $f2t
313        fi
314      fi
315    fi
316  fi
317}
318
319function runtest(){
320#
321# Run checks.
322# Check presence of E R R O R in ocean.output from each
323#
324  vdir=$1
325  nam=$2
326  pass=$3
327  ttype=$4
328  [[ $ttype == 'RST' ]] && ttype="LONG|SHORT"
329#
330# get $dorv
331  get_dorv
332#
333# no print needed if the repository is not here (already catch before)
334#
335  if [ -d $vdir/$nam/$mach/$dorv/ ]; then
336    #
337    # apply check for all ttype directory
338    rep1=$(ls -rt $vdir/$nam/$mach/$dorv/ | grep -E $ttype)
339    for tdir in $rep1 ; do
340       f1o=$vdir/$nam/$mach/$dorv/$tdir/ocean.output
341       if  [ ! -f $f1o ] ; then
342          if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam " ocean.output               MISSING : " $dorv ; fi
343          return;
344       else
345          nerr=`grep 'E R R O R' $f1o | wc -l`
346          if [[ $nerr > 0 ]]; then
347             printf "\e[38;5;196m%-27s %s %s %s\e[0m\n" $nam " run                         FAILED : " $dorv " ( E R R O R in ocean.output) " 
348             if [ $pass == 1 ]; then
349                echo "<return> to view end of ocean.output"
350                read y
351                tail -100 $f1o
352                echo ''
353                echo "full ocean.output available here: $f1o"
354             fi
355             return;
356          fi
357       fi
358    done
359  else
360    if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv ; fi
361  fi
362}
363
364function identictest(){
365#
366#  checks AGRIF does not corrupe results with no AGRIF zoom. Expects ORCA2AGUL/AGRIFNOZ and ORCA2AGUL_NAGR/AGRIFNO  run directories
367# Compares solver.stat files for each
368#
369  vdir=$1
370  dir1=$2
371  dir2=$3
372  pass=$4
373#
374  if [ -d $vdir/$dir1 ] && [ -d $vdir/$dir2 ]; then
375    dorv1=`ls -1rt $vdir/$dir1/$mach/ | tail -1l `
376    dorv1=`echo $dorv1 | sed -e 's:.*/::'`
377    dorv2=`ls -1rt $vdir/$dir2/$mach/ | tail -1l `
378    dorv2=`echo $dorv2 | sed -e 's:.*/::'`
379
380    rep1=`ls -1rt $vdir/$dir1/$mach/$dorv1/ |  tail -1l`
381#clem    rep2=`ls -1rt $vdir/$dir2/$mach/$dorv2/ |  tail -1l`
382    rep2=`ls -1rt $vdir/$dir1/$mach/$dorv1/ |  tail -1l`
383    f1s=$vdir/$dir1/$mach/$dorv1/$rep1/run.stat
384    f2s=$vdir/$dir2/$mach/$dorv2/$rep2/run.stat
385#
386    done_oce=0
387
388    if  [ -f $f1s ] && [ -f $f2s ] ; then
389      cmp -s $f1s $f2s
390      if [ $? == 0 ]; then
391        if [ $pass == 0 ]; then
392          printf "%-5s %s %-5s %s %s %s\n" $rep1 "AGRIF vs" $rep2 "NOAGRIF run.stat    unchanged  -    passed : " $dorv1 $dorv2
393        fi
394      else
395        get_ktdiff $f1s $f2s
396        printf "\e[38;5;196m%-5s %s %-5s %s %s %s %s %-5s %s\e[0m\n" $rep1 "AGRIF vs" $rep2 "NOAGRIF run.stat    changed  -     FAILED : " $dorv1 $dorv2 " (results are different after " $ktdiff " time steps)"
397#
398# Offer view of differences on the second pass
399#
400        if [ $pass == 1 ]; then
401          echo "<return> to view run.stat differences"
402          read y
403          sdiff $f1s $f2s
404          done_oce=1
405          echo "<return> to continue"
406          read y
407        fi
408      fi
409    else
410      printf "%-27s %s\n" $dir1 $dir2 " incomplete test"
411    fi
412  else
413    printf "%-27s %s\n" $dir1 $dir2 " incomplete test";
414  fi
415}
416########################### END of function definitions #################################
417##                                                                                     ##
418##    Main script                                                                      ##
419##                                                                                     ##
420#########################################################################################
421#
422# LOAD param variable (COMPILER, NEMO_VALIDATION_DIR, SVN_CMD)
423  SETTE_DIR=$(cd $(dirname "$0"); pwd)
424  MAIN_DIR=$(dirname $SETTE_DIR)
425  . ./param.cfg
426
427  mach=${COMPILER}
428  NEMO_VALID=${NEMO_VALIDATION_DIR}
429  NEMO_VALID_REF=${NEMO_VALIDATION_REF}
430  LANG=en_US
431#
432  if [ ! -d $NEMO_VALID ]; then
433    echo "$NEMO_VALID validation directory not found"
434    exit
435  fi
436#
437# Directory to run the tests
438  CONFIG_DIR0=${MAIN_DIR}/cfgs
439  TOOLS_DIR=${MAIN_DIR}/tools
440  NPROC=32
441  SAS_RESTART_DIR=${CONFIG_DIR0}/ORCA2_SAS_ICE_ST
442#
443# Show current revision tag and branch name
444#
445echo ""
446lastchange=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep 'Last Changed Rev' | awk '{print $NF}'`
447revision=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep 'Revision' | awk '{print $NF}'`
448branchname=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep ^URL | awk -F ipsl/forge/projets/nemo/svn/ '{print $NF}'`
449echo "Current code is : $branchname @ r$revision  ( last change @ r$lastchange )"
450[ `${SVN_CMD} status -q ${MAIN_DIR}/{cfg,tests,src} | wc -l` -ge 1 ] && lastchange=${lastchange}+
451
452# overwrite revision is argument 1 present
453[[ $# -eq 1 ]] && lastchange=$1
454
455echo ""
456echo "SETTE validation report generated for : "
457echo ""
458echo "       $branchname @ r$lastchange (last changed revision)"
459echo ""
460echo "       on $COMPILER arch file"
461echo ""
462
463#
464# The script also needs the date or revision tag. Currently this is taken from the latest sub-directory found in each directory
465
466for pass in  0 1
467do
468#
469 if [ $pass == 0 ]; then
470   echo "" 
471   echo "!!---------------1st pass------------------!!"
472 fi
473 if [ $pass == 1 ]; then
474    echo ""
475    echo "!!---------------2nd pass------------------!!"
476 fi
477#
478
479# Restartability test
480 echo ""
481 echo "   !----restart----!   "
482 for restart_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WOVERFLOW_ST WLOCK_EXCHANGE_ST WVORTEX_ST WICE_AGRIF_ST
483 do
484   resttest $NEMO_VALID $restart_test $pass
485 done
486#
487# Reproducibility tests
488 echo ""
489 echo "   !----repro----!   "
490 for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WORCA2_ICE_OBS_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WVORTEX_ST WICE_AGRIF_ST
491 do
492   reprotest $NEMO_VALID $repro_test $pass
493 done
494
495# AGRIF special check to ensure results are unchanged with and without key_agrif
496 echo ""
497 echo "   !----agrif check----!   "
498 dir1=WAGRIF_DEMO_NOAGRIF_ST
499 dir2=WAGRIF_DEMO_ST
500 identictest $NEMO_VALID $dir1 $dir2 $pass 
501#
502# before/after tests
503 if [ $lastchange == 'old' ] ; then
504    echo ""
505    echo "   !---- 'old' specified as revision => no comparison with reference results ----!   "
506    echo ""
507 else
508   echo ""
509   echo "   !----result comparison check----!   "
510   if [ $NEMO_VALID_REF != "/path/to/reference/sette/results" ]; then
511     echo ''
512     echo 'check result differences between :'
513     echo "VALID directory : $NEMO_VALID at rev $lastchange"
514     echo 'and'
515     echo "REFERENCE directory : $NEMO_VALID_REF at rev $NEMO_REV_REF"
516     echo ''
517     for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WISOMIP_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WVORTEX_ST WICE_AGRIF_ST
518     do
519       runcmpres $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $pass
520     done
521   else
522     echo ''
523     echo ' No path for comparison specified. Result are not compare with any other revision. '
524     echo ' To do it please fill NEMO_VALID_REF and NEMO_REV_REF in param.cfg. '
525     echo ''
526   fi
527 fi
528done
529#
530exit
Note: See TracBrowser for help on using the repository browser.