New URL for NEMO forge!   http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.
sette_rpt.sh in utils/CI/sette_ticket2304 – NEMO

source: utils/CI/sette_ticket2304/sette_rpt.sh @ 12295

Last change on this file since 12295 was 12295, checked in by mathiot, 4 years ago

add time step number when result start to be different to the reference (ticket #2304)
  • Property svn:executable set to *
File size: 15.6 KB
Line 
1#!/bin/bash -f
2# set -vx
3# simple SETTE report generator.
4#
5# This version should be run in the SETTE directory.
6# The machine name will be picked up from the sette.sh script but the location of the
7# validation directory needs to be set here (currently assumed to reside in the ../cfgs directory)
8#
9#########################################################################################
10######################### Start of function definitions #################################
11##
12
13function get_dorv() {
14  if [ $lastchange == 'old' ] ; then
15    dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l `
16    dorv=`echo $dorv | sed -e 's:.*/::'`
17    dorv2=`ls -1rt $vdir/$nam2/$mach/ | tail -1l `
18    dorv2=`echo $dorv2 | sed -e 's:.*/::'`
19  else
20    dorv=$lastchange
21    dorv2=$lastchange
22  fi
23}
24
25function get_ktdiff() {
26  ktdiff=`diff ${1} ${2} | head -2 | grep it | awk '{ print $4 }'`
27}
28
29function resttest() { 
30#
31# Restartability checks. Expects LONG and SHORT run directories
32# Compares end of LONG stat files with equivalent entries from the SHORT stat files.
33#
34  vdir=$1
35  nam=$2
36  pass=$3
37#
38# get $dorv
39  get_dorv
40#
41# check if directory is here
42  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
43    printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv
44    return
45  fi
46
47  if [ -d $vdir/$nam/$mach/$dorv ]; then
48    # check ocean output
49    runtest $vdir $nam $pass RST
50    #
51    # run restartibility test
52    f1o=$vdir/$nam/$mach/$dorv/LONG/ocean.output
53    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
54    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
55    f2o=$vdir/$nam/$mach/$dorv/SHORT/ocean.output
56    f2s=$vdir/$nam/$mach/$dorv/SHORT/run.stat
57    f2t=$vdir/$nam/$mach/$dorv/SHORT/tracer.stat
58
59    if  [ ! -f $f1s ] &&  [ ! -f $f1t ] ; then
60      printf "%-27s %s\n" $nam " incomplete test";
61      return;
62    fi
63    if  [ ! -f $f2s ] &&  [ ! -f $f2t ] ; then
64      printf "%-27s %s\n" $nam " incomplete test";
65      return;
66    fi
67#
68    done_oce=0
69
70    if  [  -f $f1s ] && [  -f $f2s ]; then
71      nl=(`wc -l $f2s`)
72      tail -${nl[0]} $f1s > f1.tmp$$
73      cmp -s f1.tmp$$ $f2s
74      if [ $? == 0 ]; then
75        if [ $pass == 0 ]; then
76          printf "%-27s %s %s\n" $nam  " run.stat    restartability  passed : " $dorv
77        fi
78      else
79        get_ktdiff f1.tmp$$ $f2s
80        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " run.stat    restartability  FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
81#
82# Offer view of differences on the second pass
83#
84        if [ $pass == 1 ]; then
85          echo "<return> to view run.stat differences"
86          read y
87          sdiff f1.tmp$$ $f2s
88          echo "<return> to view ocean.output differences"
89          read y
90          sdiff $f1o $f2o | grep "|"
91          done_oce=1
92          echo "<return> to continue"
93          read y
94        fi
95      fi
96    fi
97#
98# Check tracer.stat files (if they exist)
99#
100    if  [  -f $f1t ] && [  -f $f2t ]; then
101      nl=(`wc -l $f2t`)
102      tail -${nl[0]} $f1t > f1.tmp$$
103      cmp -s f1.tmp$$ $f2t
104      if [ $? == 0 ]; then
105        if [ $pass == 0 ]; then
106          printf "%-27s %s %s\n" $nam  " tracer.stat restartability  passed : " $dorv
107        fi
108      else
109        get_ktdiff f1.tmp$$ $f2t
110        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " tracer.stat    restartability  FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
111#
112# Offer view of differences on the second pass
113#
114        if [ $pass == 1 ]; then
115          echo "<return> to view tracer.stat differences"
116          read y
117          sdiff f1.tmp$$ $f2t
118#
119# Only offer ocean.output view if it has not been viewed previously
120#
121          if [ $done_oce == 0 ]; then
122            echo "<return> to view ocean.output differences"
123            read y
124            sdiff $f1o $f2o | grep "|"
125          fi
126          echo "<return> to continue"
127          read y
128        fi
129      fi
130    fi
131    rm f1.tmp$$
132  fi
133}
134
135function reprotest(){
136#
137# Reproducibility checks. Expects REPRO_N_M and REPRO_I_J run directories
138# Compares end of stat files from each
139#
140  vdir=$1
141  nam=$2
142  pass=$3
143#
144# get $dorv
145  get_dorv
146#
147# check if directory is here
148  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
149    printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv
150    return
151  fi
152#
153  if [ -d $vdir/$nam/$mach/$dorv ]; then
154    # check ocean output
155    runtest $vdir $nam $pass REPRO
156    #
157    # check reproducibility
158    rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -2l | head -1 `
159    rep2=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -1l`
160    f1o=$vdir/$nam/$mach/$dorv/$rep1/ocean.output
161    f1s=$vdir/$nam/$mach/$dorv/$rep1/run.stat
162    f1t=$vdir/$nam/$mach/$dorv/$rep1/tracer.stat
163    f2o=$vdir/$nam/$mach/$dorv/$rep2/ocean.output
164    f2s=$vdir/$nam/$mach/$dorv/$rep2/run.stat
165    f2t=$vdir/$nam/$mach/$dorv/$rep2/tracer.stat
166
167    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
168      printf "%-27s %s\n" $nam " incomplete test";
169      return;
170    fi
171    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
172      printf "%-27s %s\n" $nam " incomplete test";
173      return;
174    fi
175#
176    done_oce=0
177
178    if  [ -f $f1s ] && [ -f $f2s ] ; then
179      cmp -s $f1s $f2s
180      if [ $? == 0 ]; then
181        if [ $pass == 0 ]; then
182          printf "%-27s %s %s\n" $nam  " run.stat    reproducibility passed : " $dorv
183        fi
184      else
185        get_ktdiff $f1s $f2s
186        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " run.stat    reproducibility FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
187#
188# Offer view of differences on the second pass
189#
190        if [ $pass == 1 ]; then
191          echo "<return> to view run.stat differences"
192          read y
193          sdiff $f1s $f2s
194          echo "<return> to view ocean.output differences"
195          read y
196          sdiff $f1o $f2o | grep "|"
197          done_oce=1
198          echo "<return> to continue"
199          read y
200        fi
201      fi
202    fi
203#
204# Check tracer.stat files (if they exist)
205#
206    if  [ -f $f1t ] && [ -f $f2t ] ; then
207      cmp -s $f1t $f2t
208      if [ $? == 0 ]; then
209        if [ $pass == 0 ]; then           printf "%-27s %s %s\n" $nam  " tracer.stat reproducibility passed : " $dorv
210        fi
211      else
212        get_ktdiff $f1t $f2t
213        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " tracer.stat reproducibility FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
214#
215# Offer view of differences on the second pass
216#
217        if [ $pass == 1 ]; then
218          echo "<return> to view tracer.stat differences"
219          read y
220          sdiff $f1t $f2t
221#
222# Only offer ocean.output view if it has not been viewed previously
223#
224          if [ $done_oce == 0 ]; then
225            echo "<return> to view ocean.output differences"
226            read y
227            sdiff $f1o $f2o | grep "|"
228          fi
229          echo "<return> to continue"
230          read y
231        fi
232      fi
233    fi
234  fi
235}
236function runcmpres(){
237#
238# compare *.stat file with reference file from a previous sette test or previous version
239# store in NEMO_VALID_REF at revision NEMO_REV_REF
240# Compares end of stat files from each
241#
242  vdir=$1
243  nam=$2
244  vdirref=$3
245  dorvref=$4
246  pass=$5
247#
248# get $dorv
249  get_dorv
250#
251# check if reference directory is present
252  if [ ! -d $vdirref/$nam/$mach/$dorvref ]; then
253    printf "%-27s %s\n" $nam " REFERENCE directory at $dorvref is MISSING"
254    return
255  fi
256  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
257    printf "%-27s %s\n" $nam " VALID     directory at $dorv is MISSING"
258    return
259  fi
260
261#
262  if [ -d $vdir/$nam/$mach/$dorv ]; then
263    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
264    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
265    f2s=$vdirref/$nam/$mach/$dorvref/LONG/run.stat
266    f2t=$vdirref/$nam/$mach/$dorvref/LONG/tracer.stat
267    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
268      printf "%-20s %s\n" $nam " incomplete test";
269      return;
270    fi
271    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
272      printf "%-20s %s\n" $nam " incomplete test";
273      return;
274    fi
275#
276    done_oce=0
277
278    if  [ -f $f1s ] && [ -f $f2s ] ; then
279      cmp -s $f1s $f2s
280      if [ $? == 0 ]; then
281        if [ $pass == 0 ]; then
282          printf "%-20s %s %s\n" $nam  " run.stat    files are identical "
283        fi
284      else
285        get_ktdiff $f1s $f2s
286        printf "%-20s %s %s %-5s %s\n" $nam  " run.stat    files are DIFFERENT (results are different after " $ktdiff " time steps)"
287#
288# Offer view of differences on the second pass
289#
290        if [ $pass == 1 ]; then
291          echo "<return> to view run.stat differences"
292          read y
293          sdiff $f1s $f2s
294          done_oce=1
295          echo "<return> to continue"
296          read y
297        fi
298      fi
299    fi
300    # Check tracer.stat files (if they exist)
301#
302    if  [ -f $f1t ] && [ -f $f2t ] ; then
303      cmp -s $f1t $f2t
304      if [ $? == 0 ]; then
305        if [ $pass == 0 ]; then         
306          printf "%-20s %s %s\n" $nam  " tracer.stat files are identical "
307        fi
308      else
309        get_ktdiff $f1t $f2t
310        printf "%-20s %s %s %-5s %s\n" $nam  " tracer.stat files are DIFFERENT (results are different after " $ktdiff " time steps) "
311#
312# Offer view of differences on the second pass
313#
314        if [ $pass == 1 ]; then
315          echo "<return> to view tracer.stat differences"
316          read y
317          sdiff $f1t $f2t
318        fi
319      fi
320    fi
321  fi
322}
323
324function runtest(){
325#
326# Run checks.
327# Check presence of E R R O R in ocean.output from each
328#
329  vdir=$1
330  nam=$2
331  pass=$3
332  ttype=$4
333  [[ $ttype == 'RST' ]] && ttype="LONG|SHORT"
334#
335# get $dorv
336  get_dorv
337#
338# no print needed if the repository is not here (already catch before)
339#
340  if [ -d $vdir/$nam/$mach/$dorv/ ]; then
341    #
342    # apply check for all ttype directory
343    rep1=$(ls -rt $vdir/$nam/$mach/$dorv/ | grep -E $ttype)
344    for tdir in $rep1 ; do
345       f1o=$vdir/$nam/$mach/$dorv/$tdir/ocean.output
346       if  [ ! -f $f1o ] ; then
347          if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam " ocean.output               MISSING : " $dorv ; fi
348          return;
349       else
350          nerr=`grep 'E R R O R' $f1o | wc -l`
351          if [[ $nerr > 0 ]]; then
352             printf "\e[38;5;196m%-27s %s %s %s\e[0m\n" $nam " run                         FAILED : " $dorv " ( E R R O R in ocean.output) " 
353             if [ $pass == 1 ]; then
354                echo "<return> to view end of ocean.output"
355                read y
356                tail -100 $f1o
357                echo ''
358                echo "full ocean.output available here: $f1o"
359             fi
360             return;
361          fi
362       fi
363    done
364  else
365    if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv ; fi
366  fi
367}
368
369function identictest(){
370#
371# Checks AGRIF does not corrupt results with no AGRIF zoom by comparing run.stat files
372#
373  vdir=$1
374  nam=$2
375  nam2=$3
376  pass=$4
377#
378  get_dorv
379#
380  rep=`ls -1rt $vdir/$nam/$mach/$dorv/ |  tail -1l`
381  f1s=${vdir}/${nam}/${mach}/${dorv}/${rep}/run.stat
382  f2s=${vdir}/${nam2}/${mach}/${dorv2}/${rep}/run.stat
383#
384  if  [ -f $f1s ] && [ -f $f2s ] ; then
385      cmp -s $f1s $f2s
386      if [ $? == 0 ]; then
387          if [ $pass == 0 ]; then
388         printf "%-5s %s %-5s %s %s %s\n" $rep "AGRIF vs" $rep "NOAGRIF run.stat    unchanged  -    passed : " $dorv $dorv2
389          fi
390      else
391          get_ktdiff $f1s $f2s
392          printf "\e[38;5;196m%-5s %s %-5s %s %s %s %s %-5s %s\e[0m\n" $rep "AGRIF vs" $rep "NOAGRIF run.stat    changed  -     FAILED : " $dorv $dorv2 " (results are different after " $ktdiff " time steps)"
393#
394# Offer view of differences on the second pass
395#
396          if [ $pass == 1 ]; then
397         echo "<return> to view run.stat differences"
398         read y
399         sdiff $f1s $f2s
400         echo "<return> to continue"
401         read y
402          fi
403      fi
404  else
405      printf "%-27s %-27s %s\n" $nam $nam2 " incomplete test"
406  fi
407}
408########################### END of function definitions #################################
409##                                                                                     ##
410##    Main script                                                                      ##
411##                                                                                     ##
412#########################################################################################
413#
414# LOAD param variable (COMPILER, NEMO_VALIDATION_DIR, SVN_CMD)
415  SETTE_DIR=$(cd $(dirname "$0"); pwd)
416  MAIN_DIR=$(dirname $SETTE_DIR)
417  . ./param.cfg
418
419  mach=${COMPILER}
420  NEMO_VALID=${NEMO_VALIDATION_DIR}
421  NEMO_VALID_REF=${NEMO_VALIDATION_REF}
422  LANG=en_US
423#
424  if [ ! -d $NEMO_VALID ]; then
425    echo "$NEMO_VALID validation directory not found"
426    exit
427  fi
428#
429# Directory to run the tests
430  CONFIG_DIR0=${MAIN_DIR}/cfgs
431  TOOLS_DIR=${MAIN_DIR}/tools
432  NPROC=32
433  SAS_RESTART_DIR=${CONFIG_DIR0}/ORCA2_SAS_ICE_ST
434#
435# Show current revision tag and branch name
436#
437echo ""
438lastchange=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep 'Last Changed Rev' | awk '{print $NF}'`
439revision=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep 'Revision' | awk '{print $NF}'`
440branchname=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep ^URL | awk -F ipsl/forge/projets/nemo/svn/ '{print $NF}'`
441echo "Current code is : $branchname @ r$revision  ( last change @ r$lastchange )"
442[ `${SVN_CMD} status -q ${MAIN_DIR}/{cfg,tests,src} | wc -l` -ge 1 ] && lastchange=${lastchange}+
443
444# overwrite revision is argument 1 present
445[[ $# -eq 1 ]] && lastchange=$1
446
447echo ""
448echo "SETTE validation report generated for : "
449echo ""
450echo "       $branchname @ r$lastchange (last changed revision)"
451echo ""
452echo "       on $COMPILER arch file"
453echo ""
454
455#
456# The script also needs the date or revision tag. Currently this is taken from the latest sub-directory found in each directory
457# 
458for pass in  0 1
459do
460#
461 if [ $pass == 0 ]; then
462   echo "" 
463   echo "!!---------------1st pass------------------!!"
464 fi
465 if [ $pass == 1 ]; then
466    echo ""
467    echo "!!---------------2nd pass------------------!!"
468 fi
469#
470
471# Restartability test
472 echo ""
473 echo "   !----restart----!   "
474 for restart_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WOVERFLOW_ST WLOCK_EXCHANGE_ST WVORTEX_ST WICE_AGRIF_ST
475 do
476   resttest $NEMO_VALID $restart_test $pass
477 done
478#
479# Reproducibility tests
480 echo ""
481 echo "   !----repro----!   "
482 for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WORCA2_ICE_OBS_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WVORTEX_ST WICE_AGRIF_ST
483 do
484   reprotest $NEMO_VALID $repro_test $pass
485 done
486
487# AGRIF special check to ensure results are unchanged with and without key_agrif
488 echo ""
489 echo "   !----agrif check----!   "
490 dir1=WAGRIF_DEMO_NOAGRIF_ST
491 dir2=WAGRIF_DEMO_ST
492 identictest $NEMO_VALID $dir1 $dir2 $pass 
493#
494# before/after tests
495 if [ $lastchange == 'old' ] ; then
496    echo ""
497    echo "   !---- 'old' specified as revision => no comparison with reference results ----!   "
498    echo ""
499 else
500   echo ""
501   echo "   !----result comparison check----!   "
502   if [ $NEMO_VALID_REF != "/path/to/reference/sette/results" ]; then
503     echo ''
504     echo 'check result differences between :'
505     echo "VALID directory : $NEMO_VALID at rev $lastchange"
506     echo 'and'
507     echo "REFERENCE directory : $NEMO_VALID_REF at rev $NEMO_REV_REF"
508     echo ''
509     for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WISOMIP_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WVORTEX_ST WICE_AGRIF_ST
510     do
511       runcmpres $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $pass
512     done
513   else
514     echo ''
515     echo ' No path for comparison specified. Result are not compare with any other revision. '
516     echo ' To do it please fill NEMO_VALID_REF and NEMO_REV_REF in param.cfg. '
517     echo ''
518   fi
519 fi
520done
521#
522exit
Note: See TracBrowser for help on using the repository browser.