New URL for NEMO forge!   http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.
sette_rpt.sh in utils/CI/sette_ticket2673 – NEMO

source: utils/CI/sette_ticket2673/sette_rpt.sh @ 14867

Last change on this file since 14867 was 14867, checked in by acc, 3 years ago

Branch: sette_ticket2673. Fully working version of revamped submission scripts. See #2673 for details of changes. Still some work to do to bring the associated scripts such as sette_list_avail_rev.sh in line with new design.
  • Property svn:executable set to *
File size: 18.3 KB
Line 
1#!/bin/bash -f
2# set -vx
3# simple SETTE report generator.
4#
5# This version should be run in the SETTE directory.
6# The machine name will be picked up from the sette.sh script but the location of the
7# validation directory needs to be set here (currently assumed to reside in the ../cfgs directory)
8#
9#########################################################################################
10######################### Start of function definitions #################################
11##
12
13function get_dorv() {
14  if [ $lastchange == 'old' ] ; then
15    dorv=`ls -1rt $vdir/$mach/ | tail -1l `
16    dorv=`echo $dorv | sed -e 's:.*/::'`
17    dorv2=`ls -1rt $vdir/$mach/ 2>/dev/null | tail -1l `
18    dorv2=`echo $dorv2 | sed -e 's:.*/::'`
19  else
20    dorv=$lastchange
21    dorv2=$lastchange
22  fi
23}
24
25function get_ktdiff() {
26  ktdiff=`diff ${1} ${2} | head -2 | grep it | awk '{ print $4 }'`
27}
28
29function get_ktdiff2() {
30  ktdiff=`diff ${1} ${2} |  head -2 | tail -1l | awk '{print $2}'`
31}
32
33function resttest() { 
34#
35# Restartability checks. Expects LONG and SHORT run directories
36# Compares end of LONG stat files with equivalent entries from the SHORT stat files.
37#
38  vdir=$1
39  nam=$2
40  pass=$3
41#
42# get $dorv
43  get_dorv
44#
45# check if directory is here
46  if [ ! -d $vdir/$mach/$dorv/$nam ]; then
47    printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv
48    return
49  fi
50
51  if [ -d $vdir/$mach/$dorv/$nam ]; then
52    # check ocean output
53    runtest $vdir $nam $pass RST
54    #
55    # run restartibility test
56    f1o=$vdir/$mach/$dorv/$nam/LONG/ocean.output
57    f1s=$vdir/$mach/$dorv/$nam/LONG/run.stat
58    f1t=$vdir/$mach/$dorv/$nam/LONG/tracer.stat
59    f2o=$vdir/$mach/$dorv/$nam/SHORT/ocean.output
60    f2s=$vdir/$mach/$dorv/$nam/SHORT/run.stat
61    f2t=$vdir/$mach/$dorv/$nam/SHORT/tracer.stat
62
63    if  [ ! -f $f1s ] &&  [ ! -f $f1t ] ; then
64      printf "%-27s %s\n" $nam " incomplete test";
65      return;
66    fi
67    if  [ ! -f $f2s ] &&  [ ! -f $f2t ] ; then
68      printf "%-27s %s\n" $nam " incomplete test";
69      return;
70    fi
71#
72    done_oce=0
73
74    if  [  -f $f1s ] && [  -f $f2s ]; then
75      nl=(`wc -l $f2s`)
76      tail -${nl[0]} $f1s > f1.tmp$$
77      cmp -s f1.tmp$$ $f2s
78      if [ $? == 0 ]; then
79        if [ $pass == 0 ]; then
80          printf "%-27s %s %s\n" $nam  " run.stat    restartability  passed : " $dorv
81        fi
82      else
83        get_ktdiff f1.tmp$$ $f2s
84        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " run.stat    restartability  FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
85#
86# Offer view of differences on the second pass
87#
88        if [ $pass == 1 ]; then
89          echo "<return> to view run.stat differences"
90          read y
91          sdiff f1.tmp$$ $f2s
92          echo "<return> to view ocean.output differences"
93          read y
94          sdiff $f1o $f2o | grep "|"
95          done_oce=1
96          echo "<return> to continue"
97          read y
98        fi
99      fi
100    fi
101#
102# Check tracer.stat files (if they exist)
103#
104    if  [  -f $f1t ] && [  -f $f2t ]; then
105      nl=(`wc -l $f2t`)
106      tail -${nl[0]} $f1t > f1.tmp$$
107      cmp -s f1.tmp$$ $f2t
108      if [ $? == 0 ]; then
109        if [ $pass == 0 ]; then
110          printf "%-27s %s %s\n" $nam  " tracer.stat restartability  passed : " $dorv
111        fi
112      else
113        get_ktdiff2 f1.tmp$$ $f2t
114        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " tracer.stat    restartability  FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
115#
116# Offer view of differences on the second pass
117#
118        if [ $pass == 1 ]; then
119          echo "<return> to view tracer.stat differences"
120          read y
121          sdiff f1.tmp$$ $f2t
122#
123# Only offer ocean.output view if it has not been viewed previously
124#
125          if [ $done_oce == 0 ]; then
126            echo "<return> to view ocean.output differences"
127            read y
128            sdiff $f1o $f2o | grep "|"
129          fi
130          echo "<return> to continue"
131          read y
132        fi
133      fi
134    fi
135    rm f1.tmp$$
136  fi
137}
138
139function reprotest(){
140#
141# Reproducibility checks. Expects REPRO_N_M and REPRO_I_J run directories
142# Compares end of stat files from each
143#
144  vdir=$1
145  nam=$2
146  pass=$3
147#
148# get $dorv
149  get_dorv
150#
151# check if directory is here
152  if [ ! -d $vdir/$mach/$dorv/$nam ]; then
153    printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv
154    return
155  fi
156#
157  if [ -d $vdir/$mach/$dorv/$nam ]; then
158    # check ocean output
159    runtest $vdir $nam $pass REPRO
160    #
161    # check reproducibility
162    rep1=`ls -1rt $vdir/$mach/$dorv/$nam/ | grep REPRO | tail -2l | head -1 `
163    rep2=`ls -1rt $vdir/$mach/$dorv/$nam/ | grep REPRO | tail -1l`
164    if [ $rep1 == $rep2 ]; then
165       rep2=''
166    fi
167    f1o=$vdir/$mach/$dorv/$nam/$rep1/ocean.output
168    f1s=$vdir/$mach/$dorv/$nam/$rep1/run.stat
169    f1t=$vdir/$mach/$dorv/$nam/$rep1/tracer.stat
170    f2o=$vdir/$mach/$dorv/$nam/$rep2/ocean.output
171    f2s=$vdir/$mach/$dorv/$nam/$rep2/run.stat
172    f2t=$vdir/$mach/$dorv/$nam/$rep2/tracer.stat
173
174    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
175      printf "%-27s %s\n" $nam " incomplete test";
176      return;
177    fi
178    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
179      printf "%-27s %s\n" $nam " incomplete test";
180      return;
181    fi
182#
183    done_oce=0
184
185    if  [ -f $f1s ] && [ -f $f2s ] ; then
186      cmp -s $f1s $f2s
187      if [ $? == 0 ]; then
188        if [ $pass == 0 ]; then
189          printf "%-27s %s %s\n" $nam  " run.stat    reproducibility passed : " $dorv
190        fi
191      else
192        get_ktdiff $f1s $f2s
193        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " run.stat    reproducibility FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
194#
195# Offer view of differences on the second pass
196#
197        if [ $pass == 1 ]; then
198          echo "<return> to view run.stat differences"
199          read y
200          sdiff $f1s $f2s
201          echo "<return> to view ocean.output differences"
202          read y
203          sdiff $f1o $f2o | grep "|"
204          done_oce=1
205          echo "<return> to continue"
206          read y
207        fi
208      fi
209    fi
210#
211# Check tracer.stat files (if they exist)
212#
213    if  [ -f $f1t ] && [ -f $f2t ] ; then
214      cmp -s $f1t $f2t
215      if [ $? == 0 ]; then
216        if [ $pass == 0 ]; then           printf "%-27s %s %s\n" $nam  " tracer.stat reproducibility passed : " $dorv
217        fi
218      else
219        get_ktdiff2 $f1t $f2t
220        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " tracer.stat reproducibility FAILED : " $dorv " (results are different after " $ktdiff " time steps)"
221#
222# Offer view of differences on the second pass
223#
224        if [ $pass == 1 ]; then
225          echo "<return> to view tracer.stat differences"
226          read y
227          sdiff $f1t $f2t
228#
229# Only offer ocean.output view if it has not been viewed previously
230#
231          if [ $done_oce == 0 ]; then
232            echo "<return> to view ocean.output differences"
233            read y
234            sdiff $f1o $f2o | grep "|"
235          fi
236          echo "<return> to continue"
237          read y
238        fi
239      fi
240    fi
241  fi
242}
243function runcmpres(){
244#
245# compare *.stat file with reference file from a previous sette test or previous version
246# store in NEMO_VALID_REF at revision NEMO_REV_REF
247# Compares end of stat files from each
248#
249  vdir=$1
250  nam=$2
251  vdirref=$3
252  dorvref=$4
253  pass=$5
254#
255# get $dorv
256  get_dorv
257#
258# check if reference directory is present
259  if [ ! -d $vdirref/$mach/$dorv/$namref ]; then
260    printf "%-27s %s\n" $nam " REFERENCE directory at $dorvref is MISSING"
261    return
262  fi
263  if [ ! -d $vdir/$mach/$dorv/$nam ]; then
264    printf "%-27s %s\n" $nam " VALID     directory at $dorv is MISSING"
265    return
266  fi
267
268#
269  if [ -d $vdir/$mach/$dorv/$nam ]; then
270    f1s=$vdir/$mach/$dorv/$nam/LONG/run.stat
271    f1t=$vdir/$mach/$dorv/$nam/LONG/tracer.stat
272    f2s=$vdirref/$mach/$dorv/$namref/LONG/run.stat
273    f2t=$vdirref/$mach/$dorv/$namref/LONG/tracer.stat
274    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
275      printf "%-20s %s\n" $nam " incomplete test";
276      return;
277    fi
278    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
279      printf "%-20s %s\n" $nam " incomplete test";
280      return;
281    fi
282#
283    done_oce=0
284
285    if  [ -f $f1s ] && [ -f $f2s ] ; then
286      cmp -s $f1s $f2s
287      if [ $? == 0 ]; then
288        if [ $pass == 0 ]; then
289          printf "%-20s %s %s\n" $nam  " run.stat    files are identical "
290        fi
291      else
292        get_ktdiff $f1s $f2s
293        printf "%-20s %s %s %-5s %s\n" $nam  " run.stat    files are DIFFERENT (results are different after " $ktdiff " time steps)"
294#
295# Offer view of differences on the second pass
296#
297        if [ $pass == 1 ]; then
298          echo "<return> to view run.stat differences"
299          read y
300          sdiff $f1s $f2s
301          done_oce=1
302          echo "<return> to continue"
303          read y
304        fi
305      fi
306    fi
307    # Check tracer.stat files (if they exist)
308#
309    if  [ -f $f1t ] && [ -f $f2t ] ; then
310      cmp -s $f1t $f2t
311      if [ $? == 0 ]; then
312        if [ $pass == 0 ]; then         
313          printf "%-20s %s %s\n" $nam  " tracer.stat files are identical "
314        fi
315      else
316        get_ktdiff2 $f1t $f2t
317        printf "%-20s %s %s %-5s %s\n" $nam  " tracer.stat files are DIFFERENT (results are different after " $ktdiff " time steps) "
318#
319# Offer view of differences on the second pass
320#
321        if [ $pass == 1 ]; then
322          echo "<return> to view tracer.stat differences"
323          read y
324          sdiff $f1t $f2t
325        fi
326      fi
327    fi
328  fi
329}
330
331function runcmptim(){
332#
333# compare timing.output file with reference file from a previous sette test or previous version
334#
335  vdir=$1
336  nam=$2
337  vdirref=$3
338  dorvref=$4
339  pass=$5
340#
341# get $dorv
342  get_dorv
343#
344# check if reference directory is present
345  if [ ! -d $vdirref/$mach/$dorv/$namref ]; then
346    return
347  fi
348  if [ ! -d $vdir/$mach/$dorv/$nam ]; then
349    return
350  fi
351
352#
353  if [ -d $vdir/$mach/$dorv/$nam ]; then
354    f1a=$vdir/$mach/$dorv/$nam/LONG/timing.output
355    f2a=$vdirref/$mach/$dorv/$namref/LONG/timing.output
356#
357# Report average CPU time differences (if available)
358#
359    if  [ -f $f1a ] && [ -f $f2a ] ; then
360      tnew=$(grep 'Average ' $f1a  | awk '{print $5}')
361      tref=$(grep 'Average ' $f2a  | awk '{print $5}')
362      if [ $? == 0 ]; then
363        if [ $pass == 0 ]; then
364          tdif=$( echo ${tnew} ${tref} | awk '{print $1 - $2}')
365          if (( $(echo "$tnew > $tref" |bc -l) )); then
366            printf "%-20s %14s %10s %14s %10s %14s \\e[41;33;196m%10s\\e[0m\n" $nam  " ref. time:" $tref "cur. time:" $tnew "diff.:" $tdif
367          else
368            printf "%-20s %14s %10s %14s %10s %14s \\e[42;01;196m%10s\\e[0m\n" $nam  " ref. time:" $tref "cur. time:" $tnew "diff.:" $tdif
369          fi
370        fi
371      fi
372    fi
373  fi
374}
375
376function runtest(){
377#
378# Run checks.
379# Check presence of E R R O R in ocean.output from each
380#
381  vdir=$1
382  nam=$2
383  pass=$3
384  ttype=$4
385  [[ $ttype == 'RST' ]] && ttype="LONG|SHORT"
386#
387# get $dorv
388  get_dorv
389#
390# no print needed if the repository is not here (already catch before)
391#
392  if [ -d $vdir/$mach/$dorv/$nam/ ]; then
393    #
394    # apply check for all ttype directory
395    rep1=$(ls -rt $vdir/$mach/$dorv/$nam/ | grep -E $ttype)
396    for tdir in $rep1 ; do
397       f1o=$vdir/$mach/$dorv/$nam/$tdir/ocean.output
398       if  [ ! -f $f1o ] ; then
399          if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam " ocean.output               MISSING : " $dorv ; fi
400          return;
401       else
402          nerr=`grep 'E R R O R' $f1o | wc -l`
403          if [[ $nerr > 0 ]]; then
404             printf "\e[38;5;196m%-27s %s %s %s\e[0m\n" $nam " run                         FAILED : " $dorv " ( E R R O R in ocean.output) " 
405             if [ $pass == 1 ]; then
406                echo "<return> to view end of ocean.output"
407                read y
408                tail -100 $f1o
409                echo ''
410                echo "full ocean.output available here: $f1o"
411             fi
412             return;
413          fi
414       fi
415    done
416  else
417    if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv ; fi
418  fi
419}
420
421function identictest(){
422#
423# Checks AGRIF does not corrupt results with no AGRIF zoom by comparing run.stat files
424#
425  vdir=$1
426  nam=$2
427  nam2=$3
428  pass=$4
429#
430  get_dorv
431#
432  rep=`ls -1rt $vdir/$mach/$dorv/$nam/ |  tail -1l`
433  f1s=${vdir}/${mach}/${dorv}/${nam}/${rep}/run.stat
434  f2s=${vdir}/${mach}/${dorv2}/${nam2}/${rep}/run.stat
435#
436  if  [ -f $f1s ] && [ -f $f2s ] ; then
437      cmp -s $f1s $f2s
438      if [ $? == 0 ]; then
439          if [ $pass == 0 ]; then
440         printf "%-5s %s %-5s %s %s %s\n" $rep "AGRIF vs" $rep "NOAGRIF run.stat    unchanged  -    passed : " $dorv $dorv2
441          fi
442      else
443          get_ktdiff $f1s $f2s
444          printf "\e[38;5;196m%-5s %s %-5s %s %s %s %s %-5s %s\e[0m\n" $rep "AGRIF vs" $rep "NOAGRIF run.stat    changed  -     FAILED : " $dorv $dorv2 " (results are different after " $ktdiff " time steps)"
445#
446# Offer view of differences on the second pass
447#
448          if [ $pass == 1 ]; then
449         echo "<return> to view run.stat differences"
450         read y
451         sdiff $f1s $f2s
452         echo "<return> to continue"
453         read y
454          fi
455      fi
456  else
457      printf "%-27s %-27s %s\n" $nam $nam2 " incomplete test"
458  fi
459}
460########################### END of function definitions #################################
461##                                                                                     ##
462##    Main script                                                                      ##
463##                                                                                     ##
464#########################################################################################
465#
466# LOAD param variable (COMPILER, NEMO_VALIDATION_DIR, SVN_CMD)
467  SETTE_DIR=$(cd $(dirname "$0"); pwd)
468  MAIN_DIR=$(dirname $SETTE_DIR)
469  . ./param.cfg
470
471  mach=${COMPILER}
472# overwrite revision (later) or compiler
473  if [ $# -gt 0 ]; then
474    while getopts r:c:v:h option; do
475       case $option in
476          c) mach=$OPTARG;;
477          r) rev=$OPTARG;;
478          v) SETTE_SUB_VAL=$OPTARG;;
479          h | *) echo ''
480                 echo 'sette_rpt.sh : ' 
481                 echo '     display result for the latest change'
482                 echo ' -c COMPILER_name :'
483                 echo '     display result for the specified compiler'
484                 echo ' -r REVISION_number :'
485                 echo '     display sette results for the specified revision (set old for the latest revision available for each config)'
486                 echo ' -v sub_dir :'
487                 echo '     validation sub-directory below NEMO_VALIDATION_DIR'
488                 echo ''
489                 exit 42;;
490       esac
491    done
492    shift $((OPTIND - 1))
493  fi
494# if $1 (remaining arguments)
495  if [[ ! -z $1 ]] ; then rev=$1 ; fi
496
497  if [ ! -z $SETTE_SUB_VAL ] ; then
498   export NEMO_VALIDATION_DIR=$NEMO_VALIDATION_DIR/$SETTE_SUB_VAL
499   if [ -d $NEMO_VALIDATION_REF/$SETTE_SUB_VAL ] ; then
500    while true; do
501        read -p "$NEMO_VALIDATION_REF/$SETTE_SUB_VAL exists. Do you wish to use it as a reference? " yn
502        case $yn in
503            [Yy]* ) export $NEMO_VALIDATION_REF/$SETTE_SUB_VAL; break;;
504            [Nn]* ) exit;;
505            * ) echo "Please answer yes or no.";;
506        esac
507    done
508   fi
509  fi
510  NEMO_VALID=${NEMO_VALIDATION_DIR}
511  NEMO_VALID_REF=${NEMO_VALIDATION_REF}
512#
513  if [ ! -d $NEMO_VALID ]; then
514    echo "$NEMO_VALID validation directory not found"
515    exit
516  fi
517#
518#
519# Show current revision tag and branch name
520#
521echo ""
522lastchange=`${SVN_CMD} info ${MAIN_DIR} | grep 'Last Changed Rev' | awk '{print $NF}'`
523revision=`${SVN_CMD} info ${MAIN_DIR} | grep 'Revision' | awk '{print $NF}'`
524branchname=`${SVN_CMD} info ${MAIN_DIR} | grep ^URL | awk -F ipsl/forge/projets/nemo/svn/ '{print $NF}'`
525echo "Current code is : $branchname @ r$revision  ( last change @ r$lastchange )"
526[ `${SVN_CMD} status -q ${MAIN_DIR}/{cfgs,tests,src} | wc -l` -ge 1 ] && lastchange=${lastchange}+
527
528# by default use the current lastchanged revision
529lastchange=${rev:-$lastchange}
530
531echo ""
532echo "SETTE validation report generated for : "
533echo ""
534echo "       $branchname @ r$lastchange (last changed revision)"
535echo ""
536echo "       on $COMPILER arch file"
537echo ""
538
539#
540# The script also needs the date or revision tag. Currently this is taken from the latest sub-directory found in each directory
541# 
542for pass in  $RPT_PASSES 
543do
544#
545 if [ $pass == 0 ]; then
546   echo "" 
547   echo "!!---------------1st pass------------------!!"
548 fi
549 if [ $pass == 1 ]; then
550    echo ""
551    echo "!!---------------2nd pass------------------!!"
552 fi
553#
554
555# Restartability test
556 echo ""
557 echo "   !----restart----!   "
558 for restart_test in GYRE_PISCES ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 ORCA2_SAS_ICE AGRIF_DEMO WED025 ISOMIP+ OVERFLOW LOCK_EXCHANGE VORTEX ICE_AGRIF SWG
559 do
560   resttest $NEMO_VALID $restart_test $pass
561 done
562#
563# Reproducibility tests
564 echo ""
565 echo "   !----repro----!   "
566 for repro_test in GYRE_PISCES ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 ORCA2_SAS_ICE ORCA2_ICE_OBS AGRIF_DEMO WED025 ISOMIP+ VORTEX ICE_AGRIF SWG
567 do
568   reprotest $NEMO_VALID $repro_test $pass
569 done
570
571# AGRIF special check to ensure results are unchanged with and without key_agrif
572 echo ""
573 echo "   !----agrif check----!   "
574 dir1=WAGRIF_DEMO_NOAGRIF_ST
575 dir2=WAGRIF_DEMO_ST
576 identictest $NEMO_VALID $dir1 $dir2 $pass 
577#
578# before/after tests
579 if [ $lastchange == 'old' ] ; then
580    echo ""
581    echo "   !---- 'old' specified as revision => no comparison with reference results ----!   "
582    echo ""
583 else
584   echo ""
585   echo "   !----result comparison check----!   "
586   if [ $NEMO_VALID_REF != "/path/to/reference/sette/results" ]; then
587     echo ''
588     echo 'check result differences between :'
589     echo "VALID directory : $NEMO_VALID at rev $lastchange"
590     echo 'and'
591     echo "REFERENCE directory : $NEMO_VALID_REF at rev $NEMO_REV_REF"
592     echo ''
593     checklist=(GYRE_PISCES ORCA2_ICE_PISCES ORCA2_OFF_PISCES AMM12 ORCA2_SAS_ICE AGRIF_DEMO WED025 ISOMIP+ VORTEX ICE_AGRIF OVERFLOW LOCK_EXCHANGE SWG)
594     for repro_test in ${checklist[@]}
595     do
596       runcmpres $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $pass
597     done
598     echo ''
599     echo 'Report timing differences between REFERENCE and VALID (if available) :'
600     for repro_test in ${checklist[@]}
601     do
602       runcmptim $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $pass
603     done
604   else
605     echo ''
606     echo ' No path for comparison specified. Result are not compare with any other revision. '
607     echo ' To do it please fill NEMO_VALID_REF and NEMO_REV_REF in param.cfg. '
608     echo ''
609   fi
610 fi
611done
612#
613exit
Note: See TracBrowser for help on using the repository browser.