New URL for NEMO forge!   http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.
sette_rpt.sh in NEMO/branches/2019/fix_sette_ticket2239 – NEMO

source: NEMO/branches/2019/fix_sette_ticket2239/sette_rpt.sh @ 10708

Last change on this file since 10708 was 10708, checked in by mathiot, 5 years ago

add possibility to generate report on older revision + no more mix of various revision in the report output (ticket #2239)

  • Property svn:executable set to *
File size: 18.0 KB
Line 
1#!/bin/bash -f
2# set -vx
3# simple SETTE report generator.
4#
5# This version should be run in the SETTE directory.
6# The machine name will be picked up from the sette.sh script but the location of the
7# validation directory needs to be set here (currently assumed to reside in the ../cfgs directory)
8#
9#########################################################################################
10######################### Start of function definitions #################################
11##
12
13function restfile() {
14# Rebuild ice restart for SAS CONFIG, and restartability checks. Expects LONG and SHORT run directories.
15# For Stand Alone Surface configuration ocean is not running, just run ice model; so no outputs ocean files.
16# Compares LONG rebuild restart ice file with equivalent entry from the SHORT rebuild restart ice file.
17#
18# check nco module loaded, and load it if not
19if [ ! $( echo $LOADEDMODULES | grep cdo ) ]; then module load cdo >& /dev/null ; fi
20#
21  vdir=$1
22  nam=$2
23  pass=$3
24
25#
26#
27# check if directory is here
28  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
29    printf "%-27s %s %s %s\n" $nam  " directory                  MISSING : " $dorv " ( run did not start / finish correctly ) "
30    return
31  fi
32
33  if [ -d $vdir/$nam/$mach/$dorv/ ]; then
34    rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | tail -2l | head -1 `
35    rep2=`ls -1rt $vdir/$nam/$mach/$dorv/ | tail -1l`
36    cd ${SAS_RESTART_DIR}/LONG
37    #SF add here compilation of rebuild_tools to rebuild restart files, and add comparison of restart files
38    cd ${TOOLS_DIR}
39    ./maketools -n REBUILD_NEMO -m ${mach} > /dev/null 2>&1
40    cd ${TOOLS_DIR}/REBUILD_NEMO
41    #SF echo "REBUILD LONG restart SAS files, without standard output"
42    ./rebuild_nemo -t 4 ../../cfgs/ORCA2_SAS_ICE_ST/LONG/SAS_00000240_restart_ice  $NPROC > /dev/null 2>&1
43    #SF echo "REBUILD SHORT restart SAS files, without standard output"
44    ./rebuild_nemo -t 4 ../../cfgs/ORCA2_SAS_ICE_ST/SHORT/SAS_00000240_restart_ice $NPROC >&-
45    cd ${SAS_RESTART_DIR}/LONG
46    #SF echo "COPY rebuild restart files"
47    cp SAS_00000240_restart_ice.nc $vdir/$nam/$mach/$dorv/LONG/.
48    cp ../SHORT/SAS_00000240_restart_ice.nc $vdir/$nam/$mach/$dorv/SHORT/.
49
50    f1o=$vdir/$nam/$mach/$dorv/LONG/SAS_00000240_restart_ice.nc
51    f2o=$vdir/$nam/$mach/$dorv/SHORT/SAS_00000240_restart_ice.nc
52    if  [ ! -f $f1o ] &&  [ ! -f $f2o ] ; then
53      printf "%-27s %s\n" $nam " REBUILD SAS restart ice DOES NOT exists; incomplete test";
54      return;
55    fi
56    #
57    done_oce=0
58    #
59  if  [  -f $f1o ] && [  -f $f2o ]; then
60## Compare the two netcdf files
61    cdo diffn $f1o $f2o > cdo_diff.out 2> /dev/null
62## Identical if cdo_diff.out exists but has zero size
63    if [ ! -s cdo_diff.out ]; then
64       difi=0
65    else
66## Identical if first character of $dif ==0
67       dif=$( grep -om1 '[0-9]* of [0-9]* records differ' cdo_diff.out )
68# difi contains the first character of summary of cdo dif. if = 0, then 0 record differ between the 2 files   
69       if [ -n "$dif" ]; then
70           difi=`echo $dif | cut -c -1`
71       fi
72    fi
73    \rm cdo_diff.out
74
75    if [ $difi == 0 ]; then   
76       if [ $pass == 0 ]; then
77         printf "%-27s %s %s\n" $nam  " ice restarts are IDENTICAL  passed : " $dorv
78       fi
79    else
80       printf "%-27s %s %s\n" $nam  " ice restarts are DIFFERENT  FAILED : " $dorv 
81        #
82   # Offer view of differences on the second pass
83   #
84        if [ $pass == 1 ]; then
85          echo "<return> to view restart_ice.nc differences"
86          read y
87          cdo -diffv $f1o $f2o
88          done_oce=1
89          #echo "<return> to continue"
90          #read y
91        fi
92    fi
93  else
94      printf "%-27s %s\n" $nam " incomplete test";
95      return;
96  fi
97#
98fi
99}
100
101function resttest() { 
102#
103# Restartability checks. Expects LONG and SHORT run directories
104# Compares end of LONG stat files with equivalent entries from the SHORT stat files.
105#
106  vdir=$1
107  nam=$2
108  pass=$3
109#
110#
111# check if directory is here
112  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
113    printf "%-27s %s %s %s\n" $nam  " directory                  MISSING : " $dorv " ( run did not start / finish correctly ) "
114    return
115  fi
116
117  if [ -d $vdir/$nam/$mach/$dorv ]; then
118    f1o=$vdir/$nam/$mach/$dorv/LONG/ocean.output
119    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
120    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
121    f2o=$vdir/$nam/$mach/$dorv/SHORT/ocean.output
122    f2s=$vdir/$nam/$mach/$dorv/SHORT/run.stat
123    f2t=$vdir/$nam/$mach/$dorv/SHORT/tracer.stat
124
125    if  [ ! -f $f1s ] &&  [ ! -f $f1t ] ; then
126      printf "%-27s %s\n" $nam " incomplete test";
127      return;
128    fi
129    if  [ ! -f $f2s ] &&  [ ! -f $f2t ] ; then
130      printf "%-27s %s\n" $nam " incomplete test";
131      return;
132    fi
133#
134    done_oce=0
135
136    if  [  -f $f1s ] && [  -f $f2s ]; then
137      nl=(`wc -l $f2s`)
138      tail -${nl[0]} $f1s > f1.tmp$$
139      cmp -s f1.tmp$$ $f2s
140      if [ $? == 0 ]; then
141        if [ $pass == 0 ]; then
142          printf "%-27s %s %s\n" $nam  " run.stat    restartability  passed : " $dorv
143        fi
144      else
145        printf "%-27s %s %s\n" $nam  " run.stat    restartability  FAILED : " $dorv 
146#
147# Offer view of differences on the second pass
148#
149        if [ $pass == 1 ]; then
150          echo "<return> to view run.stat differences"
151          read y
152          sdiff f1.tmp$$ $f2s
153          echo "<return> to view ocean.output differences"
154          read y
155          sdiff $f1o $f2o | grep "|"
156          done_oce=1
157          echo "<return> to continue"
158          read y
159        fi
160      fi
161    fi
162#
163# Check tracer.stat files (if they exist)
164#
165    if  [  -f $f1t ] && [  -f $f2t ]; then
166      nl=(`wc -l $f2t`)
167      tail -${nl[0]} $f1t > f1.tmp$$
168      cmp -s f1.tmp$$ $f2t
169      if [ $? == 0 ]; then
170        if [ $pass == 0 ]; then
171          printf "%-27s %s %s\n" $nam  " tracer.stat restartability  passed : " $dorv
172        fi
173      else
174        printf "%-27s %s %s\n" $nam  " tracer.stat restartability  FAILED : " $dorv 
175#
176# Offer view of differences on the second pass
177#
178        if [ $pass == 1 ]; then
179          echo "<return> to view tracer.stat differences"
180          read y
181          sdiff f1.tmp$$ $f2t
182#
183# Only offer ocean.output view if it has not been viewed previously
184#
185          if [ $done_oce == 0 ]; then
186            echo "<return> to view ocean.output differences"
187            read y
188            sdiff $f1o $f2o | grep "|"
189          fi
190          echo "<return> to continue"
191          read y
192        fi
193      fi
194    fi
195    rm f1.tmp$$
196  fi
197}
198
199function reprotest(){
200#
201# Reproducibility checks. Expects REPRO_N_M and REPRO_I_J run directories
202# Compares end of stat files from each
203#
204  vdir=$1
205  nam=$2
206  pass=$3
207#
208# check if directory is here
209  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
210    printf "%-27s %s %s %s\n" $nam  " directory                  MISSING : " $dorv " ( run did not start / finish correctly ) "
211    return
212  fi
213#
214  if [ -d $vdir/$nam/$mach/$dorv ]; then
215    rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -2l | head -1 `
216    rep2=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -1l`
217    f1o=$vdir/$nam/$mach/$dorv/$rep1/ocean.output
218    f1s=$vdir/$nam/$mach/$dorv/$rep1/run.stat
219    f1t=$vdir/$nam/$mach/$dorv/$rep1/tracer.stat
220    f2o=$vdir/$nam/$mach/$dorv/$rep2/ocean.output
221    f2s=$vdir/$nam/$mach/$dorv/$rep2/run.stat
222    f2t=$vdir/$nam/$mach/$dorv/$rep2/tracer.stat
223
224    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
225      printf "%-27s %s\n" $nam " incomplete test";
226      return;
227    fi
228    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
229      printf "%-27s %s\n" $nam " incomplete test";
230      return;
231    fi
232#
233    done_oce=0
234
235    if  [ -f $f1s ] && [ -f $f2s ] ; then
236      cmp -s $f1s $f2s
237      if [ $? == 0 ]; then
238        if [ $pass == 0 ]; then
239          printf "%-27s %s %s\n" $nam  " run.stat    reproducibility passed : " $dorv
240        fi
241      else
242        printf "%-27s %s %s\n" $nam  " run.stat    reproducibility FAILED : " $dorv 
243#
244# Offer view of differences on the second pass
245#
246        if [ $pass == 1 ]; then
247          echo "<return> to view run.stat differences"
248          read y
249          sdiff $f1s $f2s
250          echo "<return> to view ocean.output differences"
251          read y
252          sdiff $f1o $f2o | grep "|"
253          done_oce=1
254          echo "<return> to continue"
255          read y
256        fi
257      fi
258    fi
259#
260# Check tracer.stat files (if they exist)
261#
262    if  [ -f $f1t ] && [ -f $f2t ] ; then
263      cmp -s $f1t $f2t
264      if [ $? == 0 ]; then
265        if [ $pass == 0 ]; then           printf "%-27s %s %s\n" $nam  " tracer.stat reproducibility passed : " $dorv
266        fi
267      else
268        printf "%-27s %s %s\n" $nam  " tracer.stat reproducibility FAILED : " $dorv
269#
270# Offer view of differences on the second pass
271#
272        if [ $pass == 1 ]; then
273          echo "<return> to view tracer.stat differences"
274          read y
275          sdiff $f1t $f2t
276#
277# Only offer ocean.output view if it has not been viewed previously
278#
279          if [ $done_oce == 0 ]; then
280            echo "<return> to view ocean.output differences"
281            read y
282            sdiff $f1o $f2o | grep "|"
283          fi
284          echo "<return> to continue"
285          read y
286        fi
287      fi
288    fi
289  fi
290}
291function runcmpres(){
292#
293# compare *.stat file with reference file from a previous sette test or previous version
294# store in NEMO_VALID_REF at revision NEMO_REV_REF
295# Compares end of stat files from each
296#
297  vdir=$1
298  nam=$2
299  vdirref=$3
300  dorvref=$4
301  pass=$5
302#
303# check if reference directory is present
304  if [ ! -d $vdirref/$nam/$mach/$dorvref ]; then
305    printf "%-25s %-27s %s\n" " REFERENCE directory for " $nam " at $dorvref is MISSING"
306    return
307  fi
308  if [ ! -d $vdir/$nam/$mach/$dorv ]; then
309    printf "%-25s %-27s %s\n" " VALID directory for " $nam " at $dorv is MISSING"
310    return
311  fi
312
313#
314  if [ -d $vdir/$nam/$mach/$dorv ]; then
315    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
316    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
317    f2s=$vdirref/$nam/$mach/$dorvref/LONG/run.stat
318    f2t=$vdirref/$nam/$mach/$dorvref/LONG/tracer.stat
319    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
320      printf "%-20s %s\n" $nam " incomplete test";
321      return;
322    fi
323    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
324      printf "%-20s %s\n" $nam " incomplete test";
325      return;
326    fi
327#
328    done_oce=0
329
330    if  [ -f $f1s ] && [ -f $f2s ] ; then
331      cmp -s $f1s $f2s
332      if [ $? == 0 ]; then
333        if [ $pass == 0 ]; then
334          printf "%-20s %s %s\n" $nam  " run.stat    before/after are similar "
335        fi
336      else
337        printf "%-20s %s %s\n" $nam  " run.stat    before/after are DIFFERENT "
338#
339# Offer view of differences on the second pass
340#
341        if [ $pass == 1 ]; then
342          echo "<return> to view run.stat differences"
343          read y
344          sdiff f1.tmp$$ $f2s
345          done_oce=1
346          echo "<return> to continue"
347          read y
348        fi
349      fi
350    fi
351    # Check tracer.stat files (if they exist)
352#
353    if  [ -f $f1t ] && [ -f $f2t ] ; then
354      cmp -s $f1t $f2t
355      if [ $? == 0 ]; then
356        if [ $pass == 0 ]; then         
357          printf "%-20s %s %s\n" $nam  " tracer.stat before/after are similar "
358        fi
359      else
360        printf "%-20s %s %s\n" $nam  " tracer.stat before/after are DIFFERENT "
361#
362# Offer view of differences on the second pass
363#
364        if [ $pass == 1 ]; then
365          echo "<return> to view tracer.stat differences"
366          read y
367          sdiff $f1t $f2t
368        fi
369      fi
370    fi
371  fi
372}
373
374function runtest(){
375#
376# Run checks.
377# Check presence of E R R O R in ocean.output from each
378#
379  vdir=$1
380  nam=$2
381  pass=$3
382#
383# no print needed if the repository is not here (already catch before)
384#
385  if [ -d $vdir/$nam/$mach/$dorv/ ]; then
386    rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep LONG | head -1 `
387    if  [  -z $rep1 ] ; then
388      # Not all configurations test restartability (LONG/SHORT);
389      # check for a reproducibility directory instead (REPRO)
390      rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | head -1 `
391    fi
392    f1o=$vdir/$nam/$mach/$dorv/$rep1/ocean.output
393    if  [ ! -f $f1o ] ; then
394      if [ $pass == 0 ]; then printf "%-27s %s %s %s\n" $nam " ocean.output               MISSING : " $dorv " ( run did not start / finish correctly ) " ; fi
395      return;
396    else
397      nerr=`grep 'E R R O R' $f1o | wc -l`
398      if [[ $nerr > 0 ]]; then
399        printf "%-27s %s %s %s\n" $nam " run                         FAILED : " $dorv " ( E R R O R in ocean.output) " 
400        if [ $pass == 1 ]; then
401          echo "<return> to view end of ocean.output"
402          read y
403          tail -100 $f1o
404          echo ''
405          echo "full ocean.output available here: $f1o"
406        fi
407      fi
408    fi
409  else
410    if [ $pass == 0 ]; then printf "%-27s %s %s %s\n" $nam  " directory                  MISSING : " $dorv " ( run did not start / finish correctly ) "; fi
411  fi
412}
413
414function identictest(){
415#
416#  checks AGRIF does not corrupe results with no AGRIF zoom. Expects ORCA2AGUL/AGRIFNOZ and ORCA2AGUL_NAGR/AGRIFNO  run directories
417# Compares solver.stat files for each
418#
419  vdir=$1
420  dir1=$2
421  dir2=$3
422  pass=$4
423#
424  if [ -d $vdir/$dir1 ] && [ -d $vdir/$dir2 ]; then
425    dorv1=`ls -1rt $vdir/$dir1/$mach/ | tail -1l `
426    dorv1=`echo $dorv1 | sed -e 's:.*/::'`
427    dorv2=`ls -1rt $vdir/$dir2/$mach/ | tail -1l `
428    dorv2=`echo $dorv2 | sed -e 's:.*/::'`
429
430    rep1=`ls -1rt $vdir/$dir1/$mach/$dorv1/ |  tail -1l`
431#clem    rep2=`ls -1rt $vdir/$dir2/$mach/$dorv2/ |  tail -1l`
432    rep2=`ls -1rt $vdir/$dir1/$mach/$dorv1/ |  tail -1l`
433    f1s=$vdir/$dir1/$mach/$dorv1/$rep1/run.stat
434    f2s=$vdir/$dir2/$mach/$dorv2/$rep2/run.stat
435
436    if  [ ! -f $f1s ] && [ ! -f $f2s ] ; then
437      printf "%-27s %s\n" $dir1 $dir2 " incomplete test";
438      return;
439    fi
440#
441    done_oce=0
442
443    if  [ -f $f1s ] && [ -f $f2s ] ; then
444      cmp -s $f1s $f2s
445      if [ $? == 0 ]; then
446        if [ $pass == 0 ]; then
447          printf "%-5s %s %-5s %s %s %s\n" $rep1 "AGRIF vs" $rep2 "NOAGRIF run.stat    unchanged  -    passed : " $dorv1 $dorv2
448        fi
449      else
450        printf "%-5s %s %-5s %s %s %s\n" $rep1 "AGRIF vs" $rep2 "NOAGRIF run.stat    changed  -     FAILED : " $dorv1 $dorv2
451#
452# Offer view of differences on the second pass
453#
454        if [ $pass == 1 ]; then
455          echo "<return> to view run.stat differences"
456          read y
457          sdiff $f1s $f2s
458          done_oce=1
459          echo "<return> to continue"
460          read y
461        fi
462      fi
463    fi
464  else
465    printf "%-27s %s\n" $dir1 $dir2 " incomplete test";
466  fi
467}
468########################### END of function definitions #################################
469##                                                                                     ##
470##    Main script                                                                      ##
471##                                                                                     ##
472#########################################################################################
473#
474# LOAD param variable (COMPILER, NEMO_VALIDATION_DIR, SVN_CMD)
475  . ./param.cfg
476
477  mach=${COMPILER}
478  NEMO_VALID=${NEMO_VALIDATION_DIR}
479  NEMO_VALID_REF=${NEMO_VALIDATION_REF}
480  LANG=en_US
481#
482  if [ ! -d $NEMO_VALID ]; then
483    echo "$NEMO_VALID validation directory not found"
484    exit
485  fi
486#
487# Directory to run the tests
488  SETTE_DIR=$(cd $(dirname "$0"); pwd)
489  MAIN_DIR=${SETTE_DIR}/..
490  CONFIG_DIR0=${MAIN_DIR}/cfgs
491  TOOLS_DIR=${MAIN_DIR}/tools
492  NPROC=32
493  SAS_RESTART_DIR=${CONFIG_DIR0}/ORCA2_SAS_ICE_ST
494#
495# Show current revision tag and branch name
496#
497echo ""
498lastchange=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep 'Last Changed Rev' | awk '{print $NF}'`
499revision=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep 'Revision' | awk '{print $NF}'`
500branchname=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep ^URL | awk -F ipsl/forge/projets/nemo/svn/ '{print $NF}'`
501echo "Current code is : $branchname @ r$revision  ( last change @ r$lastchange )"
502
503# overwrite revision is argument 1 present
504[[ $# -eq 1 ]] && revision=$1
505
506echo ""
507echo "SETTE validation report generated for : $branchname @ r$revision"
508echo ""
509
510dorv=$revision
511
512#
513# The script also needs the date or revision tag. Currently this is taken from the latest sub-directory found in each directory
514
515for pass in  0 1
516do
517#
518 if [ $pass == 0 ]; then
519   echo "" 
520   echo "!!---------------1st pass------------------!!"
521 fi
522 if [ $pass == 1 ]; then
523    echo ""
524    echo "!!---------------2nd pass------------------!!"
525 fi
526#
527
528# Rebuild and restartability test for SAS
529# clem: not needed anymore
530# for restart_file in WORCA2_SAS_ICE_ST
531# do
532#   restfile $NEMO_VALID $restart_file $pass
533# done
534#
535# Restartability test
536 echo ""
537 echo "   !----restart----!   "
538 for restart_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WOVERFLOW_ST WLOCK_EXCHANGE_ST WVORTEX_ST WICE_AGRIF_ST
539 do
540   resttest $NEMO_VALID $restart_test $pass
541   runtest $NEMO_VALID $restart_test $pass
542 done
543#
544# Reproducibility tests
545 echo ""
546 echo "   !----repro----!   "
547 for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WORCA2_ICE_OBS_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WVORTEX_ST WICE_AGRIF_ST
548 do
549   reprotest $NEMO_VALID $repro_test $pass
550   runtest $NEMO_VALID $repro_test $pass
551 done
552
553# AGRIF special check to ensure results are unchanged with and without key_agrif
554 echo ""
555 echo "   !----agrif check----!   "
556 dir1=WAGRIF_DEMO_NOAGRIF_ST
557 dir2=WAGRIF_DEMO_ST
558 identictest $NEMO_VALID $dir1 $dir2 $pass 
559#
560# before/after tests
561 echo ""
562 echo "   !----result comparison check----!   "
563 if [ $NEMO_VALID_REF != "/path/to/reference/sette/results" ]; then
564   echo ''
565   echo 'check result differences between :'
566   echo "VALID directory : $NEMO_VALID at rev $revision"
567   echo 'and'
568   echo "REFERENCE directory : $NEMO_VALID_REF at rev $NEMO_REV_REF"
569   echo ''
570   for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WISOMIP_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WVORTEX_ST WICE_AGRIF_ST
571   do
572     runcmpres $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $pass
573   done
574 else
575   echo ''
576   echo ' No path for comparison specified. Result are not compare with any other revision. '
577   echo ' To do it please fill NEMO_VALID_REF and NEMO_REV_REF in param.cfg. '
578   echo ''
579 fi
580
581done
582#
583exit
Note: See TracBrowser for help on using the repository browser.