New URL for NEMO forge!   http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.
sette_rpt.sh in NEMO/branches/2019/fix_sette_ticket2239 – NEMO

source: NEMO/branches/2019/fix_sette_ticket2239/sette_rpt.sh @ 10698

Last change on this file since 10698 was 10698, checked in by mathiot, 5 years ago

check exit code from NEMO (do not copy anything if it failed on exit code other than 0/123456/999) + rename run.stat/tracer.stat/ocean.output to *_old in the current validation dir (ticket #2239)

  • Property svn:executable set to *
File size: 17.2 KB
Line 
1#!/bin/bash -f
2# set -vx
3# simple SETTE report generator.
4#
5# This version should be run in the SETTE directory.
6# The machine name will be picked up from the sette.sh script but the location of the
7# validation directory needs to be set here (currently assumed to reside in the ../cfgs directory)
8#
9#########################################################################################
10######################### Start of function definitions #################################
11##
12
13function restfile() {
14# Rebuild ice restart for SAS CONFIG, and restartability checks. Expects LONG and SHORT run directories.
15# For Stand Alone Surface configuration ocean is not running, just run ice model; so no outputs ocean files.
16# Compares LONG rebuild restart ice file with equivalent entry from the SHORT rebuild restart ice file.
17#
18# check nco module loaded, and load it if not
19if [ ! $( echo $LOADEDMODULES | grep cdo ) ]; then module load cdo >& /dev/null ; fi
20#
21  vdir=$1
22  nam=$2
23  pass=$3
24
25#
26  if [ -d $vdir/$nam ]; then
27    dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l `
28    dorv=`echo $dorv | sed -e 's:.*/::'`
29    rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | tail -2l | head -1 `
30    rep2=`ls -1rt $vdir/$nam/$mach/$dorv/ | tail -1l`
31    cd ${SAS_RESTART_DIR}/LONG
32    #SF add here compilation of rebuild_tools to rebuild restart files, and add comparison of restart files
33    cd ${TOOLS_DIR}
34    ./maketools -n REBUILD_NEMO -m ${mach} > /dev/null 2>&1
35    cd ${TOOLS_DIR}/REBUILD_NEMO
36    #SF echo "REBUILD LONG restart SAS files, without standard output"
37    ./rebuild_nemo -t 4 ../../cfgs/ORCA2_SAS_ICE_ST/LONG/SAS_00000240_restart_ice  $NPROC > /dev/null 2>&1
38    #SF echo "REBUILD SHORT restart SAS files, without standard output"
39    ./rebuild_nemo -t 4 ../../cfgs/ORCA2_SAS_ICE_ST/SHORT/SAS_00000240_restart_ice $NPROC >&-
40    cd ${SAS_RESTART_DIR}/LONG
41    #SF echo "COPY rebuild restart files"
42    cp SAS_00000240_restart_ice.nc $vdir/$nam/$mach/$dorv/LONG/.
43    cp ../SHORT/SAS_00000240_restart_ice.nc $vdir/$nam/$mach/$dorv/SHORT/.
44
45    f1o=$vdir/$nam/$mach/$dorv/LONG/SAS_00000240_restart_ice.nc
46    f2o=$vdir/$nam/$mach/$dorv/SHORT/SAS_00000240_restart_ice.nc
47    if  [ ! -f $f1o ] &&  [ ! -f $f2o ] ; then
48      printf "%-27s %s\n" $nam " REBUILD SAS restart ice DOES NOT exists; incomplete test";
49      return;
50    fi
51    #
52    done_oce=0
53    #
54  if  [  -f $f1o ] && [  -f $f2o ]; then
55## Compare the two netcdf files
56    cdo diffn $f1o $f2o > cdo_diff.out 2> /dev/null
57## Identical if cdo_diff.out exists but has zero size
58    if [ ! -s cdo_diff.out ]; then
59       difi=0
60    else
61## Identical if first character of $dif ==0
62       dif=$( grep -om1 '[0-9]* of [0-9]* records differ' cdo_diff.out )
63# difi contains the first character of summary of cdo dif. if = 0, then 0 record differ between the 2 files   
64       if [ -n "$dif" ]; then
65           difi=`echo $dif | cut -c -1`
66       fi
67    fi
68    \rm cdo_diff.out
69
70    if [ $difi == 0 ]; then   
71       if [ $pass == 0 ]; then
72         printf "%-27s %s %s\n" $nam  " ice restarts are IDENTICAL  passed : " $dorv
73       fi
74    else
75       printf "%-27s %s %s\n" $nam  " ice restarts are DIFFERENT  FAILED : " $dorv 
76        #
77   # Offer view of differences on the second pass
78   #
79        if [ $pass == 1 ]; then
80          echo "<return> to view restart_ice.nc differences"
81          read y
82          cdo -diffv $f1o $f2o
83          done_oce=1
84          #echo "<return> to continue"
85          #read y
86        fi
87    fi
88  else
89      printf "%-27s %s\n" $nam " incomplete test";
90      return;
91  fi
92#
93fi
94}
95
96function resttest() { 
97#
98# Restartability checks. Expects LONG and SHORT run directories
99# Compares end of LONG stat files with equivalent entries from the SHORT stat files.
100#
101  vdir=$1
102  nam=$2
103  pass=$3
104#
105  if [ -d $vdir/$nam ]; then
106    dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l `
107    dorv=`echo $dorv | sed -e 's:.*/::'`
108    f1o=$vdir/$nam/$mach/$dorv/LONG/ocean.output
109    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
110    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
111    f2o=$vdir/$nam/$mach/$dorv/SHORT/ocean.output
112    f2s=$vdir/$nam/$mach/$dorv/SHORT/run.stat
113    f2t=$vdir/$nam/$mach/$dorv/SHORT/tracer.stat
114
115    if  [ ! -f $f1s ] &&  [ ! -f $f1t ] ; then
116      printf "%-27s %s\n" $nam " incomplete test";
117      return;
118    fi
119    if  [ ! -f $f2s ] &&  [ ! -f $f2t ] ; then
120      printf "%-27s %s\n" $nam " incomplete test";
121      return;
122    fi
123#
124    done_oce=0
125
126    if  [  -f $f1s ] && [  -f $f2s ]; then
127      nl=(`wc -l $f2s`)
128      tail -${nl[0]} $f1s > f1.tmp$$
129      cmp -s f1.tmp$$ $f2s
130      if [ $? == 0 ]; then
131        if [ $pass == 0 ]; then
132          printf "%-27s %s %s\n" $nam  " run.stat    restartability  passed : " $dorv
133        fi
134      else
135        printf "%-27s %s %s\n" $nam  " run.stat    restartability  FAILED : " $dorv 
136#
137# Offer view of differences on the second pass
138#
139        if [ $pass == 1 ]; then
140          echo "<return> to view run.stat differences"
141          read y
142          sdiff f1.tmp$$ $f2s
143          echo "<return> to view ocean.output differences"
144          read y
145          sdiff $f1o $f2o | grep "|"
146          done_oce=1
147          echo "<return> to continue"
148          read y
149        fi
150      fi
151    fi
152#
153# Check tracer.stat files (if they exist)
154#
155    if  [  -f $f1t ] && [  -f $f2t ]; then
156      nl=(`wc -l $f2t`)
157      tail -${nl[0]} $f1t > f1.tmp$$
158      cmp -s f1.tmp$$ $f2t
159      if [ $? == 0 ]; then
160        if [ $pass == 0 ]; then
161          printf "%-27s %s %s\n" $nam  " tracer.stat restartability  passed : " $dorv
162        fi
163      else
164        printf "%-27s %s %s\n" $nam  " tracer.stat restartability  FAILED : " $dorv 
165#
166# Offer view of differences on the second pass
167#
168        if [ $pass == 1 ]; then
169          echo "<return> to view tracer.stat differences"
170          read y
171          sdiff f1.tmp$$ $f2t
172#
173# Only offer ocean.output view if it has not been viewed previously
174#
175          if [ $done_oce == 0 ]; then
176            echo "<return> to view ocean.output differences"
177            read y
178            sdiff $f1o $f2o | grep "|"
179          fi
180          echo "<return> to continue"
181          read y
182        fi
183      fi
184    fi
185    rm f1.tmp$$
186  fi
187}
188
189function reprotest(){
190#
191# Reproducibility checks. Expects REPRO_N_M and REPRO_I_J run directories
192# Compares end of stat files from each
193#
194  vdir=$1
195  nam=$2
196  pass=$3
197#
198  if [ -d $vdir/$nam ]; then
199    dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l `
200    dorv=`echo $dorv | sed -e 's:.*/::'`
201    rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -2l | head -1 `
202    rep2=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -1l`
203    f1o=$vdir/$nam/$mach/$dorv/$rep1/ocean.output
204    f1s=$vdir/$nam/$mach/$dorv/$rep1/run.stat
205    f1t=$vdir/$nam/$mach/$dorv/$rep1/tracer.stat
206    f2o=$vdir/$nam/$mach/$dorv/$rep2/ocean.output
207    f2s=$vdir/$nam/$mach/$dorv/$rep2/run.stat
208    f2t=$vdir/$nam/$mach/$dorv/$rep2/tracer.stat
209
210    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
211      printf "%-27s %s\n" $nam " incomplete test";
212      return;
213    fi
214    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
215      printf "%-27s %s\n" $nam " incomplete test";
216      return;
217    fi
218#
219    done_oce=0
220
221    if  [ -f $f1s ] && [ -f $f2s ] ; then
222      cmp -s $f1s $f2s
223      if [ $? == 0 ]; then
224        if [ $pass == 0 ]; then
225          printf "%-27s %s %s\n" $nam  " run.stat    reproducibility passed : " $dorv
226        fi
227      else
228        printf "%-27s %s %s\n" $nam  " run.stat    reproducibility FAILED : " $dorv 
229#
230# Offer view of differences on the second pass
231#
232        if [ $pass == 1 ]; then
233          echo "<return> to view run.stat differences"
234          read y
235          sdiff $f1s $f2s
236          echo "<return> to view ocean.output differences"
237          read y
238          sdiff $f1o $f2o | grep "|"
239          done_oce=1
240          echo "<return> to continue"
241          read y
242        fi
243      fi
244    fi
245#
246# Check tracer.stat files (if they exist)
247#
248    if  [ -f $f1t ] && [ -f $f2t ] ; then
249      cmp -s $f1t $f2t
250      if [ $? == 0 ]; then
251        if [ $pass == 0 ]; then           printf "%-27s %s %s\n" $nam  " tracer.stat reproducibility passed : " $dorv
252        fi
253      else
254        printf "%-27s %s %s\n" $nam  " tracer.stat reproducibility  FAILED : " $dorv
255#
256# Offer view of differences on the second pass
257#
258        if [ $pass == 1 ]; then
259          echo "<return> to view tracer.stat differences"
260          read y
261          sdiff $f1t $f2t
262#
263# Only offer ocean.output view if it has not been viewed previously
264#
265          if [ $done_oce == 0 ]; then
266            echo "<return> to view ocean.output differences"
267            read y
268            sdiff $f1o $f2o | grep "|"
269          fi
270          echo "<return> to continue"
271          read y
272        fi
273      fi
274    fi
275  fi
276}
277function runcmpres(){
278#
279# compare *.stat file with reference file from a previous sette test or previous version
280# store in NEMO_VALID_REF at revision NEMO_REV_REF
281# Compares end of stat files from each
282#
283  vdir=$1
284  nam=$2
285  vdirref=$3
286  dorvref=$4
287  pass=$5
288#
289# check if reference directory is present
290  if [ ! -d $vdirref/$nam/$mach/$dorvref ]; then
291    printf "%s\n" " REFERENCE directory $vdirref/$nam/$mach/$dorvref is MISSING"
292    return
293  fi
294#
295  if [ -d $vdir/$nam ]; then
296    dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l `
297    dorv=`echo $dorv | sed -e 's:.*/::'`
298    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat
299    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat
300    f2s=$vdirref/$nam/$mach/$dorvref/LONG/run.stat
301    f2t=$vdirref/$nam/$mach/$dorvref/LONG/tracer.stat
302    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then
303      printf "%-20s %s\n" $nam " incomplete test";
304      return;
305    fi
306    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then
307      printf "%-20s %s\n" $nam " incomplete test";
308      return;
309    fi
310#
311    done_oce=0
312
313    if  [ -f $f1s ] && [ -f $f2s ] ; then
314      cmp -s $f1s $f2s
315      if [ $? == 0 ]; then
316        if [ $pass == 0 ]; then
317          printf "%-20s %s %s\n" $nam  " run.stat    before/after are similar "
318        fi
319      else
320        printf "%-20s %s %s\n" $nam  " run.stat    before/after are DIFFERENT "
321#
322# Offer view of differences on the second pass
323#
324        if [ $pass == 1 ]; then
325          echo "<return> to view run.stat differences"
326          read y
327          sdiff f1.tmp$$ $f2s
328          done_oce=1
329          echo "<return> to continue"
330          read y
331        fi
332      fi
333    fi
334    # Check tracer.stat files (if they exist)
335#
336    if  [ -f $f1t ] && [ -f $f2t ] ; then
337      cmp -s $f1t $f2t
338      if [ $? == 0 ]; then
339        if [ $pass == 0 ]; then         
340          printf "%-20s %s %s\n" $nam  " tracer.stat before/after are similar "
341        fi
342      else
343        printf "%-20s %s %s\n" $nam  " tracer.stat before/after are DIFFERENT "
344#
345# Offer view of differences on the second pass
346#
347        if [ $pass == 1 ]; then
348          echo "<return> to view tracer.stat differences"
349          read y
350          sdiff $f1t $f2t
351        fi
352      fi
353    fi
354  fi
355}
356
357function runtest(){
358#
359# Run checks.
360# Check presence of E R R O R in ocean.output from each
361#
362  vdir=$1
363  nam=$2
364  pass=$3
365#
366  if [ -d $vdir/$nam ]; then
367    dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l `
368    dorv=`echo $dorv | sed -e 's:.*/::'`
369    rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep LONG | head -1 `
370    if  [  -z $rep1 ] ; then
371      # Not all configurations test restartability (LONG/SHORT);
372      # check for a reproducibility directory instead (REPRO)
373      rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | head -1 `
374    fi
375    f1o=$vdir/$nam/$mach/$dorv/$rep1/ocean.output
376    if  [ ! -f $f1o ] ; then
377      if [ $pass == 0 ]; then printf "%-27s %s %s %s\n" $nam " ocean.output               MISSING : " $dorv " ( run did not start / finish correctly ) " ; fi
378      return;
379    else
380      nerr=`grep 'E R R O R' $f1o | wc -l`
381      if [[ $nerr > 0 ]]; then
382        printf "%-27s %s %s %s\n" $nam " run                         FAILED : " $dorv " ( E R R O R in ocean.output) " 
383        if [ $pass == 1 ]; then
384          echo "<return> to view end of ocean.output"
385          read y
386          tail -100 $f1o
387          echo ''
388          echo "full ocean.output available here: $f1o"
389        fi
390      fi
391    fi
392  else
393    if [ $pass == 0 ]; then printf "%-27s %s %s %s\n" $nam  " directory                  MISSING : " $dorv " ( run did not start / finish correctly ) "; fi
394  fi
395}
396
397function identictest(){
398#
399#  checks AGRIF does not corrupe results with no AGRIF zoom. Expects ORCA2AGUL/AGRIFNOZ and ORCA2AGUL_NAGR/AGRIFNO  run directories
400# Compares solver.stat files for each
401#
402  vdir=$1
403  dir1=$2
404  dir2=$3
405  pass=$4
406#
407  if [ -d $vdir/$dir1 ] && [ -d $vdir/$dir2 ]; then
408    dorv1=`ls -1rt $vdir/$dir1/$mach/ | tail -1l `
409    dorv1=`echo $dorv1 | sed -e 's:.*/::'`
410    dorv2=`ls -1rt $vdir/$dir2/$mach/ | tail -1l `
411    dorv2=`echo $dorv2 | sed -e 's:.*/::'`
412
413    rep1=`ls -1rt $vdir/$dir1/$mach/$dorv1/ |  tail -1l`
414#clem    rep2=`ls -1rt $vdir/$dir2/$mach/$dorv2/ |  tail -1l`
415    rep2=`ls -1rt $vdir/$dir1/$mach/$dorv1/ |  tail -1l`
416    f1s=$vdir/$dir1/$mach/$dorv1/$rep1/run.stat
417    f2s=$vdir/$dir2/$mach/$dorv2/$rep2/run.stat
418
419    if  [ ! -f $f1s ] && [ ! -f $f2s ] ; then
420      printf "%-27s %s\n" $dir1 $dir2 " incomplete test";
421      return;
422    fi
423#
424    done_oce=0
425
426    if  [ -f $f1s ] && [ -f $f2s ] ; then
427      cmp -s $f1s $f2s
428      if [ $? == 0 ]; then
429        if [ $pass == 0 ]; then
430          printf "%-5s %s %-5s %s %s %s\n" $rep1 "AGRIF vs" $rep2 "NOAGRIF run.stat    unchanged  -    passed : " $dorv1 $dorv2
431        fi
432      else
433        printf "%-5s %s %-5s %s %s %s\n" $rep1 "AGRIF vs" $rep2 "NOAGRIF run.stat    changed  -     FAILED : " $dorv1 $dorv2
434#
435# Offer view of differences on the second pass
436#
437        if [ $pass == 1 ]; then
438          echo "<return> to view run.stat differences"
439          read y
440          sdiff $f1s $f2s
441          done_oce=1
442          echo "<return> to continue"
443          read y
444        fi
445      fi
446    fi
447  else
448    printf "%-27s %s\n" $dir1 $dir2 " incomplete test";
449  fi
450}
451########################### END of function definitions #################################
452##                                                                                     ##
453##    Main script                                                                      ##
454##                                                                                     ##
455#########################################################################################
456#
457# LOAD param variable (COMPILER, NEMO_VALIDATION_DIR, SVN_CMD)
458  . ./param.cfg
459
460  mach=${COMPILER}
461  NEMO_VALID=${NEMO_VALIDATION_DIR}
462  NEMO_VALID_REF=${NEMO_VALIDATION_REF}
463  LANG=en_US
464#
465  if [ ! -d $NEMO_VALID ]; then
466    echo "$NEMO_VALID validation directory not found"
467    exit
468  fi
469#
470# Directory to run the tests
471  SETTE_DIR=$(cd $(dirname "$0"); pwd)
472  MAIN_DIR=${SETTE_DIR}/..
473  CONFIG_DIR0=${MAIN_DIR}/cfgs
474  TOOLS_DIR=${MAIN_DIR}/tools
475  NPROC=32
476  SAS_RESTART_DIR=${CONFIG_DIR0}/ORCA2_SAS_ICE_ST
477#
478# Show current revision tag and branch name
479#
480echo ""
481lastchange=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep 'Last Changed Rev' | awk '{print $NF}'`
482revision=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep 'Revision' | awk '{print $NF}'`
483branchname=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep ^URL | awk -F ipsl/forge/projets/nemo/svn/ '{print $NF}'`
484echo "SETTE validation report : $branchname @ r$revision  ( last change @ r$lastchange )"
485#
486# The script also needs the date or revision tag. Currently this is taken from the latest sub-directory found in each directory
487
488for pass in  0 1
489do
490#
491 if [ $pass == 0 ]; then
492   echo "" 
493   echo "!!---------------1st pass------------------!!"
494 fi
495 if [ $pass == 1 ]; then
496    echo ""
497    echo "!!---------------2nd pass------------------!!"
498 fi
499#
500
501# Rebuild and restartability test for SAS
502# clem: not needed anymore
503# for restart_file in WORCA2_SAS_ICE_ST
504# do
505#   restfile $NEMO_VALID $restart_file $pass
506# done
507#
508# Restartability test
509 echo ""
510 echo "   !----restart----!   "
511 for restart_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WOVERFLOW_ST WLOCK_EXCHANGE_ST WVORTEX_ST WICE_AGRIF_ST
512 do
513   resttest $NEMO_VALID $restart_test $pass
514   runtest $NEMO_VALID $restart_test $pass
515 done
516#
517# Reproducibility tests
518 echo ""
519 echo "   !----repro----!   "
520 for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WORCA2_SAS_ICE_ST WORCA2_ICE_OBS_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WVORTEX_ST WICE_AGRIF_ST
521 do
522   reprotest $NEMO_VALID $repro_test $pass
523   runtest $NEMO_VALID $repro_test $pass
524 done
525
526# AGRIF special check to ensure results are unchanged with and without key_agrif
527 echo ""
528 echo "   !----agrif check----!   "
529 dir1=WAGRIF_DEMO_NOAGRIF_ST
530 dir2=WAGRIF_DEMO_ST
531 identictest $NEMO_VALID $dir1 $dir2 $pass 
532#
533# before/after tests
534#
535 if [ $NEMO_VALID_REF != "/path/to/reference/sette/results" ]; then
536   echo ''
537   echo 'check result differences between :'
538   echo "$NEMO_VALID at rev $revision"
539   echo 'and'
540   echo "$NEMO_VALID_REF at rev $NEMO_REV_REF"
541   for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WISOMIP_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WVORTEX_ST WICE_AGRIF_ST
542   do
543     runcmpres $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $pass
544   done
545 else
546   echo ''
547   echo 'Result are not compare with any other revision. To do it please fill NEMO_VALID_REF and NEMO_REV_REF in param.cfg'
548   echo ''
549 fi
550
551done
552#
553exit
Note: See TracBrowser for help on using the repository browser.