New URL for NEMO forge!   http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.
Changeset 11161 for utils/CI/sette/sette_rpt.sh – NEMO

Ignore:
Timestamp:
2019-06-20T18:55:23+02:00 (5 years ago)
Author:
mathiot
Message:

merge fix_sette_ticket2239 : major simplification/improvement of SETTE (see README.rst for all the details), tested with the latest version of the trunk (r11160) (ticket #2239)

File:
1 edited

Legend:

Unmodified
Added
Removed

  • utils/CI/sette/sette_rpt.sh

    r10517 r11161  
    1111## 
    1212 
    13 function restfile() { 
    14 # Rebuild ice restart for SAS CONFIG, and restartability checks. Expects LONG and SHORT run directories. 
    15 # For Stand Alone Surface configuration ocean is not running, just run ice model; so no outputs ocean files. 
    16 # Compares LONG rebuild restart ice file with equivalent entry from the SHORT rebuild restart ice file. 
    17 # 
    18 # check nco module loaded, and load it if not 
    19 if [ ! $( echo $LOADEDMODULES | grep cdo ) ]; then module load cdo >& /dev/null ; fi 
     13function get_dorv() { 
     14  if [ $lastchange == 'old' ] ; then  
     15    dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l ` 
     16    dorv=`echo $dorv | sed -e 's:.*/::'` 
     17  else 
     18    dorv=$lastchange 
     19  fi 
     20} 
     21 
     22function get_ktdiff() { 
     23  ktdiff=`diff ${1} ${2} | head -2 | grep it | awk '{ print $4 }'` 
     24} 
     25 
     26function resttest() {  
     27# 
     28# Restartability checks. Expects LONG and SHORT run directories 
     29# Compares end of LONG stat files with equivalent entries from the SHORT stat files. 
    2030# 
    2131  vdir=$1 
    2232  nam=$2 
    2333  pass=$3 
    24  
    25 # 
    26   if [ -d $vdir/$nam ]; then 
    27     dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l ` 
    28     dorv=`echo $dorv | sed -e 's:.*/::'` 
    29     rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | tail -2l | head -1 ` 
    30     rep2=`ls -1rt $vdir/$nam/$mach/$dorv/ | tail -1l` 
    31     cd ${SAS_RESTART_DIR}/LONG 
    32     #SF add here compilation of rebuild_tools to rebuild restart files, and add comparison of restart files 
    33     cd ${TOOLS_DIR} 
    34     ./maketools -n REBUILD_NEMO -m ${mach} > /dev/null 2>&1 
    35     cd ${TOOLS_DIR}/REBUILD_NEMO 
    36     #SF echo "REBUILD LONG restart SAS files, without standard output" 
    37     ./rebuild_nemo -t 4 ../../cfgs/ORCA2_SAS_ICE_ST/LONG/SAS_00000240_restart_ice  $NPROC > /dev/null 2>&1 
    38     #SF echo "REBUILD SHORT restart SAS files, without standard output" 
    39     ./rebuild_nemo -t 4 ../../cfgs/ORCA2_SAS_ICE_ST/SHORT/SAS_00000240_restart_ice $NPROC >&- 
    40     cd ${SAS_RESTART_DIR}/LONG 
    41     #SF echo "COPY rebuild restart files" 
    42     cp SAS_00000240_restart_ice.nc $vdir/$nam/$mach/$dorv/LONG/. 
    43     cp ../SHORT/SAS_00000240_restart_ice.nc $vdir/$nam/$mach/$dorv/SHORT/. 
    44  
    45     f1o=$vdir/$nam/$mach/$dorv/LONG/SAS_00000240_restart_ice.nc 
    46     f2o=$vdir/$nam/$mach/$dorv/SHORT/SAS_00000240_restart_ice.nc 
    47     if  [ ! -f $f1o ] &&  [ ! -f $f2o ] ; then 
    48       printf "%-27s %s\n" $nam " REBUILD SAS restart ice DOES NOT exists; incomplete test"; 
    49       return; 
    50     fi 
     34# 
     35# get $dorv 
     36  get_dorv 
     37# 
     38# check if directory is here 
     39  if [ ! -d $vdir/$nam/$mach/$dorv ]; then 
     40    printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv 
     41    return 
     42  fi 
     43 
     44  if [ -d $vdir/$nam/$mach/$dorv ]; then 
     45    # check ocean output 
     46    runtest $vdir $nam $pass RST 
    5147    # 
    52     done_oce=0 
    53     # 
    54   if  [  -f $f1o ] && [  -f $f2o ]; then 
    55 ## Compare the two netcdf files 
    56     cdo diffn $f1o $f2o > cdo_diff.out 2> /dev/null 
    57 ## Identical if cdo_diff.out exists but has zero size 
    58     if [ ! -s cdo_diff.out ]; then 
    59        difi=0 
    60     else 
    61 ## Identical if first character of $dif ==0 
    62        dif=$( grep -om1 '[0-9]* of [0-9]* records differ' cdo_diff.out ) 
    63 # difi contains the first character of summary of cdo dif. if = 0, then 0 record differ between the 2 files     
    64        if [ -n "$dif" ]; then 
    65            difi=`echo $dif | cut -c -1` 
    66        fi 
    67     fi 
    68     \rm cdo_diff.out  
    69  
    70     if [ $difi == 0 ]; then    
    71        if [ $pass == 0 ]; then 
    72          printf "%-27s %s %s\n" $nam  " ice restarts are IDENTICAL  passed : " $dorv 
    73        fi 
    74     else 
    75        printf "%-27s %s %s\n" $nam  " ice restarts are DIFFERENT  FAILED : " $dorv  
    76         # 
    77    # Offer view of differences on the second pass 
    78    # 
    79         if [ $pass == 1 ]; then 
    80           echo "<return> to view restart_ice.nc differences" 
    81           read y 
    82           cdo -diffv $f1o $f2o 
    83           done_oce=1 
    84           #echo "<return> to continue" 
    85           #read y 
    86         fi 
    87     fi 
    88   else 
    89       printf "%-27s %s\n" $nam " incomplete test"; 
    90       return; 
    91   fi 
    92 # 
    93 fi 
    94 } 
    95  
    96 function resttest() {  
    97 # 
    98 # Restartability checks. Expects LONG and SHORT run directories 
    99 # Compares end of LONG stat files with equivalent entries from the SHORT stat files. 
    100 # 
    101   vdir=$1 
    102   nam=$2 
    103   pass=$3 
    104 # 
    105   if [ -d $vdir/$nam ]; then 
    106     dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l ` 
    107     dorv=`echo $dorv | sed -e 's:.*/::'` 
     48    # run restartibility test 
    10849    f1o=$vdir/$nam/$mach/$dorv/LONG/ocean.output 
    10950    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat 
    … …  
    13374        fi 
    13475      else 
    135         printf "%-27s %s %s\n" $nam  " run.stat    restartability  FAILED : " $dorv  
     76        get_ktdiff f1.tmp$$ $f2s 
     77        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " run.stat    restartability  FAILED : " $dorv " (results are different after " $ktdiff " time steps)" 
    13678# 
    13779# Offer view of differences on the second pass 
    … …  
    162104        fi 
    163105      else 
    164         printf "%-27s %s %s\n" $nam  " tracer.stat restartability  FAILED : " $dorv  
     106        get_ktdiff f1.tmp$$ $f2t 
     107        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " tracer.stat    restartability  FAILED : " $dorv " (results are different after " $ktdiff " time steps)" 
    165108# 
    166109# Offer view of differences on the second pass 
    … …  
    196139  pass=$3 
    197140# 
    198   if [ -d $vdir/$nam ]; then 
    199     dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l ` 
    200     dorv=`echo $dorv | sed -e 's:.*/::'` 
     141# get $dorv 
     142  get_dorv 
     143# 
     144# check if directory is here 
     145  if [ ! -d $vdir/$nam/$mach/$dorv ]; then 
     146    printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv 
     147    return 
     148  fi 
     149# 
     150  if [ -d $vdir/$nam/$mach/$dorv ]; then 
     151    # check ocean output 
     152    runtest $vdir $nam $pass REPRO 
     153    # 
     154    # check reproducibility 
    201155    rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -2l | head -1 ` 
    202156    rep2=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | tail -1l` 
    … …  
    226180        fi 
    227181      else 
    228         printf "%-27s %s %s\n" $nam  " run.stat    reproducibility FAILED : " $dorv  
     182        get_ktdiff $f1s $f2s 
     183        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " run.stat    reproducibility FAILED : " $dorv " (results are different after " $ktdiff " time steps)" 
    229184# 
    230185# Offer view of differences on the second pass 
    … …  
    252207        fi 
    253208      else 
    254         printf "%-27s %s %s\n" $nam  " tracer.stat reproducibility  FAILED : " $dorv 
     209        get_ktdiff $f1t $f2t 
     210        printf "\e[38;5;196m%-27s %s %s %s %-5s %s\e[0m\n" $nam  " tracer.stat    reproducibility FAILED : " $dorv " (results are different after " $ktdiff " time steps)" 
    255211# 
    256212# Offer view of differences on the second pass 
    … …  
    275231  fi 
    276232} 
     233function runcmpres(){ 
     234# 
     235# compare *.stat file with reference file from a previous sette test or previous version 
     236# store in NEMO_VALID_REF at revision NEMO_REV_REF 
     237# Compares end of stat files from each 
     238# 
     239  vdir=$1 
     240  nam=$2 
     241  vdirref=$3 
     242  dorvref=$4 
     243  pass=$5 
     244# 
     245# get $dorv 
     246  get_dorv 
     247# 
     248# check if reference directory is present 
     249  if [ ! -d $vdirref/$nam/$mach/$dorvref ]; then 
     250    printf "%-27s %s\n" $nam " REFERENCE directory at $dorvref is MISSING" 
     251    return 
     252  fi 
     253  if [ ! -d $vdir/$nam/$mach/$dorv ]; then 
     254    printf "%-27s %s\n" $nam " VALID     directory at $dorv is MISSING" 
     255    return 
     256  fi 
     257 
     258# 
     259  if [ -d $vdir/$nam/$mach/$dorv ]; then 
     260    f1s=$vdir/$nam/$mach/$dorv/LONG/run.stat 
     261    f1t=$vdir/$nam/$mach/$dorv/LONG/tracer.stat 
     262    f2s=$vdirref/$nam/$mach/$dorvref/LONG/run.stat 
     263    f2t=$vdirref/$nam/$mach/$dorvref/LONG/tracer.stat 
     264    if  [ ! -f $f1s ] && [ ! -f $f1t ] ; then 
     265      printf "%-20s %s\n" $nam " incomplete test"; 
     266      return; 
     267    fi 
     268    if  [ ! -f $f2s ] && [ ! -f $f2t ] ; then 
     269      printf "%-20s %s\n" $nam " incomplete test"; 
     270      return; 
     271    fi 
     272# 
     273    done_oce=0 
     274 
     275    if  [ -f $f1s ] && [ -f $f2s ] ; then 
     276      cmp -s $f1s $f2s 
     277      if [ $? == 0 ]; then 
     278        if [ $pass == 0 ]; then 
     279          printf "%-20s %s %s\n" $nam  " run.stat    files are identical " 
     280        fi 
     281      else 
     282        printf "%-20s %s %s\n" $nam  " run.stat    files are DIFFERENT " 
     283# 
     284# Offer view of differences on the second pass 
     285# 
     286        if [ $pass == 1 ]; then 
     287          echo "<return> to view run.stat differences" 
     288          read y 
     289          sdiff $f1s $f2s 
     290          done_oce=1 
     291          echo "<return> to continue" 
     292          read y 
     293        fi 
     294      fi 
     295    fi 
     296    # Check tracer.stat files (if they exist) 
     297# 
     298    if  [ -f $f1t ] && [ -f $f2t ] ; then 
     299      cmp -s $f1t $f2t 
     300      if [ $? == 0 ]; then 
     301        if [ $pass == 0 ]; then           
     302          printf "%-20s %s %s\n" $nam  " tracer.stat files are identical " 
     303        fi 
     304      else 
     305        printf "%-20s %s %s\n" $nam  " tracer.stat files are DIFFERENT " 
     306# 
     307# Offer view of differences on the second pass 
     308# 
     309        if [ $pass == 1 ]; then 
     310          echo "<return> to view tracer.stat differences" 
     311          read y 
     312          sdiff $f1t $f2t 
     313        fi 
     314      fi 
     315    fi 
     316  fi 
     317} 
    277318 
    278319function runtest(){ 
    … …  
    284325  nam=$2 
    285326  pass=$3 
    286 # 
    287   if [ -d $vdir/$nam ]; then 
    288     dorv=`ls -1rt $vdir/$nam/$mach/ | tail -1l ` 
    289     dorv=`echo $dorv | sed -e 's:.*/::'` 
    290     rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep LONG | head -1 ` 
    291     if  [  -z $rep1 ] ; then 
    292       # Not all configurations test restartability (LONG/SHORT);  
    293       # check for a reproducibility directory instead (REPRO) 
    294       rep1=`ls -1rt $vdir/$nam/$mach/$dorv/ | grep REPRO | head -1 ` 
    295     fi 
    296     f1o=$vdir/$nam/$mach/$dorv/$rep1/ocean.output 
    297     if  [ ! -f $f1o ] ; then 
    298       printf "%-27s %s %s\n" $nam " ocean.output is MISSING            : " $dorv 
    299       return; 
    300     else  
    301       nerr=`grep 'E R R O R' $f1o | wc -l` 
    302       if [[ $nerr > 0 ]]; then 
    303         printf "%-27s %s %s\n" $nam " run FAILED : " $dorv 
    304         if [ $pass == 1 ]; then 
    305           echo "<return> to view end of ocean.output" 
    306           read y 
    307           tail -100 $f1o 
    308           echo '' 
    309           echo "full ocean.output available here: $f1o" 
    310         fi 
    311       fi 
    312     fi 
     327  ttype=$4 
     328  [[ $ttype == 'RST' ]] && ttype="LONG|SHORT" 
     329# 
     330# get $dorv 
     331  get_dorv 
     332# 
     333# no print needed if the repository is not here (already catch before) 
     334# 
     335  if [ -d $vdir/$nam/$mach/$dorv/ ]; then 
     336    # 
     337    # apply check for all ttype directory 
     338    rep1=$(ls -rt $vdir/$nam/$mach/$dorv/ | grep -E $ttype) 
     339    for tdir in $rep1 ; do 
     340       f1o=$vdir/$nam/$mach/$dorv/$tdir/ocean.output 
     341       if  [ ! -f $f1o ] ; then 
     342          if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam " ocean.output               MISSING : " $dorv ; fi 
     343          return; 
     344       else  
     345          nerr=`grep 'E R R O R' $f1o | wc -l` 
     346          if [[ $nerr > 0 ]]; then 
     347             printf "\e[38;5;196m%-27s %s %s %s\e[0m\n" $nam " run                         FAILED : " $dorv " ( E R R O R in ocean.output) "  
     348             if [ $pass == 1 ]; then 
     349                echo "<return> to view end of ocean.output" 
     350                read y 
     351                tail -100 $f1o 
     352                echo '' 
     353                echo "full ocean.output available here: $f1o" 
     354             fi 
     355             return; 
     356          fi 
     357       fi 
     358    done 
    313359  else 
    314     printf "%-27s %s %s\n" $nam  " directory is MISSING               : " $dorv 
     360    if [ $pass == 0 ]; then printf "%-27s %s %s\n" $nam  " directory                  MISSING : " $dorv ; fi 
    315361  fi 
    316362} 
    … …  
    352398        fi 
    353399      else 
    354         printf "%-5s %s %-5s %s %s %s\n" $rep1 "AGRIF vs" $rep2 "NOAGRIF run.stat    changed  -     FAILED : " $dorv1 $dorv2 
     400        get_ktdiff $f1s $f2s 
     401        printf "\e[38;5;196m%-5s %s %-5s %s %s %s %s %-5s %s\e[0m\n" $rep1 "AGRIF vs" $rep2 "NOAGRIF run.stat    changed  -     FAILED : " $dorv1 $dorv2 " (results are different after " $ktdiff " time steps)" 
    355402# 
    356403# Offer view of differences on the second pass 
    … …  
    376423######################################################################################### 
    377424# 
    378   mach=`grep "COMPILER=" ./sette.sh | sed -e 's/COMPILER=//'` 
    379   NEMO_VALID=`grep "NEMO_VALIDATION_DIR=" ./param.cfg | sed -e 's/NEMO_VALIDATION_DIR=//'` 
    380   NEMO_VALID=`eval "echo $NEMO_VALID"` 
     425# LOAD param variable (COMPILER, NEMO_VALIDATION_DIR, SVN_CMD) 
     426  SETTE_DIR=$(cd $(dirname "$0"); pwd) 
     427  MAIN_DIR=$(dirname $SETTE_DIR) 
     428  . ./param.cfg 
     429 
     430  mach=${COMPILER} 
     431  NEMO_VALID=${NEMO_VALIDATION_DIR} 
     432  NEMO_VALID_REF=${NEMO_VALIDATION_REF} 
     433  LANG=en_US 
    381434# 
    382435  if [ ! -d $NEMO_VALID ]; then 
    … …  
    386439# 
    387440# Directory to run the tests 
    388   SETTE_DIR=$(cd $(dirname "$0"); pwd) 
    389   MAIN_DIR=${SETTE_DIR}/.. 
    390441  CONFIG_DIR0=${MAIN_DIR}/cfgs 
    391442  TOOLS_DIR=${MAIN_DIR}/tools 
    … …  
    395446# Show current revision tag and branch name 
    396447# 
    397 cmd="svn" 
    398 [ ! -d "$MAIN_DIR/.svn" ] && cmd="git $cmd" 
    399 echo $cmd  
    400 echo "$MAIN_DIR/.svn" 
    401 lastchange=`$cmd info ${MAIN_DIR} | grep -i "Last Changed Rev:" | sed -e "s/ //g" | cut -d ":" -f 2` 
    402 revision=`$cmd info | grep Revision | cut -d ":" -f 2 | tr -d ' '` 
    403 branchname=`$cmd info | grep URL | rev | cut -d "/" -f 3 | rev` 
    404 echo "SETTE validation report : $branchname @ r$revision  ( last change @ r$lastchange )" 
     448echo "" 
     449lastchange=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep 'Last Changed Rev' | awk '{print $NF}'` 
     450revision=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep 'Revision' | awk '{print $NF}'` 
     451branchname=`LC_MESSAGES=${LANG} ${SVN_CMD} info ${MAIN_DIR} | grep ^URL | awk -F ipsl/forge/projets/nemo/svn/ '{print $NF}'` 
     452echo "Current code is : $branchname @ r$revision  ( last change @ r$lastchange )" 
     453[ `${SVN_CMD} status -q ${MAIN_DIR}/{cfg,tests,src} | wc -l` -ge 1 ] && lastchange=${lastchange}+ 
     454 
     455# overwrite revision is argument 1 present 
     456[[ $# -eq 1 ]] && lastchange=$1 
     457 
     458echo "" 
     459echo "SETTE validation report generated for : " 
     460echo "" 
     461echo "       $branchname @ r$lastchange (last changed revision)" 
     462echo "" 
     463echo "       on $COMPILER arch file" 
     464echo "" 
     465 
    405466# 
    406467# The script also needs the date or revision tag. Currently this is taken from the latest sub-directory found in each directory 
    … …  
    419480# 
    420481 
    421 # Rebuild and restartability test for SAS 
    422 # clem: not needed anymore 
    423 # for restart_file in WORCA2_SAS_ICE_ST 
    424 # do 
    425 #   restfile $NEMO_VALID $restart_file $pass 
    426 # done 
    427 # 
    428482# Restartability test 
    429483 echo "" 
    … …  
    432486 do 
    433487   resttest $NEMO_VALID $restart_test $pass 
    434    runtest $NEMO_VALID $restart_test $pass 
    435488 done 
    436489# 
    … …  
    441494 do 
    442495   reprotest $NEMO_VALID $repro_test $pass 
    443    runtest $NEMO_VALID $repro_test $pass 
    444496 done 
    445497 
    … …  
    450502 dir2=WAGRIF_DEMO_ST 
    451503 identictest $NEMO_VALID $dir1 $dir2 $pass  
    452  
     504# 
     505# before/after tests 
     506 if [ $lastchange == 'old' ] ; then 
     507    echo "" 
     508    echo "   !---- 'old' specified as revision => no comparison with reference results ----!   " 
     509    echo "" 
     510 else 
     511   echo "" 
     512   echo "   !----result comparison check----!   " 
     513   if [ $NEMO_VALID_REF != "/path/to/reference/sette/results" ]; then 
     514     echo '' 
     515     echo 'check result differences between :' 
     516     echo "VALID directory : $NEMO_VALID at rev $lastchange" 
     517     echo 'and' 
     518     echo "REFERENCE directory : $NEMO_VALID_REF at rev $NEMO_REV_REF" 
     519     echo '' 
     520     for repro_test in WGYRE_PISCES_ST WORCA2_ICE_PISCES_ST WORCA2_OFF_PISCES_ST WAMM12_ST WISOMIP_ST WORCA2_SAS_ICE_ST WAGRIF_DEMO_ST WSPITZ12_ST WISOMIP_ST WVORTEX_ST WICE_AGRIF_ST 
     521     do 
     522       runcmpres $NEMO_VALID $repro_test $NEMO_VALID_REF $NEMO_REV_REF $pass 
     523     done 
     524   else 
     525     echo '' 
     526     echo ' No path for comparison specified. Result are not compare with any other revision. ' 
     527     echo ' To do it please fill NEMO_VALID_REF and NEMO_REV_REF in param.cfg. ' 
     528     echo '' 
     529   fi 
     530 fi 
    453531done 
    454532# 
    455  
    456533exit 
Note: See TracChangeset for help on using the changeset viewer.