New URL for NEMO forge!   http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.
Changeset 5438 for branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM – NEMO

Ignore:
Timestamp:
2015-06-18T23:02:31+02:00 (9 years ago)
Author:
nicolasmartin
Message:

dev_r5092_CNRS_SETTE Increase trusting main script clarity with new functions

Location:
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST
Files:
2 edited

Legend:

Unmodified
Added
Removed

  • branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh

    r5432 r5438  
    1111#--------------------------------------------------- 
    1212NEMO_TRUS=$( pwd $( dirname $0 ) ) 
    13 SVN_CMD='svn status' 
     13SVN_OPT='status' 
    1414DEBUG=0; PUBLISH=0; status='OK' 
    1515 
    … …  
    2727    esac 
    2828done 
    29  
    30 #export DEBUG PUBLISH 
    31  
    3229 
    3330# Initialization (super-computer & user environment) 
    … …  
    4744fi 
    4845 
    49 # If -v|--version option has been set, modify svn command 
    50 [ $( echo ${NEMO_VERS} | grep  "up\|update"         ) ] && SVN_CMD='svn update' 
    51 [ $( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) ] && SVN_CMD='svn update -r '$( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) 
    52  
    5346. ./trusting_func.sh 
    54  
    5547 
    5648#  
    … …  
    5951echo ${TEST_DIR} && cd ${TEST_DIR} 
    6052datestr1=$( LC_ALL=C date -d"${DATE_STR}" +%Y-%m-%dT%H:%M ) 
    61 printf "Date\n%s\n"       $datestr1 > mesg_01_date_${CFG_USER}_${CFG_ARCH}.txt 
     53printf "Date\n$datestr1\n" > mesg_01_date_${CFG_USER}_${CFG_ARCH}.txt 
    6254# Only at end status will change to OK 
    63 printf "Status\nFAILED\n"           > mesg_03_status_${CFG_USER}_${CFG_ARCH}.txt 
     55status='FAILED' 
    6456 
    65  
    66 # Check softwares versions 
     57# Check softwares versions (source arch environment) 
    6758#--------------------------------------------------- 
    68 printf "Step......................\nGet softwares release\n"                     | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
    69 . ${ARCH_ENV} > /dev/null    # Sourcing architecture environment 
     59pstep 'Get softwares release' 
     60. ${ARCH_ENV} > /dev/null 
    7061get_soft_rel 
    71 sed -n 2p model.log | sed 'i\Fortran compiler' | tee mesg_05_compiler_${CFG_USER}_${CFG_ARCH}.txt 
    72 sed -n 3p model.log | sed 'i\MPI libraries   ' | tee mesg_06_mpi_${CFG_USER}_${CFG_ARCH}.txt 
    73 sed -n 4p model.log | sed 'i\NetCDF libraries' | tee mesg_07_netcdf_${CFG_USER}_${CFG_ARCH}.txt 
    74  
    7562 
    7663# SVN action on XIOS & NEMO essentials directories 
    7764#--------------------------------------------------- 
    78 printf "Step......................\nGet NEMO SVN revision with: ${SVN_CMD}\n"    | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
     65pstep "Get NEMO SVN state with: ${SVN_OPT}" 
    7966get_nemo_rev 
    80 sed -n 6p model.log | sed 'i\NEMO revision'    | tee  mesg_02_nemo_${CFG_USER}_${CFG_ARCH}.txt 
    81  
    82  
     67exit 0 
    8368# NEMO config compiled from scratch 
    8469#--------------------------------------------------- 
    85 printf "Step......................\nCompiling ${CONF_REF} configuration\n"       | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
     70pstep "Compiling ${CONF_REF} configuration" 
    8671cd ${NEMO_CONF} 
    8772#[ -d ${TEST_CONF} ] && ./makenemo -n ${TEST_CONF} clean 
    8873./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m $ARCH -j $NPROC >& /dev/null 
    89 [ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out 
    90  
     74[ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out $step $status 
    9175 
    9276# Get namelists, xml & forced files for running 
    9377#--------------------------------------------------- 
    94 cd ${TEST_DIR} 
    95 printf "Step......................\nPreparing job\n"                             | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
     78pstep 'Preparing job' 
    9679find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(cfg\|opa\|ref\|xml\)' -exec cp {} . \; 
    9780 
    … …  
    10386 
    10487${cmd_iol} > inputs_list.txt && ${cmd_iof} > /dev/null 
    105 [ $? -ne 0 ] && get_out 
     88[ $? -ne 0 ] && get_out $step $status 
    10689[ $( find . -name '*.gz' -print -quit ) ] && gunzip *.gz 
    107 exit 0 
     90 
     91# Check input files in all cases 
     92#--------------------------------------------------- 
     93pstep 'Test input files difference' 
     94diff_inputs 
    10895 
    10996# Job submission & computing 
    11097#--------------------------------------------------- 
    111 printf "Step......................\nRunning\n"                                   | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt; echo ${JOB_SUBM} 
     98pstep 'Running' 
    11299job_num=$( ${JOB_SUBM} | awk '{print $NF}' ) 
    113 [ $? -ne 0 ] && get_out 
     100[ $? -ne 0 ] && get_out $step $status 
    114101 
    115102time_elapsed=0; time_increment=30 
    … …  
    125112# Eventual crash during computing 
    126113#--------------------------------------------------- 
    127 printf "Step......................\nTest if ASCII output files exist\n"          | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
    128 if   [ ! -f ocean.output                  ]; then 
    129     printf "Step......................\nOpa crashed at initialization\n"         | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
     114pstep 'Test if ASCII output files exist' 
     115if   [[ ! -f ocean.output || $( grep 'E R R O R' ocean.output ) ]]; then 
     116    ${JOB_DELE} ${job_num} &> /dev/null 
     117    [ ! -f ocean.output                  ] && step='nemo.exe crashed at initialization' 
     118    [ $( grep 'E R R O R' ocean.output ) ] && step='nemo.exe crashed' 
     119    comments('E R R O R') 
    130120    ${JOB_DELE} ${job_num} >/dev/null 2>&1 
    131121    status='FAILED' 
    132 elif [ $( grep 'E R R O R' ocean.output ) ]; then 
    133     printf "Step......................\nOpa crashed\n"                           | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
    134     grep 'E R R O R'       ocean.output | tee    mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt 
    135     sed 's/^/Time step :/' time.step    | tee -a mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt 
    136     ${JOB_DELE} ${job_num} >/dev/null 2>&1 
    137     status='FAILED' 
     122    get_out $step $status 
     123else 
     124    status='OK' 
     125 
     126    # Inspect output text files 
     127    #--------------------------------------------------- 
     128    pstep 'Test ASCII output files diff' 
     129    for file in ocean.output solver.stat tracer.stat; do 
     130   diff $file ${REFE_DIR}/$file 
     131   [ $? -ne 0 ] && status='FAILED' 
     132    done 
     133 
     134    # Inspect output NetCDF files 
     135    #--------------------------------------------------- 
     136    pstep 'Test last restart NetCDF files diff' 
     137    diff_restart #2> /dev/null 
     138    [ $? -ne 0 ] && status='FAILED' 
     139 
     140    # Get computation duration 
     141    #--------------------------------------------------- 
     142    pstep 'Get real CPU time' 
     143    get_cpu_time 
     144 
     145    comments('W A R N I N G') 
    138146fi 
    139  
    140  
    141 # Inspect output text files 
    142 #--------------------------------------------------- 
    143 printf "Step......................\nTest ASCII output files difference\n"        | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
    144 for file in ocean.output solver.stat tracer.stat; do 
    145     diff $file ${REFE_DIR}/$file 
    146     [ $? -ne 0 ] && status='FAILED' 
    147 done 
    148  
    149  
    150 # Inspect output NetCDF files 
    151 #--------------------------------------------------- 
    152 printf "Step......................\nTest last restart NetCDF files difference\n" | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
    153 diff_restart 2>/dev/null 
    154 [ $? -ne 0 ] && status='FAILED' 
    155  
    156  
    157 # Finals tests 
    158 #--------------------------------------------------- 
    159 if [ $status != 'FAILED' ]; then 
    160  
    161     # Comments 
    162     #--------------------------------------------------- 
    163     echo 'Comments'                    | tee    mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt 
    164     grep 'W A R N I N G:' ocean.output | tee -a mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt 
    165  
    166     # Portability ! 
    167     printf "Step......................\nGet real CPU time\n"                     | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
    168     real_cpu_time=$( ${JOB_VIEW} ${job_num} | awk 'END{print $9}' ) 
    169     printf "Real CPU time\n%s\n" ${real_cpu_time} | tee mesg_08_realcputime_${CFG_USER}_${CFG_ARCH}.txt 
    170     printf "Input files\nNA"                      | tee mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt 
    171 else 
    172     printf "Step......................\nTest input files difference\n"           | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
    173     diff_inputs 
    174     get_out 
    175 fi 
    176  
    177  
    178 # So far, everything is good 
    179 #--------------------------------------------------- 
    180 printf "Status\nOK\n" > mesg_03_status_${CFG_USER}_${CFG_ARCH}.txt 
    181 printf "Step......................\nCode is reliable\n"                          | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
    182  
    183147 
    184148# End 
    185149#--------------------------------------------------- 
    186 get_out 
     150if [ $status == 'FAILED' ]; then step='Test outputs difference'; else step='Code is reliable'; fi 
     151get_out $step $status 

  • branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting_func.sh

    r5432 r5438  
    22 
    33 
     4comments() { 
     5    echo 'Comments'      | tee    mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt 
     6    grep $1 ocean.output | tee -a mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt 
     7} 
     8 
     9get_cpu_time() { 
     10    # Portability ! 
     11    real_cpu_time=$( ${JOB_VIEW} ${job_num} | awk 'END{print $9}' ) 
     12    printf "Real CPU time\n%s\n" ${real_cpu_time} | tee mesg_08_realcputime_${CFG_USER}_${CFG_ARCH}.txt 
     13} 
     14 
    415diff_inputs() { 
     16    mesg='Same' 
     17 
    518    for file in inputs_list.txt $( ls namelist_* ) iodef.xml; do 
    619   diff $file ${REFE_DIR}/$file 
    7  
    8    if [ $? -ne 0 ]; then 
    9        printf "Input files\nDifferent" | tee mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt 
    10        get_out 
    11    fi 
    12  
     20   [ $? -ne 0 ] && mesg='Different' 
    1321    done 
    1422 
    15     printf "Input files\nSame"              | tee mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt 
     23    printf "Input files\n$mesg" | tee mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt 
    1624} 
    1725 
    18 diff_nc() { 
    19     stat=100; file1=$1; file2=$2 
    20  
    21     $CDO diffv $file1 $file2 > out_tmp 
    22     ( [ $? -ne 0 ] && stat=100 ) || stat=$( grep -m 1 'records differ' out_tmp | awk -Fof '{print $1}' ) 
    23     [ -f out_tmp ] && rm out_tmp 
    24  
    25     return $stat 
    26 }  
    27  
    2826diff_restart() { 
    29     result=0 
    3027    last_time_step=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 
    31     echo 'Last time step: '${last_time_step} 
     28    echo 'Last time step of standard run: '${last_time_step} 
    3229 
    3330    if [ $( find ${TEST_DIR} -name "*${last_time_step}_restart*.nc" -print -quit ) ]; then 
    34    basename=$( find ${TEST_DIR} -name "*${last_time_step}_restart*.nc" -print -quit \ 
    35                 | sed "s/^\(.*\)$last_time_step\_.*/\1$last_time_step/"              ) 
     31   base_name=$( find ${TEST_DIR} -name "*${last_time_step}_restart*.nc" -print -quit          \ 
     32                | awk -F/ '{print $NF}' | sed "s/^\(.*\)$last_time_step\_.*/\1$last_time_step/" ) 
    3633 
    3734   for comp in restart restart_ice restart_trc; do 
    38        printf "${TEST_DIR}/${basename}_${comp}*.nc: " 
     35       printf "${base_name}_${comp}.nc: " 
    3936 
    40        nb_dom=$( find ${TEST_DIR} -name "${basename}_${comp}_[0-9]*.nc" | wc -l | awk '{print $1}' ) 
     37       nb_dom=$( find ${TEST_DIR} -name "${base_name}_${comp}_[0-9]*.nc" | wc -l | awk '{print $1}' ) 
    4138       if [ ${nb_dom} -gt 1 ]; then 
    42       ${NEMO_TRBD}/rebuild_nemo -t $NPROC ${basename}_$comp ${nb_dom} >/dev/null 
     39      ${NEMO_TRBD}/rebuild_nemo -t $NPROC ${base_name}_$comp ${nb_dom} > /dev/null 
    4340       fi 
    4441 
    45        cmp ${TEST_DIR}/${basename}_$comp.nc ${REFE_DIR}/${basename}_$comp.nc 
     42       cmp ${TEST_DIR}/${base_name}_$comp.nc ${REFE_DIR}/${base_name}_$comp.nc 
    4643       if [ $? -ne 0 ]; then 
    47       result=$( diff_nc ${TEST_DIR}/${basename}_$comp.nc ${REFE_DIR}/${basename}_$comp.nc ) 
    48       printf "$CDO diffv $result\n" 
     44      status='FAILED' 
     45      printf "$CDO diffn\n" 
     46      $CDO diffn ${TEST_DIR}/${base_name}_$comp.nc ${REFE_DIR}/${base_name}_$comp.nc 2> /dev/null 
    4947       else 
    5048      printf "identical\n" 
    5149       fi 
    52 #      \rm *${last_time_step}_${comp}_[0-9]*.nc 
     50 
     51       \rm  ${TEST_DIR}/${base_name}_${comp}_[0-9]*.nc 
    5352   done 
    5453 
    … …  
    6059get_soft_rel() { 
    6160    for rel in $CDO $COMPILER $MPI $NETCDF; do 
    62                                  arch_rel=$( echo $LOADEDMODULES | sed "s#.*$rel/\([^:]*\).*#\1#" ) 
    63    [ -z "${arch_rel}"  ] && arch_rel=$( echo $PATH          | sed "s#.*$rel/\([^/]*\).*#\1#" ) 
    64    [ $rel == $COMPILER ] && arch_rel=$( $rel --version | grep -m1 -o ' [0-9.]* ' ) 
     61                                 arch_rel=$( echo $LOADEDMODULES | sed  "s#.*$rel/\([^:]*\).*#\1#" ) 
     62   [ -z "${arch_rel}"  ] && arch_rel=$( echo $PATH          | sed  "s#.*$rel/\([^/]*\).*#\1#" ) 
     63   [ $rel == $COMPILER ] && arch_rel=$( $rel --version      | grep -m1 -o ' [0-9.]* '        ) 
    6564   echo $rel ${arch_rel} >> model.log 
    6665    done 
     66 
     67    sed -n 2p model.log | sed 'i\Compiler'    | tee mesg_05_compiler_${CFG_USER}_${CFG_ARCH}.txt 
     68    sed -n 3p model.log | sed 'i\MPI libs'    | tee mesg_06_mpi_${CFG_USER}_${CFG_ARCH}.txt 
     69    sed -n 4p model.log | sed 'i\NetCDF libs' | tee mesg_07_netcdf_${CFG_USER}_${CFG_ARCH}.txt 
    6770} 
    6871 
    … …  
    7073    rev=0 
    7174 
     75    # If -v|--version option has been set, modify svn command 
     76    if   [ $( echo ${NEMO_VERS} | grep  "up\|update"         ) ]; then SVN_OPT='update' 
     77    elif [ $( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) ]; then SVN_OPT='checkout' 
     78   rev=$( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) 
     79    fi 
     80 
    7281    for dir in ${DIR_XIOS}  ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} ${NEMO_EIOI} \ 
    7382          ${NEMO_EFCM} ${NEMO_TRUS} ${NEMO_TCMP} ${NEMO_TRBD}               ; do 
     83 
    7484   # For time being, just get revision number from XIOS 
    75    [ $dir == ${DIR_XIOS} ] && echo 'XIOS '$( svn info $dir | awk '(NR == 9) {print $NF}' ) >> model.log && continue 
    76    ${SVN_CMD} $dir 
     85   if [ $dir == ${DIR_XIOS} ]; then 
     86       echo 'XIOS '$( svn info $dir | awk '(NR == 9) {print $NF}' ) >> model.log 
     87       continue 
     88   fi 
     89 
     90   #  
     91   if [ $( echo ${SVN_OPT} | grep 'checkout' ) ]; then 
     92       cd ../$dir  
     93       url=$( svn info $dir | grep URL | awk '{print $NF}' ) 
     94       svn ${SVN_OPT} $url -r $rev $dir && cd - 
     95   else 
     96       svn ${SVN_OPT} $dir 
     97   fi 
     98 
    7799   rev_loc=$( svn info $dir | awk '(NR == 5) {print $NF}' ) 
    78100   (( ${rev_loc} >= $rev )) && rev=${rev_loc} 
    … …  
    80102 
    81103    echo 'NEMO '$rev >> model.log 
     104    sed -n 6p model.log | sed 'i\NEMOGCM rev.' | tee mesg_02_nemo_${CFG_USER}_${CFG_ARCH}.txt 
    82105} 
    83106 
    84107mesg_make() { 
    85108    paste -d ';' mesg_*_${CFG_USER}_${CFG_ARCH}.txt > trusting_${DATE_STR}_${CFG_USER}_${CFG_ARCH}.txt 
    86     cat trusting_${DATE_STR}_${CFG_USER}_${CFG_ARCH}.txt 
     109    cat                                               trusting_${DATE_STR}_${CFG_USER}_${CFG_ARCH}.txt 
    87110} 
    88111 
    … …  
    115138} 
    116139 
     140pstep() { 
     141    printf "Step.....\n$1\n" 
     142} 
     143 
    117144get_out() { 
     145    step=$1; status=$2 
    118146    mesg_make; mesg_publish 
    119147 
    120148    # Save tested configuration if trusting failed 
    121149    if [ $status == 'FAILED' ]; then 
     150   touch mesg_08_realcputime_${CFG_USER}_${CFG_ARCH}.txt mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt 
    122151   cd ${REFE_DIR} 
    123152   tar -czf trusting_${DATE_STR}_${CFG_USER}_${CFG_ARCH}.tar.gz ${TEST_DIR}/* 
    124153   cd - 
    125154    fi 
    126     
     155 
     156    printf "Status\n$status\n"  > mesg_03_status_${CFG_USER}_${CFG_ARCH}.txt 
     157    printf "Step.....\n$step\n" > mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt 
     158 
    127159    exit 1 
    128160} 
Note: See TracChangeset for help on using the changeset viewer.