Changeset 5690


Ignore:
Timestamp:
2015-08-14T22:03:47+02:00 (5 years ago)
Author:
nicolasmartin
Message:

dev_r5092_CNRS_SETTE Add several annotations to comment script; worthy modifications to ending sequence

Location:
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST
Files:
4 edited

Legend:

Unmodified
Added
Removed
  • branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/config/X64_CURIE_cron.cfg

    r5644 r5690  
    44CDO='netcdf-utils-' 
    55COMPILER='ifort' 
    6 MPI='bullxmpi\/' 
     6MPI='bullxmpi/' 
    77NETCDF='netcdf-' 
    88 
  • branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.env

    r5683 r5690  
    22 
    33 
    4 ## Time-stamp & filename pattern 
     4## Patterns (time-stamp & filename) 
     5##-------------------------------------- 
    56dat=$( date ) 
     7## UTC time zone for timestamping 
    68export DATE=$( date -ud "$dat" +"%Y%m%d%Z%H%M" ) 
    79export PATTERNAME=${NEMO_USER}'_'${NEMO_HPCC} 
    810 
    9 ## Sourcing user configuration 
    10 ##-------------------------------------- 
     11## Source user configuration 
     12##--------------------------------------------------- 
    1113. ${NEMO_TRUS}/config/${NEMO_USER}.cfg 
    1214export NEMO_HOME NEMO_BRAN 
     
    2729export NEMO_TCMP NEMO_TRBD 
    2830 
    29 ## Sourcing super-computer configuration 
    30 ##-------------------------------------- 
     31## Source super-computer configuration 
     32##--------------------------------------------------- 
    3133. ${NEMO_TRUS}/config/${NEMO_HPCC}.cfg 
    3234export CDO      COMPILER MPI      NETCDF 
  • branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh

    r5689 r5690  
    77##--------------------------------------------------- 
    88NEMO_TRUS=$PWD 
    9 DEBUG=0; PUBLISH=0; TIME_STEP=0 
    10 IMOD=0; ST='FAILED' ## No 'module' command & 'FAILED' status 
    11 xios_full='--full'; stderr_redir='>&' ## DEBUG options 
     9DEBUG=0; PUBLISH=0 
     10IMOD=0; ST='FAILED' ## No modules system & 'FAILED' status 
    1211 
    1312 
    14 ## Get options for replacing some initials settings 
     13## Get options for replacing initials settings 
    1514##--------------------------------------------------- 
    1615while [ $# -ne 0 ]; do 
     
    2726 
    2827done 
     28 
     29## DEBUG options to speed up & expand verbosity of compilation 
     30xios_full='--full'; stderr_redir='>&' 
     31[ $DEBUG -eq 1 ] && xios_full='' && stderr_redir='>' 
    2932 
    3033 
     
    4851 
    4952 
    50 ## Summary of trusting test context 
     53## Display contextual summary of trusting test 
    5154##--------------------------------------------------- 
    5255echo 'NEMO Trusting test:' 
     
    5861 
    5962 
    60 ## Making timestamped directory with messenger files 
     63## Make timestamped directory with messenger files 
    6164##--------------------------------------------------- 
    6265cd ${TEST_DIR} 
     
    6669 
    6770 
    68 ## SVN action on XIOS & NEMO essentials directories 
     71## Get SVN revision on XIOS & NEMO essentials directories 
    6972##--------------------------------------------------- 
    7073print_step 'SVN command on NEMO directories:'; get_nemo_rev 
    7174 
    7275 
    73 ## Check softwares versions (source arch environment) 
     76## Check softwares versions (after sourced arch environment) 
    7477##--------------------------------------------------- 
    7578print_step 'Get softwares releases' 
     
    8184 
    8285 
    83 ## XIOS compiling 
     86## XIOS compilation from scratch 
    8487##--------------------------------------------------- 
    8588cd ${DIR_XIOS} 
    8689 
    8790print_step 'Compile XIOS' 
    88 [ $DEBUG -eq 1 ] && xios_full='' && stderr_redir='>' 
    8991eval ./make_xios ${xios_full} --arch ${NEMO_HPCC} --job $NPROC \ 
    9092    ${stderr_redir} /dev/null 
    9193[ ! -e ${DIR_XIOS}/lib/libxios.a ] && get_out 1 
    9294 
    93 ## NEMO config compiled from scratch 
     95 
     96## NEMO compilation from scratch 
    9497##--------------------------------------------------- 
    9598cd ${NEMO_CONF} 
     
    109112 
    110113 
    111 ## Get namelists, xml & forcing files for running 
     114## Get all inputs for running 
    112115##--------------------------------------------------- 
    113116cd ${TEST_DIR} 
    114117 
    115 print_step 'Set job (copying & extracting input files)' 
     118print_step 'Set job (copying or extracting inputs)' 
    116119cp ${NEMO_CONF}/${TEST_CONF}/cpp_* . 
    117120find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(_cfg\|.in\|opa\|_ref\|.xml\)' -exec cp {} . \; 
     
    122125 
    123126 
    124 ## Check input files in all cases 
     127## Check inputs 
    125128##--------------------------------------------------- 
    126 print_step 'Compare with benchmark input files list'; diff_inputs 
     129print_step 'Compare inputs to benchmark files'; diff_inputs 
    127130 
    128131 
    129 ## Job submission & computing 
     132## Job submission & computation 
    130133##--------------------------------------------------- 
    131134print_step 'Submit job'; JOB_ID=$( eval ${JOB_SUBM} ) 
     
    137140 
    138141 
    139 ## Job state & get time computation if succeeded 
     142## Check job state & get computing time if succeeded 
    140143##--------------------------------------------------- 
    141144print_step 'Test job state' 
     
    151154 
    152155 
    153 ## Inspect outputs 
     156## Check outputs 
    154157##--------------------------------------------------- 
    155158ST='OK' ## 'OK' by default 
     
    159162 
    160163 
    161 ## End 
     164## End, at least nothing has changed ;-) 
    162165##--------------------------------------------------- 
    163166get_out 'Code is reliable' 
  • branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting_func.sh

    r5689 r5690  
    1212FILE_COMM=mesg_11_comments_$PATTERNAME.txt; 
    1313 
    14 ## Trusting summary & archive timestamped filenames 
     14## Timestamped outcome & archive filenames 
    1515FILE_TRUS=trusting_${DATE}_$PATTERNAME.txt; FILE_ARCH=trusting_${DATE}_$PATTERNAME.tar.gz 
    1616 
     
    2727    echo 'Comments'      >  ${FILE_COMM} 
    2828 
    29     # 'Failed' status with 'Unknown error' by default 
     29    ## 'Failed' status with 'Unknown error' by default 
    3030    echo $ST             \ 
    3131   >> ${FILE_STAT} 
     
    3636 
    3737get_date() { 
    38     local dat=$( date -ud ${DATE} +"%F %R %Z" ) 
     38    ## UTC time zone for timestamping 
     39    local dat=$( date -ud "${DATE}" +"%F %R %Z" ) 
    3940 
    4041    echo $dat          \ 
     
    5859    fi 
    5960 
     61    ## Loop on essential directories 
    6062    for dir in ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} \ 
    6163               ${NEMO_EAGR} ${NEMO_EIOI} ${NEMO_EFCM} \ 
     
    6365          ${DIR_XIOS}                             ; do 
    6466 
    65    ## For time being, just get revision number from XIOS 
     67   ## For time being, just get revision from XIOS with no action on directory 
    6668   if [ $dir == ${DIR_XIOS} ]; then 
    6769       rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' ) 
     
    7678   ${svn_cmd} $dir 
    7779   rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' ) 
     80 
     81   ## Keep last rev. nb 
    7882   [ ${rev_loc} -gt $rev ] && rev=${rev_loc} 
    7983    done 
     
    9296   soft_rel='' 
    9397 
     98   ## Software strings to identify depends on use (sed is case-insensitive) 
    9499        if [ $IMOD -eq 1 ]; then 
    95        soft_rel=$( echo $LOADEDMODULES | sed "s/.*$soft\/\([0-9.a-z_]*\).*/\1/i" ) 
     100       ## Modules system: next word after "$soft/" 
     101       soft_rel=$( echo $LOADEDMODULES | sed "s#.*$soft/\([0-9.a-z_]*\).*#\1#i" ) 
     102       ## LOADEDMODULES=netcdf/mpi/4.1.3:cdo/1.5.9:intel/2013.0 
     103       ## LOADEDMODULES=bullxmpi/1.2.8.2:netcdf/4.3.3.1_hdf5_parallel:cdo/1.6.7 
    96104   else 
    97        soft_rel=$( echo $PATH          | sed "s/.*$soft\([0-9.a-z_]*\).*/\1/i"   ) 
    98    fi 
    99  
     105       ## Hard-coded paths: next word after "$soft" 
     106       soft_rel=$( echo $PATH          | sed "s#.*$soft\([0-9.a-z_]*\).*#\1#i"  ) 
     107       ## PATH=/smplocal/intel/impi/4.1.0.024/intel64/bin:/smplocal/pub/cdo/1.5.9/bin:/smplocal/pub/NetCDF/4.1.3/mpi/bin 
     108       ## PATH=/usr/local/netcdf-4.3.3.1_hdf5_parallel/bin:/opt/mpi/bullxmpi/1.2.8.2/bin:/usr/local/netcdf-utils-4.3.3.1_hdf5/bin 
     109   fi 
     110 
     111   ## option --version would work for main compilers (gfortran, intel, pgfortran, ...) 
    100112   [ $soft == $COMPILER ] && soft_rel=$( $soft --version | grep -m1 -oe '\<[0-9. ]*\>' ) 
     113 
    101114   ## Cleaning characters string to display proper soft name 
    102115   soft=$( echo $soft | sed 's#\\##g; s#[/-]$##' ) 
     116 
    103117   echo $soft ${soft_rel} \ 
    104118       >> model.log 
     
    115129 
    116130get_inputs() { 
     131    # List archive content & extract by default 
    117132    local cmd_iol="tar -tvf ${NEMO_FORC}/${NEMO_TARF}" cmd_iof="tar -vxf ${NEMO_FORC}/${NEMO_TARF}" 
    118133 
     134    ## List & copy file in case of personal inputs 
    119135    [ -z "${NEMO_TARF}" ] && cmd_iol="ls ${NEMO_FORC}/*" && cmd_iof="\cp ${NEMO_FORC}/* ." 
    120     ${cmd_iol} > inputs_list.txt && ${cmd_iof} > /dev/null 
     136 
     137    ${cmd_iol} > inputs_list.txt 
     138    ${cmd_iof} > /dev/null 
    121139} 
    122140 
     
    126144    local files_list='' mesg='Same'  
    127145 
     146    ## Simple diff 
    128147    for file in 'inputs_list.txt' *namelist_* *.xml cpp_*; do 
    129148   dif='' 
     149 
     150   ## Continue even if input file is not in here (see after) 
    130151   if [ -e ${REFE_DIR}/$file ]; then dif=$( diff -q $file ${REFE_DIR}/$file ); else dif=0; fi 
    131    if [ -n "$dif" ]; then mesg='Different'; [ "$dif" != '0' ] && echo $dif && files_list+=$file' '; fi 
     152 
     153   ## Pass over useless file omission in benckmark directory 
     154   if [ -n "$dif" && "$dif" != '0' ]; then 
     155       mesg='Different'; echo $dif; files_list+=$file' ' 
     156   fi 
     157 
    132158    done 
    133159 
     
    135161    echo $mesg         \ 
    136162   >> ${FILE_INPT} 
     163 
     164    ## List different files for web comment 
    137165    [ -n "${files_list}" ] && echo 'Inputs  : '${files_list}'differ<br>' \ 
    138166   >> temp_${FILE_COMM} 
     
    145173    sleep ${time_increment} 
    146174 
     175    ## Append a log file while pending 
    147176    while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do 
    148177   printf "\n%s\n" ${outline// /#} \ 
     
    156185    sleep ${time_increment} 
    157186 
     187    ## Kill remaining job & stop the test if it's too long 
    158188    if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then 
    159189   ${JOB_DELE} ${JOB_ID} &> /dev/null 
    160    export TIME_LIMIT/=3600 
    161190   get_out 6 
    162191    fi 
     
    167196    local files_list='' 
    168197 
     198    ## Simple diff 
    169199    for file in 'ocean.output' *.stat; do 
     200 
     201   ## Stop if no benchmark files (ocean.output, eventual stat files) 
    170202   [ ! -e ${REFE_DIR}/$file ] && export ST='FAILED' && get_out 7 
     203 
    171204   diff -q $file ${REFE_DIR}/$file 
     205 
     206   ## Continue even if it differs 
    172207   [ $? -ne 0 ] && export ST='FAILED' && files_list+=$file' ' 
    173208    done 
    174209 
     210    ## List different files for web comment 
    175211    [ -n "${files_list}" ] && echo 'Results : '${files_list}'differ<br>' \ 
    176212   >> temp_${FILE_COMM} 
     
    182218    local files_list='' dift=0 
    183219 
     220    ## Stop if no benchmark files (time.step) 
    184221    [ ! -e ${REFE_DIR}/time.step ] && export ST='FAILED' && get_out 7 
    185     export TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 
    186     echo 'Last time step of benchmark run: '${TIME_STEP} 
     222 
     223    time_step=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 
     224    echo 'Last time step of benchmark run: '${time_step} 
    187225 
    188226    ## Find all restart files to rebuild 
     
    195233                | sed "s#^.*\([0-9]\{8\}\)_restart.*#\1#"       | sort -u ) 
    196234 
     235   ## Loop on each time step 
    197236   for tmsp in ${list_tmsp}; do 
    198237       for comp in ${list_comp}; do 
     
    208247 
    209248      ## Compare restart files at same time step 
    210       if [ $tmsp -eq ${TIME_STEP} ]; then 
     249      if [ $tmsp -eq ${time_step} ]; then 
     250 
     251                    ## Stop if no benchmark files (restart file) 
    211252          if [ -e ${REFE_DIR}/$file.nc ]; then 
     253 
    212254                   ## UNIX `cmp` not suitable (timestamp in .nc file) 
    213255         dif=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null      \ 
    214256                | awk '/records/ {print $0}' | sed '2 s/^/,/' | tr -d '\n'  ) 
    215257 
     258         ## CDO can return no output on stdout with no difference 
    216259         if [[ -n "$dif" && $( echo $dif | awk '{print $1}' ) -ne 0 ]]; then 
    217260             export ST='FAILED' && files_list+=$file' ' 
     
    231274   done 
    232275 
     276        ## List different files for web comment with sum of different parameters 
    233277   if [ $dift -ne 0 ]; then 
    234278       echo 'Restarts: '${files_list}$dift' record(s) differ<br>' | tee -a temp_${FILE_COMM} 
     
    245289 
    246290get_cpu_time() { 
     291    ## Interest  
    247292    local real_cpu_time=$( eval ${JOB_TIME} ) 
    248293 
     
    252297 
    253298comments() { 
     299    local opt 
    254300    local state=$1 
    255301 
    256     if [ "$state" == 'E R R O R' ]; then 
    257    [ -e time.step ] && export TIME_STEP=$( cat time.step | tr -d ' ' ) 
    258    [ -e ocean.output ] && line=$( grep -m1 -A4 "$state"   ocean.output | tr -d '\n' ) 
    259     else 
    260    [ -e ocean.output ] && line=$( grep -m1 -A2 "^ $state" ocean.output | tr -d '\n' ) 
    261     fi 
     302    ## 'W A R N I N G' by default 
     303    opt='-A2 "^ $state"' 
     304    [ "$state" == 'E R R O R' ] && opt="-A4 $state" 
     305 
     306    ## Select first occurence for web comment 
     307    line=$( grep -m1 $opt ocean.output | tr -d '\n' ) 
    262308 
    263309    echo   $line 
     
    272318   >> ${FILE_COMM} 
    273319 
     320    ## Construct txt file for web with all messenger files 
    274321    paste -d ';' mesg_*.txt | tee ${FILE_TRUS} 
    275322} 
     
    279326    local cmd 
    280327 
     328    ## Production mode (-p|--publish) 
    281329    if [ $PUBLISH -eq 1 ]; then 
    282330 
     331   ## Create or append historical trusting file 
    283332   if [ -f ${REFE_DIR}/trusting_$PATTERNAME.txt ]; then cmd='tail -1'; else cmd='cat'; fi 
    284333 
     
    286335       >> ${REFE_DIR}/trusting_$PATTERNAME.txt 
    287336 
    288   ## Send mail only when FAILED 
     337        ## Send mail only when FAILED 
    289338   if [[ ! -z "$EMAIL" && $ST == 'FAILED' ]]; then 
     339 
     340       ## Content 
    290341       cat <<END_MAIL \ 
    291342      > trusting.mail 
     
    302353`cat ${TEST_DIR}/${FILE_TRUS}` 
    303354END_MAIL 
     355 
     356       ## Send with detailed subject 
    304357       mail -s "[NEMO Trusting ${NEMO_HPCC} ${REFE_CONF} ${NEMO_BRAN}] $ST $ERR" $EMAIL \ 
    305358      <  trusting.mail 
     
    320373   case $ERR in 
    321374       ## Compilation 
    322        1) ERR='XIOS compilation failed'          ;; 2) ERR='NEMO compilation failed'            ;; 
     375       1) ERR='XIOS compilation failed' ;; 2) ERR='NEMO compilation failed';; 
    323376       ## Submission 
    324        3) ERR='Missing input files'              ;; 4) ERR='Job submission error'               ;; 
     377       3) ERR='Missing input files'     ;; 4) ERR='Job submission error'   ;; 
    325378       ## Running 
    326        5) ERR='Crashed at time step '${TIME_STEP};; 6) ERR='Exceeded time limit '${TIME_LIMI}'h';; 
     379       5) ERR='Crashed at time step'    ;; 6) ERR='Exceeded time limit'    ;; 
    327380       ## Results 
    328        7) ERR='Missing previous outputs'         ;; 8) ERR='New outputs differ'                 ;; 
     381       7) ERR='Missing previous outputs';; 8) ERR='New outputs differ'     ;; 
    329382       ## Undefined 
    330        *) ERR='Unknown error'                    ;; 
     383       *) ERR='Unknown error'           ;; 
    331384   esac 
    332385 
    333    export ERR 
    334     fi 
    335  
    336     sed -i "2 s/.*/$ST/" ${FILE_STAT} && sed -i "2 s/.*/$ERR/" ${FILE_RESU} 
    337  
    338     ## Save tested configuration if trusting failed 
     386    fi 
     387 
     388    ## Eventual comments from ocean.output 
     389    print_step 'Comments' 
     390    if [ -e ocean.output ]; then  
     391   if [ $ERR -eq 5 ]; then 
     392       comments 'E R R O R' 
     393       [ -e time.step ] && ERR+=' '$( cat time.step | tr -d ' ' ) 
     394   else 
     395       comments 'W A R N I N G' 
     396       [ $ERR -eq 6   ] && ERR+=' '$(( ${TIME_LIMIT}/3600 ))'h' 
     397   fi 
     398    fi 
     399 
     400    ## Share trusting result (locally or not) 
     401    export ERR 
     402    sed -i "2 s/.*/$ST/"  ${FILE_STAT} 
     403    sed -i "2 s/.*/$ERR/" ${FILE_RESU} 
     404 
     405    ## Save tested configuration if trusting failed in production mode (-p|--publish) 
    339406    if [[ $ST == 'FAILED' && $PUBLISH -eq 1 && $DEBUG -eq 0 ]]; then 
    340407   echo 'Creating archive '${FILE_ARCH}' under '${REFE_DIR} 
     
    343410    fi 
    344411 
    345     if [ $ERR -eq 5 ]; then 
    346    print_step 'Comments'; comments 'E R R O R' 
    347     else 
    348    print_step 'Comments'; comments 'W A R N I N G' 
    349     fi 
    350  
     412    ##  
    351413    print_step 'Trusting outcome'; mesg_make; mesg_publish 
    352414 
Note: See TracChangeset for help on using the changeset viewer.