source: trunk/NEMOGCM/TRUST/inc/trusting_func.sh @ 5936

Last change on this file since 5936 was 5936, checked in by nicolasmartin, 5 years ago

Merge dev_r5092_CNRS18_TRUST onto the trunk: add new tool Trusting (ie TRUST directory at root of NEMGOCM) for continuous integration tests on HPC centers & basic validation tests for developers

  • Property svn:executable set to *
File size: 12.7 KB
Line 
1#!/bin/bash
2
3
4## Messenger filenames
5FILE_DATE=mesg_01_date_$PATTERNAME.txt  ; FILE_RSLT=mesg_02_result_$PATTERNAME.txt
6FILE_STAT=mesg_03_state_$PATTERNAME.txt ; FILE_NEMO=mesg_04_nemo_$PATTERNAME.txt
7FILE_XIOS=mesg_05_xios_$PATTERNAME.txt  ; FILE_CMPF=mesg_06_compiler_$PATTERNAME.txt
8FILE_LMPI=mesg_07_mpi_$PATTERNAME.txt   ; FILE_NCDF=mesg_08_netcdf_$PATTERNAME.txt
9FILE_INPT=mesg_09_inputs_$PATTERNAME.txt; FILE_TIME=mesg_10_time_$PATTERNAME.txt
10FILE_MEMY=mesg_11_memory_$PATTERNAME.txt; FILE_NOTE=mesg_12_comments_$PATTERNAME.txt
11
12## Trusting timestamped logfile & archive
13TRUS_FILE=trusting_${DATE}_$PATTERNAME.txt; TRUS_ARCH=trusting_${DATE}_$PATTERNAME.tgz
14
15
16## Functions in order of use
17print_step() {
18    local char_nb=$( echo "$1" | wc -c )
19    local outline=$( printf "%${char_nb}s" )
20
21    printf "\nStep.....\n%s\n%s\n" "$1" ${outline// /-}
22}
23
24init_files() {
25    echo 'Date'               > ${FILE_DATE}; echo 'Result'           > ${FILE_RSLT}
26    echo 'State'              > ${FILE_STAT}; echo 'NEMOGCM rev.'     > ${FILE_NEMO}
27    echo 'XIOS rev.'          > ${FILE_XIOS}; echo 'Fortran compiler' > ${FILE_CMPF}
28    echo 'MPI libs'           > ${FILE_LMPI}; echo 'NetCDF libs'      > ${FILE_NCDF}
29    echo 'Input files'        > ${FILE_INPT}; echo 'Elapsed time'     > ${FILE_TIME}
30    echo 'Memory usage (P/V)' > ${FILE_MEMY}; echo 'Comments'         > ${FILE_NOTE}
31
32    ## 'Failed' status with 'Unknown error' by default
33    echo ${TRUS_RSLT}      \
34   >> ${FILE_RSLT}
35    echo 'Unknown error' \
36   >> ${FILE_STAT}
37}
38
39get_date() {
40    ## UTC time zone for timestamping
41    local dat=$( date -ud "${DATE}" +"%F %R %Z" )
42
43    echo $dat           \
44   >> ${FILE_DATE}
45}
46
47get_nemo_rev() {
48    local dir rev_loc
49    local rev=0
50
51    ## Loop on essential NEMO directories
52    for dir in ${TRUS_CKOT} ${TRUS_XIOS}; do
53
54   ## For time being, just get revision from XIOS with no action on directory
55   if [ $dir == ${TRUS_XIOS} ]; then
56       rev_loc=$( svn info $dir | awk '/Last Changed Rev/ {print $NF}' )
57       echo 'XIOS '${rev_loc} \
58      >> model.log
59       echo "<a href=\"https://forge.ipsl.jussieu.fr/ioserver/changeset/${rev_loc}\" target=\"_blank\">${rev_loc}</a>" \
60      >> ${FILE_XIOS}
61       continue
62   fi
63
64   echo $dir && ${TRUS_SVNA} ${TRUS_NGCM}/$dir
65   rev_loc=$( svn info ${TRUS_NGCM}/$dir | awk '/Last Changed Rev/ {print $NF}' )
66
67   ## Keep last rev. nb
68   [ ${rev_loc} -gt $rev ] && rev=${rev_loc}
69    done
70
71    echo 'NEMOGCM '$rev \
72   >> model.log
73    echo "<a href=\"https://forge.ipsl.jussieu.fr/nemo/changeset/$rev\" target=\"_blank\">$rev</a>" \
74   >> ${FILE_NEMO}
75}
76
77get_soft_rel() {
78    local soft_rel str
79
80    ## Sourcing environment
81    if [ -n "${TRUS_ENVI}" ]; then
82   if [[  -e ${TRUS_ENVI}.env && $( declare -F | grep ' module' ) ]]; then
83            ## .env file if module function is available
84       . ${TRUS_ENVI}.env
85   else
86            ## .path file if existing, if not the given file
87       [ -e ${TRUS_ENVI}.path ] && . ${TRUS_ENVI}.path || . ${TRUS_ENVI}
88   fi
89    fi
90
91    ## Problem with `prepend-path` of modulefile that use ':' instead of ' ' as delimiter
92    [ $TRUS_HPCC == 'X64_ADA' ] && WRAPPER_LDFLAGS='-L/smplocal/pub/IdrMemMPI/1.4/lib -lidrmem '${WRAPPER_LDFLAGS}
93
94    for str in ${TRUS_CMPV} ${TRUS_MPIR} ${TRUS_CDFR} ${TRUS_CDOR}; do
95   [ -z "$str" ] && continue
96   soft_rel=''
97
98   ## Software release: next word after "$soft" in $PATH (case-insensitive)
99   soft_rel=$( echo $PATH | sed "s#.*$str\([0-9.a-z_]*\).*#\1#i" )
100
101   ## option --version would work for main compilers (gfortran, intel, pgfortran, ...)
102   [ $str == ${TRUS_CMPV} ] && soft_rel=$( $str --version | grep -m1 -oe '\<[0-9. ]*\>' )
103
104   ## Cleaning characters string to display proper soft name
105   str=$( echo $str | sed 's#\\##g; s#[/-]$##' )
106
107   echo $str ${soft_rel} \
108       >> model.log
109    done
110
111    sed -n 3p model.log \
112   >> ${FILE_CMPF}
113    sed -n 4p model.log \
114   >> ${FILE_LMPI}
115    sed -n 5p model.log \
116   >> ${FILE_NCDF}
117}
118
119get_inputs() {
120    ## Extract archive or copy files in case of personal inputs
121    [ -z "${TRUS_TARF}" ] && get_io="cp ${TRUS_FORC}/* ." || get_io="tar -vxf ${TRUS_FORC}/${TRUS_TARF}"
122
123    ${get_io} > /dev/null
124    [ $? -ne 0 ] && get_out 3 || echo 'Success'
125    [ $( find -name '*.gz' -print -quit ) ] && gunzip *.gz
126
127    ls -lh > inputs_list.txt
128}
129
130diff_inputs() {
131    local dif file
132    local files_list='' mesg='Same' 
133
134    ## Simple diff
135    for file in 'inputs_list.txt' *namelist_* *.xml cpp_*; do
136   dif=''
137
138   ## Continue even if input file is not in here (see after)
139   if [ -e ${TRUS_STOR}/$file ]; then dif=$( diff -q $file ${TRUS_STOR}/$file ); else dif=0; fi
140
141   ## Pass over useless file omission in benckmark directory
142   [[ -n "$dif" && "$dif" != '0' ]] && { mesg='Different'; echo $dif; files_list+=$file' '; }
143    done
144
145    [ $mesg == 'Same' ] && echo $mesg
146    echo $mesg          \
147   >> ${FILE_INPT}
148
149    ## List different files for web comment
150    [ -n "${files_list}" ] && echo 'Inputs  : '${files_list}'differ<br>' \
151   >> temp_${FILE_NOTE}
152}
153
154job_pending() {
155    local outline=$( printf "%100s" ) time_elapsed=0 time_increment=30
156
157    sleep ${time_increment}
158
159    ## Append a log file while pending
160    while [[ $( eval ${TRUS_JSTA} ) && ${time_elapsed} -lt ${TRUS_TOUT} ]]; do
161   printf "\n%s\n" ${outline// /#}          \
162       >> computation.log
163   [ -n "${TRUS_JINF}" ] && eval ${JOB_INFO} \
164       >> computation.log
165   sleep ${time_increment}
166   time_elapsed=$(( ${time_elapsed} + ${time_increment} ))
167    done
168
169    sleep ${time_increment}
170
171    ## Kill remaining job & stop the test if it's too long
172    [ ${time_elapsed} -eq ${TRUS_TOUT} ] && { eval ${JOB_DELE} &> /dev/null; get_out 6; }
173}
174
175diff_results() {
176    local file
177    local files_list='' mesg='Same'
178
179    ## Simple diff
180    for file in 'ocean.output' *.stat; do
181   ## Stop if no benchmark files (ocean.output, eventual stat files)
182   [ ! -e ${TRUS_STOR}/$file ] && { TRUS_RSLT='FAILED'; get_out 7; }
183
184   diff -q $file ${TRUS_STOR}/$file
185
186   ## Continue even if it differs
187   [ $? -ne 0 ] && { TRUS_RSLT='FAILED'; mesg='Different'; files_list+=$file' '; }
188    done
189
190    [ $mesg == 'Same' ] && echo $mesg
191
192    ## List different files for web comment
193    [ -n "${files_list}" ] && echo 'Results : '${files_list}'differ<br>' \
194   >> temp_${FILE_NOTE}
195}
196
197diff_restart() {
198    local base_name comp dif file list_comp list_tmsp nb_dom time_step tmsp
199    local files_list='' dif_sum=0
200
201    ## Stop if no benchmark files (ie time.step)
202    [ ! -e ${TRUS_STOR}/time.step ] && { TRUS_RSLT='FAILED'; get_out 7; }
203    time_step=$( cat ${TRUS_STOR}/time.step | tr -d [:space:] )
204
205    ## Find all restart files to rebuild
206    if [ $( find -regex ".*_restart.*[0-9]\.nc" -print -quit ) ]; then
207   base_name=$( find -regex ".*_restart.*[0-9]\.nc"                       \
208                | sed "s#^\./\(.*\)_[0-9]*_restart.*#\1#"       | sort -u   )
209   list_comp=$( find -regex ".*_restart.*[0-9]\.nc"                       \
210                | sed "s#^.*\(restart[a-z_]*\)_[0-9].*\.nc#\1#" | sort -u   )
211   list_tmsp=$( find -regex ".*_restart.*[0-9]\.nc"                       \
212                | sed "s#^.*\([0-9]\{8\}\)_restart.*#\1#"       | sort -u   )
213
214   ## Loop on each time step
215   for tmsp in ${list_tmsp}; do
216
217       for comp in ${list_comp}; do
218      file=${base_name}_${tmsp}_${comp}
219      nb_dom=$( find -name "${file}_[0-9]*.nc" | wc -l | awk '{ print $1 }' )
220
221      if   [ ${nb_dom} -gt 1 ]; then
222          ${TRUS_NGCM}/TOOLS/REBUILD_NEMO/rebuild_nemo -t ${TRUS_NPRO} $file ${nb_dom} \
223         > /dev/null
224
225           ## Possibility of remaining decomposed restarts (even after rebuild)
226          [ $? -eq 0 ] && rm -f ${file}_[0-9]*.nc \
227                        > /dev/null
228
229      elif [ ${nb_dom} -eq 0 ]; then
230          TRUS_RSLT='FAILED' && get_out 8
231      fi
232
233      ## Compare restart files at same time step
234      if [ $tmsp -eq ${time_step} ]; then
235
236                    ## Stop if no benchmark files (restart file)
237          if [ -e ${TRUS_STOR}/$file.nc ]; then
238
239                   ## UNIX `cmp` not suitable (timestamp in .nc file)
240         dif=$( $TRUS_CDOD $file.nc ${TRUS_STOR}/$file.nc 2> /dev/null          \
241                | awk '/records/ {print $0}' | sed '2 s/^/,/' | tr -d '\n' )
242
243         ## CDO can return void stdout with no difference
244         if [[ -n "$dif" && $( echo $dif | awk '{print $1}' ) -ne 0 ]]; then
245             TRUS_RSLT='FAILED'
246             files_list+=$comp' ' && let dif_sum+=$( echo $dif | awk '{print $1}' )
247             echo $file.nc': '$dif
248         fi
249
250          else
251         TRUS_RSLT='FAILED' && get_out 7
252          fi
253
254      else
255          continue
256      fi
257
258       done
259
260   done
261
262        ## List different files for web comment with sum of different records
263   if [ ${dif_sum} -ne 0 ]; then
264       echo 'Restarts: '${files_list}${dif_sum}' record(s) differ<br>' \
265      >> temp_${FILE_NOTE}
266   else
267       echo 'Same'
268   fi
269
270    else
271   TRUS_RSLT='FAILED'
272    fi
273
274}
275
276get_time() {
277    [ -z "${TRUS_JTIM}" ] && return
278
279    ## Interest for checking unusual time computation
280    local time_cpu=$( eval ${TRUS_JTIM} )
281
282    printf "Elapsed time: "
283    echo ${time_cpu} | tee -a ${FILE_TIME}
284}
285
286get_memy() {
287    [[ -z "${TRUS_JPME}" && -z "${TRUS_JVME}" ]] && return
288
289    ## Interest for checking unusual memory usage
290    local memory_pmax=$( eval ${TRUS_JPME} ) memory_vmax=$( eval ${TRUS_JVME} )
291
292    printf "Memory max usage (physical/virtual): "
293    echo ${memory_pmax}' / '${memory_vmax} | tee -a ${FILE_MEMY}
294}
295
296comments() {
297    local opat
298    local line='' state=$1
299
300    if [ -e ocean.output ]; then
301        ## 'W A R N I N G' pattern by default
302   opat="-A2 \"^ $state\""
303   [ "$state" == 'E R R O R' ] && opat="-A4 \"$state\""
304
305        ## Select first occurence for web comment
306   line=$( eval grep -m1 $opat ocean.output | tr -d '\n' )
307    fi
308
309    [ -n "$line" ] && ( echo $line; printf "$line<br>" \
310   >> temp_${FILE_NOTE} )
311}
312
313log_make() {
314    ## Format comments for web
315    [ -e temp_${FILE_NOTE} ] && cat temp_${FILE_NOTE} | tr -d '\n' | sed 's/<br>$//' \
316   >> ${FILE_NOTE}
317
318    ## Construct txt file with all messenger files
319    paste -d ';' mesg_*.txt | tee ${TRUS_FILE}
320}
321
322prod_publish() {
323    local cmd
324    local rev=$( awk '/NEMOGCM/ {print $NF}' model.log )
325
326    ## Production mode (-p|--prod)
327    if [ ${TRUS_PROD} -eq 1 ]; then
328
329   ## Create or append trusting logfile
330   if [ -f ${TRUS_STOR}/trusting_$PATTERNAME.txt ]; then cmd='tail -1'; else cmd='cat'; fi
331
332   $cmd ${TRUS_FILE}                            \
333       >> ${TRUS_STOR}/trusting_$PATTERNAME.txt
334
335        ## Send mail only when FAILED
336   if [[ ! -z "${TRUS_MAIL}" && ${TRUS_RSLT} == 'FAILED' ]]; then
337
338       ## Content
339       cat <<END_MAIL      \
340      > trusting.mail
341Dear all,
342
343
344The trusting sequence has not completed successfully on new configuration ${TRUS_CONF} based on ${TRUS_REFE}.
345
346Here is the model summary:
347`cat model.log`
348
349First checking would be on the trusting environment files:
350${TRUS_USER}.cfg & ${TRUS_HPCC}.cfg
351
352For more details, look into the testing folder at:
353${TRUS_SCRA}
354
355An archive has been created to share the questionable configuration for further studies:
356${TRUS_STOR}/${TRUS_ARCH}
357
358END_MAIL
359
360       ## Send with detailed subject
361       mail -s "[NEMO Trusting][$rev][${TRUS_BRAN}][${TRUS_REFE}] ${TRUS_RSLT} ${TRUS_RORR}" ${TRUS_MAIL} \
362      <  trusting.mail
363   fi
364
365    fi
366}
367
368get_out() {
369    local time_step=0
370
371    TRUS_RORR=$1
372
373    printf "\n\nEnd of test\n"
374
375    ## In case of compilation error
376    cd ${TRUS_SCRA}
377
378    if [ ${TRUS_RSLT} == 'FAILED' ]; then
379   echo 'Failure'
380
381        ## Error identification
382   case ${TRUS_RORR} in
383            ## Compilation
384       '1') TRUS_RORR='XIOS compilation failed' ;; '2') TRUS_RORR='NEMO compilation failed';;
385       ## Submission
386       '3') TRUS_RORR='Missing input files'     ;; '4') TRUS_RORR='Job submission error'   ;;
387       ## Computation
388       '5') TRUS_RORR='Crashed at time step'    ;; '6') TRUS_RORR='Exceeded time limit'    ;;
389       ## Results
390       '7') TRUS_RORR='Missing previous outputs';; '8') TRUS_RORR='New outputs differ'     ;;
391       ## Other
392       '*') TRUS_RORR='Unknown error'           ;;
393   esac
394
395    else
396   echo 'Success' && TRUS_RORR='Code is reliable'
397    fi
398
399    ## Eventual comments from ocean.output
400    if [ "${TRUS_RORR}" == 'Crashed at time step' ]; then
401   comments 'E R R O R'
402   [ -e time.step ] && time_step=$( grep -o [0-9]* time.step )
403   TRUS_RORR+=' '$time_step
404    else
405   comments 'W A R N I N G'
406   [ "${TRUS_RORR}" == 'Exceeded time limit' ] && TRUS_RORR+=' '$(( ${TRUS_TOUT}/3600 ))'h'
407    fi
408
409    ## Last messenger files
410    #export TRUS_RORR
411    sed -i "2 s/.*/$TRUS_RSLT/" ${FILE_RSLT}; sed -i "2 s/.*/$TRUS_RORR/" ${FILE_STAT}
412
413    ## Save tested configuration if trusting failed in production mode (-p|--prod)
414    if [[ ${TRUS_RSLT} == 'FAILED' && ${TRUS_PROD} -eq 1 ]]; then
415   echo 'Creating archive '${TRUS_ARCH}' under '${TRUS_STOR}
416   tar -czf ${TRUS_STOR}/${TRUS_ARCH}               *                    \
417       -C   ${TRUS_NGCM}/CONFIG/${TRUS_CONF}/MY_SRC .                    \
418       -C   ${TRUS_NGCM}/CONFIG/${TRUS_CONF}        cpp_${TRUS_CONF}.fcm
419    fi
420
421    ## Logfile construct & eventual sending of notification email
422    printf "\nTrusting digest:\n----------------\n"
423    log_make
424    prod_publish
425
426    exit 0
427}
Note: See TracBrowser for help on using the repository browser.