Changeset 5690 for branches/2015
- Timestamp:
- 2015-08-14T22:03:47+02:00 (9 years ago)
- Location:
- branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST
- Files:
-
- 4 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/config/X64_CURIE_cron.cfg
r5644 r5690 4 4 CDO='netcdf-utils-' 5 5 COMPILER='ifort' 6 MPI='bullxmpi \/'6 MPI='bullxmpi/' 7 7 NETCDF='netcdf-' 8 8 -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.env
r5683 r5690 2 2 3 3 4 ## Time-stamp & filename pattern 4 ## Patterns (time-stamp & filename) 5 ##-------------------------------------- 5 6 dat=$( date ) 7 ## UTC time zone for timestamping 6 8 export DATE=$( date -ud "$dat" +"%Y%m%d%Z%H%M" ) 7 9 export PATTERNAME=${NEMO_USER}'_'${NEMO_HPCC} 8 10 9 ## Sourc inguser configuration10 ##-------------------------------------- 11 ## Source user configuration 12 ##--------------------------------------------------- 11 13 . ${NEMO_TRUS}/config/${NEMO_USER}.cfg 12 14 export NEMO_HOME NEMO_BRAN … … 27 29 export NEMO_TCMP NEMO_TRBD 28 30 29 ## Sourc ingsuper-computer configuration30 ##-------------------------------------- 31 ## Source super-computer configuration 32 ##--------------------------------------------------- 31 33 . ${NEMO_TRUS}/config/${NEMO_HPCC}.cfg 32 34 export CDO COMPILER MPI NETCDF -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh
r5689 r5690 7 7 ##--------------------------------------------------- 8 8 NEMO_TRUS=$PWD 9 DEBUG=0; PUBLISH=0; TIME_STEP=0 10 IMOD=0; ST='FAILED' ## No 'module' command & 'FAILED' status 11 xios_full='--full'; stderr_redir='>&' ## DEBUG options 9 DEBUG=0; PUBLISH=0 10 IMOD=0; ST='FAILED' ## No modules system & 'FAILED' status 12 11 13 12 14 ## Get options for replacing someinitials settings13 ## Get options for replacing initials settings 15 14 ##--------------------------------------------------- 16 15 while [ $# -ne 0 ]; do … … 27 26 28 27 done 28 29 ## DEBUG options to speed up & expand verbosity of compilation 30 xios_full='--full'; stderr_redir='>&' 31 [ $DEBUG -eq 1 ] && xios_full='' && stderr_redir='>' 29 32 30 33 … … 48 51 49 52 50 ## Summary of trusting test context53 ## Display contextual summary of trusting test 51 54 ##--------------------------------------------------- 52 55 echo 'NEMO Trusting test:' … … 58 61 59 62 60 ## Mak ingtimestamped directory with messenger files63 ## Make timestamped directory with messenger files 61 64 ##--------------------------------------------------- 62 65 cd ${TEST_DIR} … … 66 69 67 70 68 ## SVN action on XIOS & NEMO essentials directories71 ## Get SVN revision on XIOS & NEMO essentials directories 69 72 ##--------------------------------------------------- 70 73 print_step 'SVN command on NEMO directories:'; get_nemo_rev 71 74 72 75 73 ## Check softwares versions ( sourcearch environment)76 ## Check softwares versions (after sourced arch environment) 74 77 ##--------------------------------------------------- 75 78 print_step 'Get softwares releases' … … 81 84 82 85 83 ## XIOS compil ing86 ## XIOS compilation from scratch 84 87 ##--------------------------------------------------- 85 88 cd ${DIR_XIOS} 86 89 87 90 print_step 'Compile XIOS' 88 [ $DEBUG -eq 1 ] && xios_full='' && stderr_redir='>'89 91 eval ./make_xios ${xios_full} --arch ${NEMO_HPCC} --job $NPROC \ 90 92 ${stderr_redir} /dev/null 91 93 [ ! -e ${DIR_XIOS}/lib/libxios.a ] && get_out 1 92 94 93 ## NEMO config compiled from scratch 95 96 ## NEMO compilation from scratch 94 97 ##--------------------------------------------------- 95 98 cd ${NEMO_CONF} … … 109 112 110 113 111 ## Get namelists, xml & forcing files for running114 ## Get all inputs for running 112 115 ##--------------------------------------------------- 113 116 cd ${TEST_DIR} 114 117 115 print_step 'Set job (copying & extracting input files)'118 print_step 'Set job (copying or extracting inputs)' 116 119 cp ${NEMO_CONF}/${TEST_CONF}/cpp_* . 117 120 find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(_cfg\|.in\|opa\|_ref\|.xml\)' -exec cp {} . \; … … 122 125 123 126 124 ## Check input files in all cases127 ## Check inputs 125 128 ##--------------------------------------------------- 126 print_step 'Compare with benchmark input files list'; diff_inputs129 print_step 'Compare inputs to benchmark files'; diff_inputs 127 130 128 131 129 ## Job submission & comput ing132 ## Job submission & computation 130 133 ##--------------------------------------------------- 131 134 print_step 'Submit job'; JOB_ID=$( eval ${JOB_SUBM} ) … … 137 140 138 141 139 ## Job state & get time computationif succeeded142 ## Check job state & get computing time if succeeded 140 143 ##--------------------------------------------------- 141 144 print_step 'Test job state' … … 151 154 152 155 153 ## Inspectoutputs156 ## Check outputs 154 157 ##--------------------------------------------------- 155 158 ST='OK' ## 'OK' by default … … 159 162 160 163 161 ## End 164 ## End, at least nothing has changed ;-) 162 165 ##--------------------------------------------------- 163 166 get_out 'Code is reliable' -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting_func.sh
r5689 r5690 12 12 FILE_COMM=mesg_11_comments_$PATTERNAME.txt; 13 13 14 ## T rusting summary & archive timestampedfilenames14 ## Timestamped outcome & archive filenames 15 15 FILE_TRUS=trusting_${DATE}_$PATTERNAME.txt; FILE_ARCH=trusting_${DATE}_$PATTERNAME.tar.gz 16 16 … … 27 27 echo 'Comments' > ${FILE_COMM} 28 28 29 # 'Failed' status with 'Unknown error' by default29 ## 'Failed' status with 'Unknown error' by default 30 30 echo $ST \ 31 31 >> ${FILE_STAT} … … 36 36 37 37 get_date() { 38 local dat=$( date -ud ${DATE} +"%F %R %Z" ) 38 ## UTC time zone for timestamping 39 local dat=$( date -ud "${DATE}" +"%F %R %Z" ) 39 40 40 41 echo $dat \ … … 58 59 fi 59 60 61 ## Loop on essential directories 60 62 for dir in ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} \ 61 63 ${NEMO_EAGR} ${NEMO_EIOI} ${NEMO_EFCM} \ … … 63 65 ${DIR_XIOS} ; do 64 66 65 ## For time being, just get revision number from XIOS67 ## For time being, just get revision from XIOS with no action on directory 66 68 if [ $dir == ${DIR_XIOS} ]; then 67 69 rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' ) … … 76 78 ${svn_cmd} $dir 77 79 rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' ) 80 81 ## Keep last rev. nb 78 82 [ ${rev_loc} -gt $rev ] && rev=${rev_loc} 79 83 done … … 92 96 soft_rel='' 93 97 98 ## Software strings to identify depends on use (sed is case-insensitive) 94 99 if [ $IMOD -eq 1 ]; then 95 soft_rel=$( echo $LOADEDMODULES | sed "s/.*$soft\/\([0-9.a-z_]*\).*/\1/i" ) 100 ## Modules system: next word after "$soft/" 101 soft_rel=$( echo $LOADEDMODULES | sed "s#.*$soft/\([0-9.a-z_]*\).*#\1#i" ) 102 ## LOADEDMODULES=netcdf/mpi/4.1.3:cdo/1.5.9:intel/2013.0 103 ## LOADEDMODULES=bullxmpi/1.2.8.2:netcdf/4.3.3.1_hdf5_parallel:cdo/1.6.7 96 104 else 97 soft_rel=$( echo $PATH | sed "s/.*$soft\([0-9.a-z_]*\).*/\1/i" ) 98 fi 99 105 ## Hard-coded paths: next word after "$soft" 106 soft_rel=$( echo $PATH | sed "s#.*$soft\([0-9.a-z_]*\).*#\1#i" ) 107 ## PATH=/smplocal/intel/impi/4.1.0.024/intel64/bin:/smplocal/pub/cdo/1.5.9/bin:/smplocal/pub/NetCDF/4.1.3/mpi/bin 108 ## PATH=/usr/local/netcdf-4.3.3.1_hdf5_parallel/bin:/opt/mpi/bullxmpi/1.2.8.2/bin:/usr/local/netcdf-utils-4.3.3.1_hdf5/bin 109 fi 110 111 ## option --version would work for main compilers (gfortran, intel, pgfortran, ...) 100 112 [ $soft == $COMPILER ] && soft_rel=$( $soft --version | grep -m1 -oe '\<[0-9. ]*\>' ) 113 101 114 ## Cleaning characters string to display proper soft name 102 115 soft=$( echo $soft | sed 's#\\##g; s#[/-]$##' ) 116 103 117 echo $soft ${soft_rel} \ 104 118 >> model.log … … 115 129 116 130 get_inputs() { 131 # List archive content & extract by default 117 132 local cmd_iol="tar -tvf ${NEMO_FORC}/${NEMO_TARF}" cmd_iof="tar -vxf ${NEMO_FORC}/${NEMO_TARF}" 118 133 134 ## List & copy file in case of personal inputs 119 135 [ -z "${NEMO_TARF}" ] && cmd_iol="ls ${NEMO_FORC}/*" && cmd_iof="\cp ${NEMO_FORC}/* ." 120 ${cmd_iol} > inputs_list.txt && ${cmd_iof} > /dev/null 136 137 ${cmd_iol} > inputs_list.txt 138 ${cmd_iof} > /dev/null 121 139 } 122 140 … … 126 144 local files_list='' mesg='Same' 127 145 146 ## Simple diff 128 147 for file in 'inputs_list.txt' *namelist_* *.xml cpp_*; do 129 148 dif='' 149 150 ## Continue even if input file is not in here (see after) 130 151 if [ -e ${REFE_DIR}/$file ]; then dif=$( diff -q $file ${REFE_DIR}/$file ); else dif=0; fi 131 if [ -n "$dif" ]; then mesg='Different'; [ "$dif" != '0' ] && echo $dif && files_list+=$file' '; fi 152 153 ## Pass over useless file omission in benckmark directory 154 if [ -n "$dif" && "$dif" != '0' ]; then 155 mesg='Different'; echo $dif; files_list+=$file' ' 156 fi 157 132 158 done 133 159 … … 135 161 echo $mesg \ 136 162 >> ${FILE_INPT} 163 164 ## List different files for web comment 137 165 [ -n "${files_list}" ] && echo 'Inputs : '${files_list}'differ<br>' \ 138 166 >> temp_${FILE_COMM} … … 145 173 sleep ${time_increment} 146 174 175 ## Append a log file while pending 147 176 while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do 148 177 printf "\n%s\n" ${outline// /#} \ … … 156 185 sleep ${time_increment} 157 186 187 ## Kill remaining job & stop the test if it's too long 158 188 if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then 159 189 ${JOB_DELE} ${JOB_ID} &> /dev/null 160 export TIME_LIMIT/=3600161 190 get_out 6 162 191 fi … … 167 196 local files_list='' 168 197 198 ## Simple diff 169 199 for file in 'ocean.output' *.stat; do 200 201 ## Stop if no benchmark files (ocean.output, eventual stat files) 170 202 [ ! -e ${REFE_DIR}/$file ] && export ST='FAILED' && get_out 7 203 171 204 diff -q $file ${REFE_DIR}/$file 205 206 ## Continue even if it differs 172 207 [ $? -ne 0 ] && export ST='FAILED' && files_list+=$file' ' 173 208 done 174 209 210 ## List different files for web comment 175 211 [ -n "${files_list}" ] && echo 'Results : '${files_list}'differ<br>' \ 176 212 >> temp_${FILE_COMM} … … 182 218 local files_list='' dift=0 183 219 220 ## Stop if no benchmark files (time.step) 184 221 [ ! -e ${REFE_DIR}/time.step ] && export ST='FAILED' && get_out 7 185 export TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 186 echo 'Last time step of benchmark run: '${TIME_STEP} 222 223 time_step=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 224 echo 'Last time step of benchmark run: '${time_step} 187 225 188 226 ## Find all restart files to rebuild … … 195 233 | sed "s#^.*\([0-9]\{8\}\)_restart.*#\1#" | sort -u ) 196 234 235 ## Loop on each time step 197 236 for tmsp in ${list_tmsp}; do 198 237 for comp in ${list_comp}; do … … 208 247 209 248 ## Compare restart files at same time step 210 if [ $tmsp -eq ${TIME_STEP} ]; then 249 if [ $tmsp -eq ${time_step} ]; then 250 251 ## Stop if no benchmark files (restart file) 211 252 if [ -e ${REFE_DIR}/$file.nc ]; then 253 212 254 ## UNIX `cmp` not suitable (timestamp in .nc file) 213 255 dif=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \ 214 256 | awk '/records/ {print $0}' | sed '2 s/^/,/' | tr -d '\n' ) 215 257 258 ## CDO can return no output on stdout with no difference 216 259 if [[ -n "$dif" && $( echo $dif | awk '{print $1}' ) -ne 0 ]]; then 217 260 export ST='FAILED' && files_list+=$file' ' … … 231 274 done 232 275 276 ## List different files for web comment with sum of different parameters 233 277 if [ $dift -ne 0 ]; then 234 278 echo 'Restarts: '${files_list}$dift' record(s) differ<br>' | tee -a temp_${FILE_COMM} … … 245 289 246 290 get_cpu_time() { 291 ## Interest 247 292 local real_cpu_time=$( eval ${JOB_TIME} ) 248 293 … … 252 297 253 298 comments() { 299 local opt 254 300 local state=$1 255 301 256 if [ "$state" == 'E R R O R' ]; then257 [ -e time.step ] && export TIME_STEP=$( cat time.step | tr -d ' ' ) 258 [ -e ocean.output ] && line=$( grep -m1 -A4 "$state" ocean.output | tr -d '\n' ) 259 else 260 [ -e ocean.output ] && line=$( grep -m1 -A2 "^ $state" ocean.output | tr -d '\n' ) 261 fi302 ## 'W A R N I N G' by default 303 opt='-A2 "^ $state"' 304 [ "$state" == 'E R R O R' ] && opt="-A4 $state" 305 306 ## Select first occurence for web comment 307 line=$( grep -m1 $opt ocean.output | tr -d '\n' ) 262 308 263 309 echo $line … … 272 318 >> ${FILE_COMM} 273 319 320 ## Construct txt file for web with all messenger files 274 321 paste -d ';' mesg_*.txt | tee ${FILE_TRUS} 275 322 } … … 279 326 local cmd 280 327 328 ## Production mode (-p|--publish) 281 329 if [ $PUBLISH -eq 1 ]; then 282 330 331 ## Create or append historical trusting file 283 332 if [ -f ${REFE_DIR}/trusting_$PATTERNAME.txt ]; then cmd='tail -1'; else cmd='cat'; fi 284 333 … … 286 335 >> ${REFE_DIR}/trusting_$PATTERNAME.txt 287 336 288 ## Send mail only when FAILED337 ## Send mail only when FAILED 289 338 if [[ ! -z "$EMAIL" && $ST == 'FAILED' ]]; then 339 340 ## Content 290 341 cat <<END_MAIL \ 291 342 > trusting.mail … … 302 353 `cat ${TEST_DIR}/${FILE_TRUS}` 303 354 END_MAIL 355 356 ## Send with detailed subject 304 357 mail -s "[NEMO Trusting ${NEMO_HPCC} ${REFE_CONF} ${NEMO_BRAN}] $ST $ERR" $EMAIL \ 305 358 < trusting.mail … … 320 373 case $ERR in 321 374 ## Compilation 322 1) ERR='XIOS compilation failed' ;; 2) ERR='NEMO compilation failed';;375 1) ERR='XIOS compilation failed' ;; 2) ERR='NEMO compilation failed';; 323 376 ## Submission 324 3) ERR='Missing input files' ;; 4) ERR='Job submission error';;377 3) ERR='Missing input files' ;; 4) ERR='Job submission error' ;; 325 378 ## Running 326 5) ERR='Crashed at time step '${TIME_STEP};; 6) ERR='Exceeded time limit '${TIME_LIMI}'h';;379 5) ERR='Crashed at time step' ;; 6) ERR='Exceeded time limit' ;; 327 380 ## Results 328 7) ERR='Missing previous outputs' ;; 8) ERR='New outputs differ';;381 7) ERR='Missing previous outputs';; 8) ERR='New outputs differ' ;; 329 382 ## Undefined 330 *) ERR='Unknown error' 383 *) ERR='Unknown error' ;; 331 384 esac 332 385 333 export ERR 334 fi 335 336 sed -i "2 s/.*/$ST/" ${FILE_STAT} && sed -i "2 s/.*/$ERR/" ${FILE_RESU} 337 338 ## Save tested configuration if trusting failed 386 fi 387 388 ## Eventual comments from ocean.output 389 print_step 'Comments' 390 if [ -e ocean.output ]; then 391 if [ $ERR -eq 5 ]; then 392 comments 'E R R O R' 393 [ -e time.step ] && ERR+=' '$( cat time.step | tr -d ' ' ) 394 else 395 comments 'W A R N I N G' 396 [ $ERR -eq 6 ] && ERR+=' '$(( ${TIME_LIMIT}/3600 ))'h' 397 fi 398 fi 399 400 ## Share trusting result (locally or not) 401 export ERR 402 sed -i "2 s/.*/$ST/" ${FILE_STAT} 403 sed -i "2 s/.*/$ERR/" ${FILE_RESU} 404 405 ## Save tested configuration if trusting failed in production mode (-p|--publish) 339 406 if [[ $ST == 'FAILED' && $PUBLISH -eq 1 && $DEBUG -eq 0 ]]; then 340 407 echo 'Creating archive '${FILE_ARCH}' under '${REFE_DIR} … … 343 410 fi 344 411 345 if [ $ERR -eq 5 ]; then 346 print_step 'Comments'; comments 'E R R O R' 347 else 348 print_step 'Comments'; comments 'W A R N I N G' 349 fi 350 412 ## 351 413 print_step 'Trusting outcome'; mesg_make; mesg_publish 352 414
Note: See TracChangeset
for help on using the changeset viewer.