Changeset 5689 for branches/2015
- Timestamp:
- 2015-08-14T18:05:48+02:00 (9 years ago)
- Location:
- branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST
- Files:
-
- 2 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh
r5685 r5689 8 8 NEMO_TRUS=$PWD 9 9 DEBUG=0; PUBLISH=0; TIME_STEP=0 10 I _MODULE=0; ST='FAILED' ## No 'module' command & 'FAILED' status10 IMOD=0; ST='FAILED' ## No 'module' command & 'FAILED' status 11 11 xios_full='--full'; stderr_redir='>&' ## DEBUG options 12 12 … … 50 50 ## Summary of trusting test context 51 51 ##--------------------------------------------------- 52 printf "\n\ 53 NEMO Trusting test:\n\ 54 -------------------\n\ 55 Testing ${TEST_CONF} based on ${REFE_CONF} from ${NEMO_BRAN}\n\ 56 NEMO installation: ${NEMO_USER}\n\ 57 Computing architecture: ${NEMO_HPCC}\n\ 58 Trusting SVN version: $( svn info | awk '(NR == 9) {print $NF}' )\n" 52 echo 'NEMO Trusting test:' 53 echo '-------------------' 54 echo 'Testing '${TEST_CONF}' based on '${REFE_CONF}' from '${NEMO_BRAN} 55 echo 'NEMO installation: '${NEMO_USER} 56 echo 'Computing architecture: '${NEMO_HPCC} 57 echo 'Trusting SVN version: '$( svn info | awk '(NR == 9) {print $NF}' ) 59 58 60 59 61 60 ## Making timestamped directory with messenger files 62 61 ##--------------------------------------------------- 63 print_step 'Time-stamp generation & testing directory creation' 64 mkdir -p ${TEST_DIR} ${REFE_DIR} 65 echo ${TEST_DIR} && cd ${TEST_DIR} 66 init_files && get_date62 cd ${TEST_DIR} 63 64 print_step 'Create timestamped testing directory' 65 echo ${TEST_DIR} && init_files && get_date 67 66 68 67 69 68 ## SVN action on XIOS & NEMO essentials directories 70 69 ##--------------------------------------------------- 71 print_step 'SVN command on NEMO directories:' 72 get_nemo_rev 70 print_step 'SVN command on NEMO directories:'; get_nemo_rev 73 71 74 72 … … 76 74 ##--------------------------------------------------- 77 75 print_step 'Get softwares releases' 76 78 77 [ -e ${ARCH_ENV} ] && . ${ARCH_ENV} > /dev/null 79 (( $? == 0 )) && export I_MODULE=180 get_soft_rel 81 cat model.log78 [ $? -eq 0 ] && export IMOD=1 79 80 get_soft_rel; cat model.log 82 81 83 82 84 83 ## XIOS compiling 85 84 ##--------------------------------------------------- 85 cd ${DIR_XIOS} 86 86 87 print_step 'Compile XIOS' 87 cd ${DIR_XIOS}88 88 [ $DEBUG -eq 1 ] && xios_full='' && stderr_redir='>' 89 89 eval ./make_xios ${xios_full} --arch ${NEMO_HPCC} --job $NPROC \ … … 93 93 ## NEMO config compiled from scratch 94 94 ##--------------------------------------------------- 95 print_step "Compile ${REFE_CONF} configuration"96 95 cd ${NEMO_CONF} 97 96 97 print_step "Compile ${REFE_CONF} configuration" 98 98 99 if [[ -d ${TEST_CONF} && $DEBUG -eq 0 ]]; then 99 eval./makenemo -n ${TEST_CONF} clean_config > /dev/null <<EOF100 ./makenemo -n ${TEST_CONF} clean_config > /dev/null <<EOF 100 101 y 101 102 EOF … … 103 104 104 105 eval ./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m ${NEMO_HPCC} -j $NPROC \ 105 ${KEYS_ADD} ${KEYS_DEL}\106 ${KEYS_ADD} ${KEYS_DEL} \ 106 107 ${stderr_redir} /dev/null 107 108 [ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out 2 … … 110 111 ## Get namelists, xml & forcing files for running 111 112 ##--------------------------------------------------- 113 cd ${TEST_DIR} 114 112 115 print_step 'Set job (copying & extracting input files)' 113 cd ${TEST_DIR}114 116 cp ${NEMO_CONF}/${TEST_CONF}/cpp_* . 115 117 find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(_cfg\|.in\|opa\|_ref\|.xml\)' -exec cp {} . \; 118 116 119 get_inputs 117 (( $? != 0 ))&& get_out 3120 [ $? -ne 0 ] && get_out 3 118 121 [ $( find . -name '*.gz' -print -quit ) ] && find . -name '*.gz' -exec gunzip {} \; 119 122 … … 121 124 ## Check input files in all cases 122 125 ##--------------------------------------------------- 123 print_step 'Compare with standard input files list' 124 diff_inputs 126 print_step 'Compare with benchmark input files list'; diff_inputs 125 127 126 128 127 129 ## Job submission & computing 128 130 ##--------------------------------------------------- 129 print_step 'Submit job' 130 JOB_ID=$( eval ${JOB_SUBM} ) 131 (( $? != 0 )) && get_out 4 132 print_step 'Pending job' 133 job_pending 131 print_step 'Submit job'; JOB_ID=$( eval ${JOB_SUBM} ) 132 [ $? -ne 0 ] && get_out 4 133 134 print_step 'Pending job'; job_pending 135 134 136 print_step 'Job finished' 135 137 136 138 137 ## Job state 139 ## Job state & get time computation if succeeded 138 140 ##--------------------------------------------------- 139 141 print_step 'Test job state' 140 142 141 if [[ ! -e time.step || $( grep -q 'E R R O R' ocean.output ) ]]; then 142 echo 'Error' 143 comments 'E R R O R' 143 if [[ ! -e time.step || $( grep 'E R R O R' ocean.output ) ]]; then 144 echo 'Failure' 144 145 get_out 5 145 146 else 146 echo 'Good' 147 ## Get time computation 148 ##--------------------------------------------------- 149 print_step 'Get real CPU time' 150 get_cpu_time 147 echo 'Success' 151 148 fi 152 149 150 print_step 'Get real CPU time'; get_cpu_time 151 152 153 ## Inspect outputs 154 ##--------------------------------------------------- 153 155 ST='OK' ## 'OK' by default 154 156 155 156 ## Inspect output text files 157 ##--------------------------------------------------- 158 print_step 'Test output files diff' 159 diff_results 160 161 162 ## Inspect output NetCDF files 163 ##--------------------------------------------------- 164 print_step 'Test last restart files diff' 165 diff_restart 157 print_step 'Compare new outputs to benchmark files'; diff_results; diff_restart 166 158 [ $ST == 'FAILED' ] && get_out 8 167 168 169 ## Get comments (ocean.output & diff model.log)170 ##---------------------------------------------------171 comments 'W A R N I N G'172 159 173 160 -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting_func.sh
r5685 r5689 1 1 #!/bin/bash 2 2 3 4 mkdir -p ${TEST_DIR} ${REFE_DIR} 3 5 4 6 ## Messenger filenames … … 26 28 27 29 # 'Failed' status with 'Unknown error' by default 28 echo $ST >> ${FILE_STAT}; echo 'Unknown error' >> ${FILE_RESU} 30 echo $ST \ 31 >> ${FILE_STAT} 32 echo 'Unknown error' \ 33 >> ${FILE_RESU} 29 34 } 30 35 … … 33 38 local dat=$( date -ud ${DATE} +"%F %R %Z" ) 34 39 35 echo $dat >> ${FILE_DATE} 40 echo $dat \ 41 >> ${FILE_DATE} 36 42 } 37 43 … … 60 66 if [ $dir == ${DIR_XIOS} ]; then 61 67 rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' ) 62 echo 'XIOS '${rev_loc} >> model.log 68 echo 'XIOS '${rev_loc} \ 69 >> model.log 63 70 echo "<a href=\"https://forge.ipsl.jussieu.fr/ioserver/changeset/${rev_loc}\" target=\"_blank\">${rev_loc}</a>" \ 64 71 >> ${FILE_XIOS} … … 69 76 ${svn_cmd} $dir 70 77 rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' ) 71 (( ${rev_loc} >= $rev )) && rev=${rev_loc} 72 done 73 74 echo 'NEMOGCM '$rev >> model.log 78 [ ${rev_loc} -gt $rev ] && rev=${rev_loc} 79 done 80 81 echo 'NEMOGCM '$rev \ 82 >> model.log 75 83 echo "<a href=\"https://forge.ipsl.jussieu.fr/nemo/changeset/$rev\" target=\"_blank\">$rev</a>" \ 76 84 >> ${FILE_NEMO} … … 84 92 soft_rel='' 85 93 86 if (( ${I_MODULE} == 1 )); then94 if [ $IMOD -eq 1 ]; then 87 95 soft_rel=$( echo $LOADEDMODULES | sed "s/.*$soft\/\([0-9.a-z_]*\).*/\1/i" ) 88 96 else … … 93 101 ## Cleaning characters string to display proper soft name 94 102 soft=$( echo $soft | sed 's#\\##g; s#[/-]$##' ) 95 echo $soft ${soft_rel} >> model.log 96 done 97 98 sed -n 4p model.log >> ${FILE_COMP}; sed -n 5p model.log >> ${FILE_MPIN} 99 sed -n 6p model.log >> ${FILE_NCDF} 103 echo $soft ${soft_rel} \ 104 >> model.log 105 done 106 107 sed -n 4p model.log \ 108 >> ${FILE_COMP} 109 sed -n 5p model.log \ 110 >> ${FILE_MPIN} 111 sed -n 6p model.log \ 112 >> ${FILE_NCDF} 100 113 } 101 114 … … 113 126 local files_list='' mesg='Same' 114 127 115 for file in inputs_list.txtnamelist_* *.xml cpp_*; do128 for file in 'inputs_list.txt' *namelist_* *.xml cpp_*; do 116 129 dif='' 117 118 if [ -e ${REFE_DIR}/$file ]; then 119 dif=$( diff -q $file ${REFE_DIR}/$file ) 120 else dif=0; fi 121 122 if [ -n "$dif" ]; then 123 mesg='Different' 124 [ "$dif" != '0' ] && echo $dif && files_list+=$file' ' 125 fi 126 127 done 128 129 echo $mesg | tee -a ${FILE_INPT} 130 if [ -e ${REFE_DIR}/$file ]; then dif=$( diff -q $file ${REFE_DIR}/$file ); else dif=0; fi 131 if [ -n "$dif" ]; then mesg='Different'; [ "$dif" != '0' ] && echo $dif && files_list+=$file' '; fi 132 done 133 134 [ $mesg == 'Same' ] && echo $mesg 135 echo $mesg \ 136 >> ${FILE_INPT} 130 137 [ -n "${files_list}" ] && echo 'Inputs : '${files_list}'differ<br>' \ 131 138 >> temp_${FILE_COMM} … … 139 146 140 147 while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do 141 printf "\n%s\n" ${outline// /#} >> computation.log 142 ${JOB_INFO} ${JOB_ID} >> computation.log 148 printf "\n%s\n" ${outline// /#} \ 149 >> computation.log 150 ${JOB_INFO} ${JOB_ID} \ 151 >> computation.log 143 152 sleep ${time_increment} 144 153 time_elapsed=$(( ${time_elapsed} + ${time_increment} )) … … 147 156 sleep ${time_increment} 148 157 149 if (( ${time_elapsed} == ${TIME_LIMI} )); then158 if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then 150 159 ${JOB_DELE} ${JOB_ID} &> /dev/null 151 let TIME_LIMIT/=3600 152 export ${TIME_LIMIT} 160 export TIME_LIMIT/=3600 153 161 get_out 6 154 162 fi … … 159 167 local files_list='' 160 168 161 for file in ocean.output*.stat; do169 for file in 'ocean.output' *.stat; do 162 170 [ ! -e ${REFE_DIR}/$file ] && export ST='FAILED' && get_out 7 163 171 diff -q $file ${REFE_DIR}/$file 164 (( $? != 0 ))&& export ST='FAILED' && files_list+=$file' '172 [ $? -ne 0 ] && export ST='FAILED' && files_list+=$file' ' 165 173 done 166 174 … … 192 200 nb_dom=$( find -name "${file}_[0-9]*.nc" | wc -l | awk '{ print $1 }' ) 193 201 194 if (( ${nb_dom} > 1 )); then202 if [ ${nb_dom} -gt 1 ]; then 195 203 ${NEMO_TRBD}/rebuild_nemo -t $NPROC $file ${nb_dom} > /dev/null 196 204 [ $? -eq 0 ] && rm -f ${file}_[0-9]*.nc > /dev/null 197 elif (( ${nb_dom} == 0 )); then205 elif [ ${nb_dom} -eq 0 ]; then 198 206 export ST='FAILED' && get_out 8 199 207 fi 200 208 201 209 ## Compare restart files at same time step 202 if (( $tmsp == ${TIME_STEP} )); then210 if [ $tmsp -eq ${TIME_STEP} ]; then 203 211 if [ -e ${REFE_DIR}/$file.nc ]; then 204 212 ## UNIX `cmp` not suitable (timestamp in .nc file) … … 206 214 | awk '/records/ {print $0}' | sed '2 s/^/,/' | tr -d '\n' ) 207 215 208 if [[ -n "$dif" && (( $( echo $dif | awk '{print $1}' ) -ne 0 ))]]; then216 if [[ -n "$dif" && $( echo $dif | awk '{print $1}' ) -ne 0 ]]; then 209 217 export ST='FAILED' && files_list+=$file' ' 210 218 let dift+=$( echo $dif | awk '{print $1}' ) … … 212 220 fi 213 221 214 else export ST='FAILED' && get_out 7; fi 215 216 else continue; fi 222 else 223 export ST='FAILED' && get_out 7 224 fi 225 226 else 227 continue 228 fi 217 229 218 230 done 219 231 done 220 232 221 if (( $dift != 0 )); then 222 echo 'Restarts: '${files_list}$dift' record(s) differ<br>' \ 223 >> temp_${FILE_COMM} 224 else echo 'No restart differ'; fi 225 226 else export ST='FAILED'; fi 233 if [ $dift -ne 0 ]; then 234 echo 'Restarts: '${files_list}$dift' record(s) differ<br>' | tee -a temp_${FILE_COMM} 235 else 236 echo 'No restart differ' 237 fi 238 239 else 240 export ST='FAILED' 241 fi 227 242 228 243 } … … 247 262 248 263 echo $line 249 printf "$line<br>" >> temp_${FILE_COMM} 264 printf "$line<br>" \ 265 >> temp_${FILE_COMM} 250 266 } 251 267 … … 267 283 if [ -f ${REFE_DIR}/trusting_$PATTERNAME.txt ]; then cmd='tail -1'; else cmd='cat'; fi 268 284 269 $cmd ${FILE_TRUS} >> ${REFE_DIR}/trusting_$PATTERNAME.txt 285 $cmd ${FILE_TRUS} \ 286 >> ${REFE_DIR}/trusting_$PATTERNAME.txt 270 287 271 288 ## Send mail only when FAILED … … 285 302 `cat ${TEST_DIR}/${FILE_TRUS}` 286 303 END_MAIL 287 # cat ${TEST_DIR}/${FILE_TRUS} \288 # >> trusting.mail289 304 mail -s "[NEMO Trusting ${NEMO_HPCC} ${REFE_CONF} ${NEMO_BRAN}] $ST $ERR" $EMAIL \ 290 305 < trusting.mail … … 319 334 fi 320 335 321 sed -i "2 s/.*/$ST/" 336 sed -i "2 s/.*/$ST/" ${FILE_STAT} && sed -i "2 s/.*/$ERR/" ${FILE_RESU} 322 337 323 338 ## Save tested configuration if trusting failed … … 328 343 fi 329 344 330 mesg_make 331 mesg_publish 345 if [ $ERR -eq 5 ]; then 346 print_step 'Comments'; comments 'E R R O R' 347 else 348 print_step 'Comments'; comments 'W A R N I N G' 349 fi 350 351 print_step 'Trusting outcome'; mesg_make; mesg_publish 332 352 333 353 exit 0
Note: See TracChangeset
for help on using the changeset viewer.