- Timestamp:
- 2015-07-29T20:33:51+02:00 (9 years ago)
- File:
-
- 1 edited
Legend:
- Unmodified
- Added
- Removed
-
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting_func.sh
r5638 r5644 2 2 3 3 4 comments() { 5 state=$1; LAST_TIME_STEP=0 6 7 if [ "$state" == 'E R R O R' ]; then 8 [ -e time.step ] && LAST_TIME_STEP=$( cat time.step | tr -d ' ' ) 9 export LAST_TIME_STEP 10 [ -e ocean.output ] && line=$( grep -m1 -A4 "$state" ocean.output | tr -d '\n' ) 11 else 12 [ -e ocean.output ] && line=$( grep -m1 -A2 "^ $state" ocean.output | tr -d '\n' ) 13 fi 14 15 # if [[ -e ${REFE_DIR}/model.log && $( diff -q model.log ${REFE_DIR}/model.log ) ]]; then 16 # line=$( diff model.log ${REFE_DIR}/model.log | grep '>' | sed '{N;s/\n/<BR>/}' ) 17 # fi 18 19 printf "Comments\n$line\n" | tee mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt 20 } 21 22 23 diff_inputs() { 24 mesg='Same' 25 for file in inputs_list.txt $( ls namelist_* ) $( ls *.xml ); do 26 diff -q $file ${REFE_DIR}/$file > /dev/null 27 [ $? -ne 0 ] && mesg='Different' 28 done 29 30 printf "Input files\n$mesg\n" | tee mesg_09_inputs_${CFG_USER}_${CFG_ARCH}.txt 31 } 32 33 diff_results() { 34 for file in ocean.output $( ls *.stat ); do 35 [ ! -e ${REFE_DIR}/$file ] && export ST='FAILED' && get_out 7 36 diff -q $file ${REFE_DIR}/$file 37 [ $? -ne 0 ] && export ST='FAILED' 38 done 39 } 40 41 diff_restart() { 42 [ ! -e ${REFE_DIR}/time.step ] && export ST='FAILED' && get_out 9 43 44 export LAST_TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 45 echo 'Last time step of standard run: '${LAST_TIME_STEP} 46 47 if [ $( find -regex ".*_0+${LAST_TIME_STEP}_restart.*\.nc" -print -quit ) ]; then 48 base_name=$( find -regex ".*_0+${LAST_TIME_STEP}_restart.*\.nc" -print -quit | awk -F/ '{print $NF}' \ 49 | sed "s/^\(.*\)$LAST_TIME_STEP\_restart.*/\1$LAST_TIME_STEP\_/" ) 50 51 for comp in restart restart_ice restart_trc; do 52 file=${base_name}${comp} && printf "$file.nc: " 53 54 nb_dom=$( find -name "${file}_[0-9]*.nc" | wc -l | awk '{print $1}' ) 55 if [ ${nb_dom} -gt 1 ]; then 56 ${NEMO_TRBD}/rebuild_nemo -t $NPROC $file ${nb_dom} > /dev/null 57 [ $? -eq 0 ] && rm -f ${base_name}${comp}_[0-9]*.nc > /dev/null 58 elif [ ${nb_dom} -eq 0 ]; then 59 export ST='FAILED' && get_out 10 60 fi 61 62 if [ -e ${REFE_DIR}/$file.nc ]; then 63 ## UNIX `cmp` not suitable (filename & timestamp in .nc file) 64 nc_diff=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \ 65 | awk '/records/ {print $1}' ) 66 67 if [[ ! -z ${nc_diff} && ${nc_diff} -ne 0 ]]; then 68 export ST='FAILED' 69 printf "$CDO ${nc_diff}\n" 70 else 71 printf "identical\n" 72 fi 73 74 else 75 export ST='FAILED' && get_out 9 76 fi 77 78 done 79 80 else 81 export ST='FAILED' 82 fi 83 84 } 85 86 get_cpu_time() { 87 real_cpu_time=$( eval ${JOB_TIME} ) 88 printf "Real CPU time\n${real_cpu_time}\n" | tee mesg_10_cputime_${CFG_USER}_${CFG_ARCH}.txt 89 } 90 91 get_inputs() { 92 if [ ! -z "${FORC_TARF}" ]; then 93 cmd_iol="tar -tvf ${NEMO_FORC}/${FORC_TARF}"; cmd_iof="tar -vxf ${NEMO_FORC}/${FORC_TARF}" 94 else 95 cmd_iol="ls ${NEMO_FORC}/*" ; cmd_iof="\cp ${NEMO_FORC}/* ." 96 fi 97 98 ${cmd_iol} > inputs_list.txt && ${cmd_iof} > /dev/null 99 } 100 101 get_soft_rel() { 102 for soft in $CDO $COMPILER $MPI $NETCDF; do 103 soft_rel='' 104 (( ${I_MODULE} == 1 )) && soft_rel=$( echo $LOADEDMODULES | sed "s/.*$soft\([0-9.a-z_]*\).*/\1/i" ) 105 [ -z "${soft_rel}" ] && soft_rel=$( echo $PATH | sed "s/.*$soft\([0-9.a-z_]*\).*/\1/i" ) 106 [ "$soft" == "$COMPILER" ] && soft_rel=$( $soft --version | grep -m1 -oe '\<[0-9. ]*\>' ) 107 # Cleaning characters string to display proper soft name 108 soft=$( echo $soft | sed 's#\\##; s#[/-]$##' ) 109 echo $soft ${soft_rel} >> model.log 110 done 111 112 sed -n 4p model.log | sed 'i\Fortran compiler' > mesg_06_compiler_${CFG_USER}_${CFG_ARCH}.txt 113 sed -n 5p model.log | sed 'i\MPI libs' > mesg_07_mpi_${CFG_USER}_${CFG_ARCH}.txt 114 sed -n 6p model.log | sed 'i\NetCDF libs' > mesg_08_netcdf_${CFG_USER}_${CFG_ARCH}.txt 4 ## Messenger filenames 5 FILE_DATE=mesg_01_date_$PATTERNAME.txt ; FILE_STAT=mesg_02_status_$PATTERNAME.txt 6 FILE_RESU=mesg_03_result_$PATTERNAME.txt ; FILE_NEMO=mesg_04_nemogcm_$PATTERNAME.txt 7 FILE_XIOS=mesg_05_xios_$PATTERNAME.txt ; FILE_COMP=mesg_06_compiler_$PATTERNAME.txt 8 FILE_MPIN=mesg_07_mpi_$PATTERNAME.txt ; FILE_NCDF=mesg_08_netcdf_$PATTERNAME.txt 9 FILE_INPT=mesg_09_inputs_$PATTERNAME.txt ; FILE_CPUT=mesg_10_cputime_$PATTERNAME.txt 10 FILE_COMM=mesg_11_comments_$PATTERNAME.txt; 11 12 ## 13 FILE_TRUS=trusting_${DATE}_$PATTERNAME.txt; FILE_ARCH=trusting_${DATE}_$PATTERNAME.tar.gz 14 15 print_step() { printf "\nStep.....\n$1\n"; } 16 17 init_files() { 18 echo 'Date' > ${FILE_DATE}; echo 'Status' > ${FILE_STAT} 19 echo 'Result' > ${FILE_RESU}; echo 'NEMOGCM rev.' > ${FILE_NEMO} 20 echo 'XIOS rev.' > ${FILE_XIOS}; echo 'Fortran compiler' > ${FILE_COMP} 21 echo 'MPI libs' > ${FILE_MPIN}; echo 'NetCDF libs' > ${FILE_NCDF} 22 echo 'Input files' > ${FILE_INPT}; echo 'Real CPU time' > ${FILE_CPUT} 23 echo 'Comments' > ${FILE_COMM} 24 } 25 26 get_date() { 27 local dat=$( date -ud ${DATE} +"%F %R %Z" ) 28 29 echo $dat >> ${FILE_DATE} 115 30 } 116 31 117 32 get_nemo_rev() { 118 svn_opt='status' 33 local rev_loc svn_cmd 34 local rev=0 svn_opt='status' 119 35 120 36 ## If -v|--version option has been set, modify svn command … … 129 45 fi 130 46 131 rev=0132 47 for dir in ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} \ 133 48 ${NEMO_EIOI} ${NEMO_EFCM} \ … … 138 53 if [ "$dir" == "${DIR_XIOS}" ]; then 139 54 rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' ) 140 echo 'XIOS '${rev_loc} >> model.log 141 printf "XIOS rev.\n" > mesg_05_xios_${CFG_USER}_${CFG_ARCH}.txt 142 printf "<a href=\"https://forge.ipsl.jussieu.fr/ioserver/changeset/${rev_loc}\" target=\"_blank\">${rev_loc}</a>" \ 143 >> mesg_05_xios_${CFG_USER}_${CFG_ARCH}.txt 55 echo 'XIOS '${rev_loc} >> model.log 56 echo "<a href=\"https://forge.ipsl.jussieu.fr/ioserver/changeset/${rev_loc}\" target=\"_blank\">${rev_loc}</a>" \ 57 >> ${FILE_XIOS} 144 58 continue 145 59 fi 146 60 147 echo ${svn_cmd} $dir': '$( ${svn_cmd} $dir ) 61 echo $dir 62 ${svn_cmd} $dir 148 63 rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' ) 149 64 (( ${rev_loc} >= $rev )) && rev=${rev_loc} 150 65 done 151 66 152 echo 'NEMOGCM '$rev >> model.log 153 printf "NEMOGCM rev.\n" > mesg_04_nemogcm_${CFG_USER}_${CFG_ARCH}.txt 154 printf "<a href=\"https://forge.ipsl.jussieu.fr/nemo/changeset/$rev\" target=\"_blank\">$rev</a>" \ 155 >> mesg_04_nemogcm_${CFG_USER}_${CFG_ARCH}.txt 67 echo 'NEMOGCM '$rev >> model.log 68 echo "<a href=\"https://forge.ipsl.jussieu.fr/nemo/changeset/$rev\" target=\"_blank\">$rev</a>" \ 69 >> ${FILE_NEMO} 70 } 71 72 get_soft_rel() { 73 local soft_rel 74 75 for soft in $CDO $COMPILER $MPI $NETCDF; do 76 soft_rel='' 77 (( ${I_MODULE} == 1 )) && soft_rel=$( echo $LOADEDMODULES | sed "s/.*$soft\/\([0-9.a-z_]*\).*/\1/i" ) 78 [ -z "${soft_rel}" ] && soft_rel=$( echo $PATH | sed "s/.*$soft\([0-9.a-z_]*\).*/\1/i" ) 79 [ "$soft" == "$COMPILER" ] && soft_rel=$( $soft --version | grep -m1 -oe '\<[0-9. ]*\>' ) 80 ## Cleaning characters string to display proper soft name 81 soft=$( echo $soft | sed 's#\\##g; s#[/-]$##' ) 82 echo $soft ${soft_rel} >> model.log 83 done 84 85 sed -n 4p model.log > ${FILE_COMP}; sed -n 5p model.log > ${FILE_MPIN}; sed -n 6p model.log > ${FILE_NCDF} 86 } 87 88 get_inputs() { 89 local cmd_iol="tar -tvf ${NEMO_FORC}/${NEMO_TARF}"; cmd_iof="tar -vxf ${NEMO_FORC}/${NEMO_TARF}" 90 91 [ -z "${NEMO_TARF}" ] && cmd_iol="ls ${NEMO_FORC}/*" && cmd_iof="\cp ${NEMO_FORC}/* ." 92 ${cmd_iol} > inputs_list.txt && ${cmd_iof} > /dev/null 93 } 94 95 diff_inputs() { 96 local dif 97 local mesg='Same' 98 99 for file in inputs_list.txt $( ls namelist_* ) $( ls *.xml ); do 100 dif='' 101 if [ -e ${REFE_DIR}/$file ]; then dif=$( diff -q $file ${REFE_DIR}/$file ); else dif=0; fi 102 if [ -n "$dif" ]; then mesg='Different'; [ "$dif" != '0' ] && echo $dif | tee -a ${FILE_COMM}; fi 103 done 104 105 echo $mesg | tee -a ${FILE_INPT} 106 } 107 108 job_pending() { 109 local time_elapsed=0; time_increment=30 110 111 sleep ${time_increment} 112 113 while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do 114 printf "\n####################################################\n" >> computation.log 115 ${JOB_INFO} ${JOB_ID} >> computation.log 116 sleep ${time_increment} 117 time_elapsed=$(( ${time_elapsed} + ${time_increment} )) 118 done 119 120 sleep ${time_increment} 121 122 if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then 123 ${JOB_DELE} ${JOB_ID} &> /dev/null 124 TIME_LIMIT=$(( ${TIME_LIMI} / 3600 )) 125 # export TIME_LIMIT 126 get_out 6 127 fi 128 } 129 130 diff_results() { 131 for file in ocean.output $( ls *.stat ); do 132 [ ! -e ${REFE_DIR}/$file ] && export ST='FAILED' && get_out 7 133 diff -q $file ${REFE_DIR}/$file | tee -a ${FILE_COMM} 134 (( $? != 0 )) && export ST='FAILED' 135 done 136 } 137 138 diff_restart() { 139 local base_name file nb_dom diff_rcd 140 141 [ ! -e ${REFE_DIR}/time.step ] && export ST='FAILED' && get_out 9 142 143 # export LAST_TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 144 LAST_TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 145 echo 'Last time step of standard run: '${LAST_TIME_STEP} 146 147 if [ $( find -regex ".*_0+${LAST_TIME_STEP}_restart.*\.nc" -print -quit ) ]; then 148 base_name=$( find -regex ".*_0+${LAST_TIME_STEP}_restart.*\.nc" -print -quit | awk -F/ '{print $NF}' \ 149 | sed "s/^\(.*\)$LAST_TIME_STEP\_restart.*/\1$LAST_TIME_STEP\_/" ) 150 151 for comp in restart restart_ice restart_trc; do 152 file=${base_name}${comp} && printf "$file.nc: " 153 154 nb_dom=$( find -name "${file}_[0-9]*.nc" | wc -l | awk '{print $1}' ) 155 if [ ${nb_dom} -gt 1 ]; then 156 ${NEMO_TRBD}/rebuild_nemo -t $NPROC $file ${nb_dom} > /dev/null 157 [ $? -eq 0 ] && rm -f ${base_name}${comp}_[0-9]*.nc > /dev/null 158 elif [ ${nb_dom} -eq 0 ]; then 159 export ST='FAILED' && get_out 10 160 fi 161 162 if [ -e ${REFE_DIR}/$file.nc ]; then 163 ## UNIX `cmp` not suitable (filename & timestamp in .nc file) 164 # diff_rcd=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \ 165 # | awk '/records/ {print $1}' ) 166 diff_rcd=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \ 167 | awk '/records/ {print $0}' ) 168 169 # if [[ ! -z ${diff_rcd} && ${diff_rcd} -ne 0 ]]; then 170 if [[ ! -z ${diff_rcd} && $( echo ${diff_rcd} | awk '{print $1}') -ne 0 ]]; then 171 export ST='FAILED' 172 echo $CDO $( echo ${diff_rcd} | sed 's/differ$/differ,/' | tr -d '\n' ) 173 # printf "$CDO ${diff_rcd}\n" 174 else 175 echo 'identical' 176 # printf "identical\n" 177 fi 178 179 else 180 export ST='FAILED' && get_out 9 181 fi 182 183 done 184 185 else 186 export ST='FAILED' 187 fi 188 189 } 190 191 get_cpu_time() { 192 local real_cpu_time=$( eval ${JOB_TIME} ) 193 194 echo ${real_cpu_time} | tee -a ${FILE_CPUT} 195 } 196 197 comments() { 198 local state=$1 199 200 if [ "$state" == 'E R R O R' ]; then 201 [ -e time.step ] && LAST_TIME_STEP=$( cat time.step | tr -d ' ' ) 202 # export LAST_TIME_STEP 203 [ -e ocean.output ] && line=$( grep -m1 -A4 "$state" ocean.output | tr -d '\n' ) 204 echo $line >> ${FILE_COMM} 205 else 206 [ -e ocean.output ] && line=$( grep -m1 -A2 "^ $state" ocean.output | tr -d '\n' ) 207 fi 208 209 echo $line 156 210 } 157 211 158 212 mesg_make() { 159 paste -d ';' mesg_* _${CFG_USER}_${CFG_ARCH}.txt > trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt160 cat trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt213 paste -d ';' mesg_*.txt > ${FILE_TRUS} 214 cat ${FILE_TRUS} 161 215 } 162 216 163 217 mesg_publish() { 164 165 if [ $PUBLISH -eq 1 ]; then 166 167 if [ -f ${REFE_DIR}/trusting_${CFG_USER}_${CFG_ARCH}.txt ]; then cmd='tail -1'; else cmd='cat'; fi 168 169 $cmd trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt >> ${REFE_DIR}/trusting_${CFG_USER}_${CFG_ARCH}.txt 218 local cmd 219 220 if [ $PUBLISH -eq 1 ]; then 221 222 if [ -f ${REFE_DIR}/trusting_$PATTERNAME.txt ]; then cmd='tail -1'; else cmd='cat'; fi 223 224 $cmd ${FILE_TRUS} >> ${REFE_DIR}/trusting_$PATTERNAME.txt 170 225 171 226 ## Send mail only when FAILED 172 173 227 if [[ ! -z $EMAIL && "$ST" == 'FAILED' ]]; then 228 cat << END_MAIL > trusting.mail 174 229 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 175 230 … … 178 233 The trusting sequence for ${REFE_CONF} has failed. 179 234 Directory: ${TEST_DIR} 180 Archive created: trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.tar.gz235 Archive created: ${FILE_ARCH} in ${REFE_DIR} 181 236 182 237 XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX 183 238 184 239 END_MAIL 185 186 if [ -e ${TEST_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt ]; then 187 cat ${TEST_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt >> trusting.mail 188 fi 189 mail -s "[trusting ${REFE_CONF} $BRANCH] $ST $ERR" $EMAIL < trusting.mail 190 fi 191 192 fi 193 } 194 195 print_step() { printf "Step.....\n$1\n"; } 240 [ -e ${TEST_DIR}/${FILE_TRUS} ] && cat ${TEST_DIR}/${FILE_TRUS} >> trusting.mail 241 mail -s "[trusting ${REFE_CONF} $BRANCH] $ST $ERR" $EMAIL < trusting.mail 242 fi 243 244 fi 245 } 196 246 197 247 get_out() { … … 199 249 200 250 cd ${TEST_DIR} 201 printf "Status\n$ST\n" > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt251 echo $ST >> ${FILE_STAT} 202 252 203 253 if [ "$ST" == 'FAILED' ]; then 204 file=mesg_09_inputs_${CFG_USER}_${CFG_ARCH}.txt205 [ ! -e $file ] && printf "Input files\n\n" > $file206 file=mesg_10_cputime_${CFG_USER}_${CFG_ARCH}.txt207 [ ! -e $file ] && printf "Real CPU time\n\n" > $file208 file=mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt209 [ ! -e $file ] && printf "Comments\n\n" > $file210 254 211 255 ## Error identification … … 227 271 fi 228 272 229 printf "Result\n$ERR\n" > mesg_03_result_${CFG_USER}_${CFG_ARCH}.txt273 echo $ERR >> ${FILE_RESU} 230 274 231 275 ## Save tested configuration if trusting failed 232 276 if [[ "$ST" == 'FAILED' && $DEBUG -eq 0 ]]; then 233 echo 'Creating archive ' trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.tar.gz' under '${REFE_DIR}234 tar -czf ${REFE_DIR}/ trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.tar.gz*277 echo 'Creating archive '${FILE_ARCH}' under '${REFE_DIR} 278 tar -czf ${REFE_DIR}/${FILE_ARCH} ../* 235 279 fi 236 280 … … 240 284 exit 0 241 285 } 242 243 job_pending() {244 time_elapsed=0; time_increment=30245 246 sleep ${time_increment}247 while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do248 printf "\n####################################################\n" >> computation.log249 ${JOB_INFO} ${JOB_ID} >> computation.log250 sleep ${time_increment}251 time_elapsed=$(( ${time_elapsed} + ${time_increment} ))252 done253 sleep ${time_increment}254 255 if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then256 ${JOB_DELE} ${JOB_ID} &> /dev/null257 TIME_LIMIT=$(( ${TIME_LIMI} / 3600 ))258 export ${TIME_LIMIT}259 get_out $ST ERR=6260 fi261 }
Note: See TracChangeset
for help on using the changeset viewer.