Changeset 5664 for branches/2015
- Timestamp:
- 2015-08-03T21:34:03+02:00 (9 years ago)
- Location:
- branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST
- Files:
-
- 4 added
- 4 edited
- 1 moved
Legend:
- Unmodified
- Added
- Removed
-
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/config/X64_ADA.cfg
r5644 r5664 2 2 3 3 # Available softwares: 4 CDO='cdo \/'4 CDO='cdo' 5 5 COMPILER='ifort' 6 MPI='intel \/impi\/'7 NETCDF='netcdf\/ '6 MPI='intel' 7 NETCDF='netcdf\/mpi' 8 8 9 9 # Job commands: LoadLeveler -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/inputs/ORCA2_LIM_PISCES/namelist_cfg
r5622 r5664 6 6 &namrun ! parameters of the run 7 7 !----------------------------------------------------------------------- 8 cn_exp = "O2L 3P_trust"! experience name8 cn_exp = "O2LP_trust" ! experience name 9 9 nn_itend = 150 ! last time step (std 5475) 10 10 nn_euler = 0 ! = 0 : start with forward time step if ln_rstart=T -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh
r5644 r5664 6 6 ## Set defaults 7 7 ##--------------------------------------------------- 8 NEMO_TRUS=$ ( pwd )9 DEBUG=0; PUBLISH=0 10 I_MODULE=0 ## No 'module' commandby default11 ST='FAILED' ## 'FAILED' "" " " 8 NEMO_TRUS=$PWD 9 DEBUG=0; PUBLISH=0; 10 I_MODULE=0; ST='FAILED' ## No 'module' command & 'FAILED' status by default 11 xios_full='--full'; stderr_redir='>&' ## DEBUG options 12 12 13 13 ## Get options for replacing some initials settings … … 45 45 ## Summary of trusting test context 46 46 ##--------------------------------------------------- 47 printf "\ 47 printf "\n\ 48 48 NEMO Trusting test:\n\ 49 49 -------------------\n\ 50 50 Testing ${TEST_CONF} based on ${REFE_CONF} from ${NEMO_BRAN}\n\ 51 Configuration files: ${NEMO_USER}.cfg & ${NEMO_HPCC}.cfg\n" 51 NEMO installation: ${NEMO_USER}\n\ 52 Computing architecture: ${NEMO_HPCC}\n" 52 53 53 54 ## Making timestamped directory with messenger files … … 75 76 print_step 'Compile XIOS' 76 77 cd ${DIR_XIOS} 77 full=''; [ $DEBUG -eq 0 ] && full='--full'78 ./make_xios $full --arch ${NEMO_HPCC} --job $NPROC >&/dev/null78 [ $DEBUG -eq 1 ] && xios_full='' && stderr_redir='>' 79 eval ./make_xios ${xios_full} --arch ${NEMO_HPCC} --job $NPROC ${stderr_redir} /dev/null 79 80 [ ! -e ${DIR_XIOS}/lib/libxios.a ] && get_out 1 80 81 … … 83 84 print_step "Compile ${REFE_CONF} configuration" 84 85 cd ${NEMO_CONF} 85 [[ -d ${TEST_CONF} && $DEBUG -eq 0 ]] && ./makenemo -n ${TEST_CONF} clean >/dev/null86 ./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m ${NEMO_HPCC} -j $NPROC >&/dev/null86 [[ -d ${TEST_CONF} && $DEBUG -eq 0 ]] && eval ./makenemo -n ${TEST_CONF} clean > /dev/null 87 eval ./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m ${NEMO_HPCC} -j $NPROC ${stderr_redir} /dev/null 87 88 [ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out 2 88 89 … … 114 115 print_step 'Test job state' 115 116 if [[ ! -e time.step || $( grep -q 'E R R O R' ocean.output ) ]]; then 117 echo 'Error' 116 118 comments 'E R R O R' 117 119 get_out 5 118 120 else 121 echo 'Good' 119 122 ## Get time computation 120 123 ##--------------------------------------------------- … … 127 130 ## Inspect output text files 128 131 ##--------------------------------------------------- 129 print_step 'Test ASCIIoutput files diff'132 print_step 'Test output files diff' 130 133 diff_results 131 [ "$ST" == 'FAILED' ] && get_out 8132 134 133 135 ## Inspect output NetCDF files 134 136 ##--------------------------------------------------- 135 print_step 'Test last restart NetCDFfiles diff'137 print_step 'Test last restart files diff' 136 138 diff_restart 137 [ "$ST" == 'FAILED' ] && get_out 10139 [ "$ST" == 'FAILED' ] && get_out 8 138 140 139 141 ## Get comments (ocean.output & diff model.log) -
branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting_func.sh
r5644 r5664 10 10 FILE_COMM=mesg_11_comments_$PATTERNAME.txt; 11 11 12 ## 12 ## Trusting summary & archive filenames 13 13 FILE_TRUS=trusting_${DATE}_$PATTERNAME.txt; FILE_ARCH=trusting_${DATE}_$PATTERNAME.tar.gz 14 14 … … 16 16 17 17 init_files() { 18 echo 'Date' > ${FILE_DATE}; echo 'Status' > ${FILE_STAT} 19 echo 'Result' > ${FILE_RESU}; echo 'NEMOGCM rev.' > ${FILE_NEMO} 20 echo 'XIOS rev.' > ${FILE_XIOS}; echo 'Fortran compiler' > ${FILE_COMP} 21 echo 'MPI libs' > ${FILE_MPIN}; echo 'NetCDF libs' > ${FILE_NCDF} 22 echo 'Input files' > ${FILE_INPT}; echo 'Real CPU time' > ${FILE_CPUT} 23 echo 'Comments' > ${FILE_COMM} 18 echo 'Date' > ${FILE_DATE}; echo 'Status' > ${FILE_STAT} 19 echo 'Result' > ${FILE_RESU}; echo 'NEMOGCM rev.' > ${FILE_NEMO} 20 echo 'XIOS rev.' > ${FILE_XIOS}; echo 'Fortran compiler' > ${FILE_COMP} 21 echo 'MPI libs' > ${FILE_MPIN}; echo 'NetCDF libs' > ${FILE_NCDF} 22 echo 'Input files' > ${FILE_INPT}; echo 'Real CPU time' > ${FILE_CPUT} 23 echo 'Comments' > ${FILE_COMM} 24 25 echo $ST >> ${FILE_STAT}; echo 'Unknown error' >> ${FILE_RESU} 24 26 } 25 27 … … 40 42 svn_cmd='svn update -r '$( echo ${NEMO_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' ) 41 43 elif [ $( echo ${NEMO_VERS} | grep -o '[0-9]*' ) ]; then 42 svn_cmd='svn update -r '$( echo ${NEMO_VERS} | grep -o '[0-9]*' )44 svn_cmd='svn update -r '$( echo ${NEMO_VERS} | grep -o '[0-9]*' ) 43 45 else 44 46 svn_cmd='svn '${svn_opt} … … 75 77 for soft in $CDO $COMPILER $MPI $NETCDF; do 76 78 soft_rel='' 77 (( ${I_MODULE} == 1 )) && soft_rel=$( echo $LOADEDMODULES | sed "s/.*$soft\/\([0-9.a-z_]*\).*/\1/i" ) 78 [ -z "${soft_rel}" ] && soft_rel=$( echo $PATH | sed "s/.*$soft\([0-9.a-z_]*\).*/\1/i" ) 79 [ "$soft" == "$COMPILER" ] && soft_rel=$( $soft --version | grep -m1 -oe '\<[0-9. ]*\>' ) 80 ## Cleaning characters string to display proper soft name 81 soft=$( echo $soft | sed 's#\\##g; s#[/-]$##' ) 79 80 if (( ${I_MODULE} == 1 )); then 81 soft_rel=$( echo $LOADEDMODULES | sed "s/.*$soft\/\([0-9.a-z_]*\).*/\1/i" ) 82 else 83 soft_rel=$( echo $PATH | sed "s/.*$soft\([0-9.a-z_]*\).*/\1/i" ) 84 fi 85 86 [ "$soft" == "$COMPILER" ] && soft_rel=$( $soft --version | grep -m1 -oe '\<[0-9. ]*\>' ) 87 soft=$( echo $soft | sed 's#\\##g; s#[/-]$##' ) ## Cleaning characters string to display proper soft name 82 88 echo $soft ${soft_rel} >> model.log 83 89 done 84 90 85 sed -n 4p model.log > ${FILE_COMP}; sed -n 5p model.log > ${FILE_MPIN}; sed -n 6p model.log> ${FILE_NCDF}91 sed -n 4p model.log >> ${FILE_COMP}; sed -n 5p model.log >> ${FILE_MPIN}; sed -n 6p model.log >> ${FILE_NCDF} 86 92 } 87 93 88 94 get_inputs() { 89 local cmd_iol="tar -tvf ${NEMO_FORC}/${NEMO_TARF}" ;cmd_iof="tar -vxf ${NEMO_FORC}/${NEMO_TARF}"95 local cmd_iol="tar -tvf ${NEMO_FORC}/${NEMO_TARF}" cmd_iof="tar -vxf ${NEMO_FORC}/${NEMO_TARF}" 90 96 91 97 [ -z "${NEMO_TARF}" ] && cmd_iol="ls ${NEMO_FORC}/*" && cmd_iof="\cp ${NEMO_FORC}/* ." … … 95 101 diff_inputs() { 96 102 local dif 97 local mesg='Same'98 99 for file in inputs_list.txt $( ls namelist_* ) $( ls *.xml ); do103 local files_list='' mesg='Same' 104 105 for file in inputs_list.txt namelist_* *.xml; do 100 106 dif='' 101 107 if [ -e ${REFE_DIR}/$file ]; then dif=$( diff -q $file ${REFE_DIR}/$file ); else dif=0; fi 102 if [ -n "$dif" ]; then mesg='Different'; [ "$dif" != '0' ] && echo $dif | tee -a ${FILE_COMM}; fi 103 done 104 105 echo $mesg | tee -a ${FILE_INPT} 108 109 if [ -n "$dif" ]; then 110 mesg='Different' 111 [ "$dif" != '0' ] && echo $dif && files_list+=$file' ' 112 fi 113 114 done 115 116 [ "$mesg" == 'Same' ] && echo $mesg 117 echo $mesg >> ${FILE_INPT} 118 119 [ -n "${files_list}" ] && echo 'Inputs : '${files_list}'differ(s)<br>' >> temp_${FILE_COMM} 106 120 } 107 121 108 122 job_pending() { 109 local time_elapsed=0 ;time_increment=30123 local time_elapsed=0 time_increment=30 110 124 111 125 sleep ${time_increment} … … 122 136 if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then 123 137 ${JOB_DELE} ${JOB_ID} &> /dev/null 124 TIME_LIMIT=$(( ${TIME_LIMI} / 3600 )) 125 # export TIME_LIMIT 138 let TIME_LIMIT/=3600 126 139 get_out 6 127 140 fi … … 129 142 130 143 diff_results() { 131 for file in ocean.output $( ls *.stat ); do 144 local files_list='' 145 146 for file in ocean.output *.stat; do 132 147 [ ! -e ${REFE_DIR}/$file ] && export ST='FAILED' && get_out 7 133 diff -q $file ${REFE_DIR}/$file | tee -a ${FILE_COMM} 134 (( $? != 0 )) && export ST='FAILED' 135 done 148 diff -q $file ${REFE_DIR}/$file 149 (( $? != 0 )) && export ST='FAILED' && files_list+=$file' ' 150 done 151 152 [ -n "${files_list}" ] && echo 'Results : '${files_list}'differ(s)<br>' >> temp_${FILE_COMM} 136 153 } 137 154 138 155 diff_restart() { 139 local base_name file nb_dom diff_rcd140 141 [ ! -e ${REFE_DIR}/time.step ] && export ST='FAILED' && get_out 9 142 143 # export LAST_TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 144 LAST_TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] )145 echo 'Last time step of standard run: '${ LAST_TIME_STEP}146 147 if [ $( find -regex ".*_0+${ LAST_TIME_STEP}_restart.*\.nc" -print -quit ) ]; then148 base_name=$( find -regex ".*_0+${ LAST_TIME_STEP}_restart.*\.nc" -print -quit | awk -F/ '{print $NF}' \149 | sed "s/^\(.*\)$ LAST_TIME_STEP\_restart.*/\1$LAST_TIME_STEP\_/")156 local base_name diff_rcd file nb_dom 157 local dif=0 files_list='' 158 159 [ ! -e ${REFE_DIR}/time.step ] && export ST='FAILED' && get_out 7 160 161 TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) 162 echo 'Last time step of standard run: '${TIME_STEP} 163 164 if [ $( find -regex ".*_0+${TIME_STEP}_restart.*\.nc" -print -quit ) ]; then 165 base_name=$( find -regex ".*_0+${TIME_STEP}_restart.*\.nc" -print -quit | awk -F/ '{print $NF}' \ 166 | sed "s/^\(.*\)$TIME_STEP\_restart.*/\1$TIME_STEP\_/" ) 150 167 151 168 for comp in restart restart_ice restart_trc; do 152 file=${base_name}${comp} && printf "$file.nc: "169 file=${base_name}${comp} 153 170 154 171 nb_dom=$( find -name "${file}_[0-9]*.nc" | wc -l | awk '{print $1}' ) … … 157 174 [ $? -eq 0 ] && rm -f ${base_name}${comp}_[0-9]*.nc > /dev/null 158 175 elif [ ${nb_dom} -eq 0 ]; then 159 export ST='FAILED' && get_out 10176 export ST='FAILED' && get_out 8 160 177 fi 161 178 162 179 if [ -e ${REFE_DIR}/$file.nc ]; then 163 180 ## UNIX `cmp` not suitable (filename & timestamp in .nc file) 164 # diff_rcd=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \ 165 # | awk '/records/ {print $1}' ) 166 diff_rcd=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \ 167 | awk '/records/ {print $0}' ) 168 169 # if [[ ! -z ${diff_rcd} && ${diff_rcd} -ne 0 ]]; then 181 diff_rcd=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \ 182 | awk '/records/ {print $0}' | sed 's/differ$/differ,/' | tr -d '\n' ) 183 170 184 if [[ ! -z ${diff_rcd} && $( echo ${diff_rcd} | awk '{print $1}') -ne 0 ]]; then 171 185 export ST='FAILED' 172 echo $CDO $( echo ${diff_rcd} | sed 's/differ$/differ,/' | tr -d '\n' ) 173 # printf "$CDO ${diff_rcd}\n" 186 files_list+=$file' ' 187 let dif+=$( echo ${diff_rcd} | awk '{print $1}' ) 188 echo $file.nc': '${diff_rcd} 174 189 else 175 190 echo 'identical' 176 # printf "identical\n"177 191 fi 178 192 179 193 else 180 export ST='FAILED' && get_out 9194 export ST='FAILED' && get_out 7 181 195 fi 182 196 183 197 done 184 198 199 [ -n "${files_list}" ] && echo 'Restarts: '${files_list}$dif' record(s) differ<br>' >> temp_${FILE_COMM} 185 200 else 186 201 export ST='FAILED' … … 199 214 200 215 if [ "$state" == 'E R R O R' ]; then 201 [ -e time.step ] && LAST_TIME_STEP=$( cat time.step | tr -d ' ' ) 202 # export LAST_TIME_STEP 216 [ -e time.step ] && TIME_STEP=$( cat time.step | tr -d ' ' ) 203 217 [ -e ocean.output ] && line=$( grep -m1 -A4 "$state" ocean.output | tr -d '\n' ) 204 echo $line >> ${FILE_COMM}205 218 else 206 219 [ -e ocean.output ] && line=$( grep -m1 -A2 "^ $state" ocean.output | tr -d '\n' ) 207 220 fi 208 221 209 echo $line 222 echo $line 223 printf $line >> temp_${FILE_COMM} 210 224 } 211 225 212 226 mesg_make() { 213 paste -d ';' mesg_*.txt > ${FILE_TRUS}214 cat${FILE_TRUS}227 cat temp_${FILE_COMM} | tr -d '\n' | sed 's/<br>$//' >> ${FILE_COMM} ## Special construct for comments 228 paste -d ';' mesg_*.txt | tee ${FILE_TRUS} 215 229 } 216 230 … … 238 252 239 253 END_MAIL 240 [ -e ${TEST_DIR}/${FILE_TRUS} ] && cat ${TEST_DIR}/${FILE_TRUS} >> trusting.mail241 mail -s "[ trusting ${REFE_CONF} $BRANCH] $ST $ERR" $EMAIL< trusting.mail254 [ -e ${TEST_DIR}/${FILE_TRUS} ] && cat ${TEST_DIR}/${FILE_TRUS} >> trusting.mail 255 mail -s "[NEMO Trusting ${REFE_CONF} ${NEMO_BRAN}] $ST $ERR" $EMAIL < trusting.mail 242 256 fi 243 257 … … 249 263 250 264 cd ${TEST_DIR} 251 echo $ST >> ${FILE_STAT}252 265 253 266 if [ "$ST" == 'FAILED' ]; then … … 256 269 case $ERR in 257 270 ## Compilation 258 1) ERR='XIOS compilation failed' 271 1) ERR='XIOS compilation failed' ;; 2) ERR='NEMO compilation failed' ;; 259 272 ## Submission 260 3) ERR='Missing input files' 273 3) ERR='Missing input files' ;; 4) ERR='Job submission error' ;; 261 274 ## Running 262 5) ERR='Crashed at time step '${ LAST_TIME_STEP};; 6) ERR='Exceeded time limit '${TIME_LIMI}'h';;275 5) ERR='Crashed at time step '${TIME_STEP};; 6) ERR='Exceeded time limit '${TIME_LIMI}'h';; 263 276 ## Results 264 7) ERR='Missing previous outputs ' ;; 8) ERR='New outputs differ/missing' ;; 265 9) ERR='Missing previous restarts' ;; 10) ERR='New restarts differ/missing' ;; 277 7) ERR='Missing previous outputs' ;; 8) ERR='New outputs differ' ;; 266 278 ## Undefined 267 *) ERR='Unknown error' 279 *) ERR='Unknown error' ;; 268 280 esac 269 281 … … 271 283 fi 272 284 273 echo $ERR >> ${FILE_RESU} 285 sed -i "2 s/.*/$ST/" ${FILE_STAT} 286 sed -i "2 s/.*/$ERR/" ${FILE_RESU} 274 287 275 288 ## Save tested configuration if trusting failed 276 if [[ "$ST" == 'FAILED' && $ DEBUG -eq 0 ]]; then289 if [[ "$ST" == 'FAILED' && $PUBLISH -eq 1 && $DEBUG -eq 0 ]]; then 277 290 echo 'Creating archive '${FILE_ARCH}' under '${REFE_DIR} 278 tar -czf ${REFE_DIR}/${FILE_ARCH} ../* 291 tar -czf ${REFE_DIR}/${FILE_ARCH} * ${TEST_DIR}/${NEMO_CONF}/${TEST_CONF}/cpp_* \ 292 ${TEST_DIR}/${NEMO_CONF}/${TEST_CONF}/MY_SRC \ 293 ${TEST_DIR}/${NEMO_CONF}/${TEST_CONF}/WORK 279 294 fi 280 295
Note: See TracChangeset
for help on using the changeset viewer.