#!/bin/bash comments() { state=$1 if [ "$state" == 'E R R O R' ]; then if [ -e ocean.output ]; then last_time_step=$( cat ${TEST_DIR}/time.step | tr -d [:space:] ) STEP='nemo.exe crashed at '${last_time_step} && export STEP line=$( grep -m1 -A5 "$state" ocean.output | tr -d '\n' ) else STEP='nemo.exe crashed at initialization' && export STEP return fi fi if [ $( diff -q model.log ${REFE_DIR}/model.log ) ]; then line=$( diff model.log ${REFE_DIR}/model.log | tr -d '\n' ) fi printf "Comments\n$line\n" | tee mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt } get_cpu_time() { # Portability ! real_cpu_time=$( ${JOB_VIEW} ${JOB_ID} | awk 'END{print $9}' ) printf "Real CPU time\n${real_cpu_time}\n" | tee mesg_10_realcputime_${CFG_USER}_${CFG_ARCH}.txt } get_inputs() { if [ ! -z "${FORC_TAR}" ]; then cmd_iol="tar -tvf ${NEMO_FORC}/${FORC_TAR}"; cmd_iof="tar -vxf ${NEMO_FORC}/${FORC_TAR}" else cmd_iol="ls ${NEMO_FORC}/*" ; cmd_iof="\cp ${NEMO_FORC}/* ." fi ${cmd_iol} > inputs_list.txt ${cmd_iof} > /dev/null } diff_inputs() { mesg='Same' for file in inputs_list.txt $( ls namelist_* ) $( ls *.xml ); do diff $file ${REFE_DIR}/$file [ $? -ne 0 ] && mesg='Different' done printf "Input files\n$mesg\n" | tee mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt } diff_restart() { last_time_step=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) echo 'Last time step of standard run: '${last_time_step} if [ $( find ${TEST_DIR} -name "*${last_time_step}_restart*.nc" -print -quit ) ]; then base_name=$( find ${TEST_DIR} -name "*${last_time_step}_restart*.nc" -print -quit \ | awk -F/ '{print $NF}' | sed "s/^\(.*\)$last_time_step\_.*/\1$last_time_step/" ) for comp in restart restart_ice restart_trc; do file=${base_name}'_'${comp} && printf "$file'.nc': " nb_dom=$( find ${TEST_DIR} -name "${file}_[0-9]*.nc" | wc -l | awk '{print $1}' ) if [ ${nb_dom} -gt 1 ]; then ${NEMO_TRBD}/rebuild_nemo -t $NPROC $file ${nb_dom} > /dev/null fi # UNIX `cmp` not suitable (filename & timestamp in .nc file) nc_diff=$( $CDO diffn ${TEST_DIR}/$file'.nc' ${REFE_DIR}/$file'.nc' 2> /dev/null | sed -n '$p' ) if [ ! -z "${nc_diff}" ]; then export STATUS='FAILED' STEP='Restart files different' printf "$CDO diffn ${nc_diff}\n" else printf "identical\n" fi rm -f ${TEST_DIR}/${base_name}_${comp}_[0-9]*.nc &> /dev/null done else export STATUS='FAILED'; STEP='No restart files found at last time step' printf "no restart files found for comparison\n" fi } diff_textfiles() { for file in ocean.output $( ls *.stat ); do diff $file ${REFE_DIR}/$file [ $? -ne 0 ] && export STATUS='FAILED' STEP='ASCII output file diff' done } get_soft_rel() { for rel in $CDO $COMPILER $MPI $NETCDF; do arch_rel=$( echo $LOADEDMODULES | sed "s#.*$rel/\([^:]*\).*#\1#" ) [ -z "${arch_rel}" ] && arch_rel=$( echo $PATH | sed "s#.*$rel/\([^/]*\).*#\1#" ) [ "$rel" == "$COMPILER" ] && arch_rel=$( $rel --version | grep -m1 -o ' [0-9.]* ' ) echo $rel ${arch_rel} >> model.log done sed -n 4p model.log | sed 'i\Compiler' > mesg_06_compiler_${CFG_USER}_${CFG_ARCH}.txt sed -n 5p model.log | sed 'i\MPI libs' > mesg_07_mpi_${CFG_USER}_${CFG_ARCH}.txt sed -n 6p model.log | sed 'i\NetCDF libs' > mesg_08_netcdf_${CFG_USER}_${CFG_ARCH}.txt } get_nemo_rev() { svn_opt='status' # If -v|--version option has been set, modify svn command if [[ $( echo ${NEMO_VERS} | grep "HEAD\|up\|update" ) \ || $( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) ]]; then rev=$( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) || rev='HEAD' svn_cmd='svn update -r '$rev else svn_cmd='svn '${svn_opt} fi rev=0 for dir in ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} \ ${NEMO_EIOI} ${NEMO_EFCM} \ ${NEMO_TCMP} ${NEMO_TRBD} \ ${DIR_XIOS} ; do # For time being, just get revision number from XIOS if [ "$dir" == "${DIR_XIOS}" ]; then rev_loc=$( svn info $dir | awk '(NR == 5) {print $NF}' ) echo 'XIOS '${rev_loc} >> model.log printf "XIOS rev.\n${rev_loc}\n" > mesg_05_xios__${CFG_USER}_${CFG_ARCH}.txt continue fi echo ${svn_cmd} $dir': '$( ${svn_cmd} $dir ) rev_loc=$( svn info $dir | awk '(NR == 5) {print $NF}' ) (( ${rev_loc} >= $rev )) && rev=${rev_loc} done echo 'NEMOGCM '$rev >> model.log # printf "NEMOGCM rev.\n$rev\n" > mesg_04_nemogcm_${CFG_USER}_${CFG_ARCH}.txt printf "NEMOGCM rev.\n\ $rev" \ > mesg_04_nemogcm_${CFG_USER}_${CFG_ARCH}.txt } mesg_make() { paste -d ';' mesg_*_${CFG_USER}_${CFG_ARCH}.txt > trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt cat trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt } mesg_publish() { if [ $PUBLISH -eq 1 ]; then if [ -f ${REFE_DIR}/trusting_${CFG_USER}_${CFG_ARCH}.txt ]; then cmd='sed -n 2p'; else cmd='cat'; fi $cmd trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt >> ${REFE_DIR}/trusting_${CFG_USER}_${CFG_ARCH}.txt # Send mail only when FAILED if [[ ! -z $EMAIL && "$STATUS" == 'FAILED' ]]; then cat << END_MAIL > trusting.mail XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX Dear $USER, The trusting sequence for ${CONF_REF} has failed at step: `tail -n 1 ${TEST_DIR}/mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt` Directory : ${TEST_DIR} XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX END_MAIL if [ -e ${TEST_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt ]; then cat ${TEST_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt >> trusting.mail fi mail -s "[trusting ${CONF_REF}] $STATUS $STEP" $EMAIL < trusting.mail fi fi } print_step() { [ ! -z "$1" ] && STEP=$1 export STEP && printf "Step.....\n$STEP\n" } get_out() { # Save tested configuration if trusting failed if [ "$STATUS" == 'FAILED' ]; then cd ${TEST_DIR} touch mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt mesg_10_realcputime_${CFG_USER}_${CFG_ARCH}.txt \ mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt #tar -czf ${REFE_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.tar.gz * fi printf "Status\n$STATUS\n" > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt printf "Step.....\n$STEP\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt mesg_make; mesg_publish exit 1 } job_pending() { time_elapsed=0; time_increment=30 sleep ${time_increment} while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do printf "\n####################################################" >> computation.log ${JOB_VIEW} ${JOB_ID} >> computation.log sleep ${time_increment} let time_elapsed+=${time_increment} done sleep ${time_increment} if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then STEP='Exceeded time limit' if [ $( ${JOB_LIST} | grep ${JOB_ID} ) ]; then STEP='Job not finished on time: '$(( ${TIME_LIMI}/3600 ))'h' fi print_step ${JOB_DELE} ${JOB_ID} &> /dev/null get_out $STATUS $STEP fi }