#!/bin/bash
## Messenger filenames
FILE_DATE=mesg_01_date_$PATTERNAME.txt ; FILE_STAT=mesg_02_status_$PATTERNAME.txt
FILE_RESU=mesg_03_result_$PATTERNAME.txt ; FILE_NEMO=mesg_04_nemogcm_$PATTERNAME.txt
FILE_XIOS=mesg_05_xios_$PATTERNAME.txt ; FILE_COMP=mesg_06_compiler_$PATTERNAME.txt
FILE_MPIN=mesg_07_mpi_$PATTERNAME.txt ; FILE_NCDF=mesg_08_netcdf_$PATTERNAME.txt
FILE_INPT=mesg_09_inputs_$PATTERNAME.txt ; FILE_CPUT=mesg_10_cputime_$PATTERNAME.txt
FILE_COMM=mesg_11_comments_$PATTERNAME.txt;
## Timestamped outcome & archive filenames
FILE_TRUS=trusting_${DATE}_$PATTERNAME.txt; FILE_ARCH=trusting_${DATE}_$PATTERNAME.tar.gz
print_step() { printf "\nStep.....\n$1\n"; }
init_files() {
echo 'Date' > ${FILE_DATE}; echo 'Status' > ${FILE_STAT}
echo 'Result' > ${FILE_RESU}; echo 'NEMOGCM rev.' > ${FILE_NEMO}
echo 'XIOS rev.' > ${FILE_XIOS}; echo 'Fortran compiler' > ${FILE_COMP}
echo 'MPI libs' > ${FILE_MPIN}; echo 'NetCDF libs' > ${FILE_NCDF}
echo 'Input files' > ${FILE_INPT}; echo 'Real CPU time' > ${FILE_CPUT}
echo 'Comments' > ${FILE_COMM}
## 'Failed' status with 'Unknown error' by default
echo $ST \
>> ${FILE_STAT}
echo 'Unknown error' \
>> ${FILE_RESU}
}
get_date() {
## UTC time zone for timestamping
local dat=$( date -ud "${DATE}" +"%F %R %Z" )
echo $dat \
>> ${FILE_DATE}
}
get_nemo_rev() {
local dir rev_loc
local rev=0
## Loop on essential directories
for dir in ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} \
${NEMO_EAGR} ${NEMO_EIOI} ${NEMO_EFCM} \
${NEMO_TCMP} ${NEMO_TRBD} \
${DIR_XIOS} ; do
## For time being, just get revision from XIOS with no action on directory
if [ $dir == ${DIR_XIOS} ]; then
rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' )
echo 'XIOS '${rev_loc} \
>> model.log
echo "${rev_loc}" \
>> ${FILE_XIOS}
continue
fi
echo $dir && ${SVN_CMD} $dir
rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' )
## Keep last rev. nb
[ ${rev_loc} -gt $rev ] && rev=${rev_loc}
done
echo 'NEMOGCM '$rev \
>> model.log
echo "$rev" \
>> ${FILE_NEMO}
}
get_soft_rel() {
local soft soft_rel
for soft in $CDO $COMPILER $MPI $NETCDF; do
soft_rel=''
## Software strings to identify depends on use (sed is case-insensitive)
if [ $IMOD -eq 1 ]; then
## Modules system: next word after "$soft/"
soft_rel=$( echo $LOADEDMODULES | sed "s#.*$soft/\([0-9.a-z_]*\).*#\1#i" )
## LOADEDMODULES=netcdf/mpi/4.1.3:cdo/1.5.9:intel/2013.0
## LOADEDMODULES=bullxmpi/1.2.8.2:netcdf/4.3.3.1_hdf5_parallel:cdo/1.6.7
else
## Hard-coded paths: next word after "$soft"
soft_rel=$( echo $PATH | sed "s#.*$soft\([0-9.a-z_]*\).*#\1#i" )
## PATH=/smplocal/intel/impi/4.1.0.024/intel64/bin:/smplocal/pub/cdo/1.5.9/bin:/smplocal/pub/NetCDF/4.1.3/mpi/bin
## PATH=/usr/local/netcdf-4.3.3.1_hdf5_parallel/bin:/opt/mpi/bullxmpi/1.2.8.2/bin:/usr/local/netcdf-utils-4.3.3.1_hdf5/bin
fi
## option --version would work for main compilers (gfortran, intel, pgfortran, ...)
[ $soft == $COMPILER ] && soft_rel=$( $soft --version | grep -m1 -oe '\<[0-9. ]*\>' )
## Cleaning characters string to display proper soft name
soft=$( echo $soft | sed 's#\\##g; s#[/-]$##' )
echo $soft ${soft_rel} \
>> model.log
done
sed -n 4p model.log \
>> ${FILE_COMP}
sed -n 5p model.log \
>> ${FILE_MPIN}
sed -n 6p model.log \
>> ${FILE_NCDF}
}
get_inputs() {
# List archive content & extract it by default
local cmd_iol="tar -tvf ${NEMO_FORC}/${NEMO_TARF}" cmd_iof="tar -vxf ${NEMO_FORC}/${NEMO_TARF}"
## List & copy files in case of personal inputs
[ -z "${NEMO_TARF}" ] && { cmd_iol="ls ${NEMO_FORC}/*"; cmd_iof="\cp ${NEMO_FORC}/* ."; }
${cmd_iol} > inputs_list.txt
${cmd_iof} > /dev/null
}
diff_inputs() {
local dif file
local files_list='' mesg='Same'
## Simple diff
for file in 'inputs_list.txt' *namelist_* *.xml cpp_*; do
dif=''
## Continue even if input file is not in here (see after)
if [ -e ${REFE_DIR}/$file ]; then dif=$( diff -q $file ${REFE_DIR}/$file ); else dif=0; fi
## Pass over useless file omission in benckmark directory
if [[ -n "$dif" && "$dif" != '0' ]]; then
mesg='Different'; echo $dif; files_list+=$file' '
fi
done
[ $mesg == 'Same' ] && echo $mesg
echo $mesg \
>> ${FILE_INPT}
## List different files for web comment
[ -n "${files_list}" ] && echo 'Inputs : '${files_list}'differ
' \
>> temp_${FILE_COMM}
}
job_pending() {
local outline=$( printf "%100s" ) time_elapsed=0 time_increment=30
sleep ${time_increment}
## Append a log file while pending
while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do
printf "\n%s\n" ${outline// /#} \
>> computation.log
${JOB_INFO} ${JOB_ID} \
>> computation.log
sleep ${time_increment}
time_elapsed=$(( ${time_elapsed} + ${time_increment} ))
done
sleep ${time_increment}
## Kill remaining job & stop the test if it's too long
[ ${time_elapsed} -eq ${TIME_LIMI} ] && { ${JOB_DELE} ${JOB_ID} &> /dev/null; get_out 6; }
}
diff_results() {
local file
local files_list='' mesg='Same'
## Simple diff
for file in 'ocean.output' *.stat; do
## Stop if no benchmark files (ocean.output, eventual stat files)
[ ! -e ${REFE_DIR}/$file ] && { export ST='FAILED'; get_out 7; }
diff -q $file ${REFE_DIR}/$file
## Continue even if it differs
[ $? -ne 0 ] && { export ST='FAILED'; mesg='Different'; files_list+=$file' '; }
done
[ $mesg == 'Same' ] && echo $mesg
## List different files for web comment
[ -n "${files_list}" ] && echo 'Results : '${files_list}'differ
' \
>> temp_${FILE_COMM}
}
diff_restart() {
local base_name comp dif file list_comp list_tmsp nb_dom time_step tmsp
local files_list='' dift=0
## Stop if no benchmark files (ie time.step)
[ ! -e ${REFE_DIR}/time.step ] && { export ST='FAILED'; get_out 7; }
time_step=$( cat ${REFE_DIR}/time.step | tr -d [:space:] )
## Find all restart files to rebuild
if [ $( find -regex ".*_restart.*[0-9]\.nc" -print -quit ) ]; then
base_name=$( find -regex ".*_restart.*[0-9]\.nc" \
| sed "s#^\./\(.*\)_[0-9]*_restart.*#\1#" | sort -u )
list_comp=$( find -regex ".*_restart.*[0-9]\.nc" \
| sed "s#^.*\(restart[a-z_]*\)_[0-9].*\.nc#\1#" | sort -u )
list_tmsp=$( find -regex ".*_restart.*[0-9]\.nc" \
| sed "s#^.*\([0-9]\{8\}\)_restart.*#\1#" | sort -u )
## Loop on each time step
for tmsp in ${list_tmsp}; do
for comp in ${list_comp}; do
file=${base_name}_${tmsp}_${comp}
nb_dom=$( find -name "${file}_[0-9]*.nc" | wc -l | awk '{ print $1 }' )
if [ ${nb_dom} -gt 1 ]; then
${NEMO_TRBD}/rebuild_nemo -t $NPROC $file ${nb_dom} > /dev/null
[ $? -eq 0 ] && rm -f ${file}_[0-9]*.nc > /dev/null
elif [ ${nb_dom} -eq 0 ]; then
export ST='FAILED' && get_out 8
fi
## Compare restart files at same time step
if [ $tmsp -eq ${time_step} ]; then
## Stop if no benchmark files (restart file)
if [ -e ${REFE_DIR}/$file.nc ]; then
## UNIX `cmp` not suitable (timestamp in .nc file)
dif=$( $CDOD $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \
| awk '/records/ {print $0}' | sed '2 s/^/,/' | tr -d '\n' )
## CDO can return void stdout with no difference
if [[ -n "$dif" && $( echo $dif | awk '{print $1}' ) -ne 0 ]]; then
export ST='FAILED'
files_list+=$comp' ' && let dift+=$( echo $dif | awk '{print $1}' )
echo $file.nc': '$dif
fi
else
export ST='FAILED' && get_out 7
fi
else
continue
fi
done
done
## List different files for web comment with sum of different parameters
if [ $dift -ne 0 ]; then
echo 'Restarts: '${files_list}$dift' record(s) differ
' \
>> temp_${FILE_COMM}
else
echo 'Same'
fi
else
export ST='FAILED'
fi
}
get_cpu_time() {
## Interest for checking unusual time computing
local real_cpu_time=$( eval ${JOB_TIME} )
echo ${real_cpu_time} | tee -a ${FILE_CPUT}
}
comments() {
local opat
local line='' state=$1
if [ -e ocean.output ]; then
## 'W A R N I N G' by default
opat="-A2 \"^ $state\""
[ "$state" == 'E R R O R' ] && opat="-A4 \"$state\""
## Select first occurence for web comment
line=$( eval grep -m1 $opat ocean.output | tr -d '\n' )
echo $line
fi
printf "$line
" \
>> temp_${FILE_COMM}
}
mesg_make() {
## Format comments for web
[ -e temp_${FILE_COMM} ] && cat temp_${FILE_COMM} | tr -d '\n' | sed 's/
$//' \
>> ${FILE_COMM}
## Construct txt file for web with all messenger files
paste -d ';' mesg_*.txt | tee ${FILE_TRUS}
}
mesg_publish() {
local cmd
## Production mode (-p|--publish)
if [ $PUBLISH -eq 1 ]; then
## Create or append historical trusting file
if [ -f ${REFE_DIR}/trusting_$PATTERNAME.txt ]; then cmd='tail -1'; else cmd='cat'; fi
$cmd ${FILE_TRUS} \
>> ${REFE_DIR}/trusting_$PATTERNAME.txt
## Send mail only when FAILED
if [[ ! -z "$EMAIL" && $ST == 'FAILED' ]]; then
## Content
cat < trusting.mail
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
Dear $USER,
The trusting sequence for ${REFE_CONF} has failed.
Directory: ${TEST_DIR}
Archive created: ${FILE_ARCH} in ${REFE_DIR}
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
`cat ${TEST_DIR}/${FILE_TRUS}`
END_MAIL
## Send with detailed subject
mail -s "[NEMO Trusting ${NEMO_HPCC} ${REFE_CONF} ${NEMO_BRAN}] $ST $ERR" $EMAIL \
< trusting.mail
fi
fi
}
get_out() {
local time_step=0
ERR=$1
print_step 'End of trusting test'
## In case of compilation error
cd ${TEST_DIR}
if [ $ST == 'FAILED' ]; then
echo 'Failure'
## Error identification
case $ERR in
## Compilation
'1') ERR='XIOS compilation failed' ;; '2') ERR='NEMO compilation failed';;
## Submission
'3') ERR='Missing input files' ;; '4') ERR='Job submission error' ;;
## Running
'5') ERR='Crashed at time step' ;; '6') ERR='Exceeded time limit' ;;
## Results
'7') ERR='Missing previous outputs';; '8') ERR='New outputs differ' ;;
## Undefined
"*") ERR='Unknown error' ;;
esac
else
echo 'Success' && ERR='Code is reliable'
fi
## Eventual comments from ocean.output
if [ "$ERR" == 'Crashed at time step' ]; then
comments 'E R R O R'
[ -e time.step ] && time_step=$( cat time.step | tr -d ' ' )
ERR+=' '$time_step
else
comments 'W A R N I N G'
[ "$ERR" == 'Exceeded time limit' ] && ERR+=' '$(( ${TIME_LIMIT}/3600 ))'h'
fi
## Last messenger files
export ERR
sed -i "2 s/.*/$ST/" ${FILE_STAT}; sed -i "2 s/.*/$ERR/" ${FILE_RESU}
## Save tested configuration if trusting failed in production mode (-p|--publish)
if [[ $ST == 'FAILED' && $PUBLISH -eq 1 ]]; then
echo 'Creating archive '${FILE_ARCH}' under '${REFE_DIR}
tar -czf ${REFE_DIR}/${FILE_ARCH} * ${NEMO_HOME}/CONFIG/${TEST_CONF}/MY_SRC \
${NEMO_HOME}/CONFIG/${TEST_CONF}/WORK
fi
## Share trusting result (.txt file/mail)
print_step 'Trusting digest'
mesg_make
mesg_publish
exit 0
}