1 | #!/bin/bash |
---|
2 | |
---|
3 | |
---|
4 | comments() { |
---|
5 | echo 'Comments' | tee mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt |
---|
6 | grep $1 ocean.output | tee -a mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt |
---|
7 | } |
---|
8 | |
---|
9 | get_cpu_time() { |
---|
10 | # Portability ! |
---|
11 | real_cpu_time=$( ${JOB_VIEW} ${job_num} | awk 'END{print $9}' ) |
---|
12 | printf "Real CPU time\n%s\n" ${real_cpu_time} | tee mesg_08_realcputime_${CFG_USER}_${CFG_ARCH}.txt |
---|
13 | } |
---|
14 | |
---|
15 | diff_inputs() { |
---|
16 | mesg='Same' |
---|
17 | |
---|
18 | for file in inputs_list.txt $( ls namelist_* ) iodef.xml; do |
---|
19 | diff $file ${REFE_DIR}/$file |
---|
20 | [ $? -ne 0 ] && mesg='Different' |
---|
21 | done |
---|
22 | |
---|
23 | printf "Input files\n$mesg" | tee mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt |
---|
24 | } |
---|
25 | |
---|
26 | diff_restart() { |
---|
27 | last_time_step=$( cat ${REFE_DIR}/time.step | tr -d [:space:] ) |
---|
28 | echo 'Last time step of standard run: '${last_time_step} |
---|
29 | |
---|
30 | if [ $( find ${TEST_DIR} -name "*${last_time_step}_restart*.nc" -print -quit ) ]; then |
---|
31 | base_name=$( find ${TEST_DIR} -name "*${last_time_step}_restart*.nc" -print -quit \ |
---|
32 | | awk -F/ '{print $NF}' | sed "s/^\(.*\)$last_time_step\_.*/\1$last_time_step/" ) |
---|
33 | |
---|
34 | for comp in restart restart_ice restart_trc; do |
---|
35 | printf "${base_name}_${comp}.nc: " |
---|
36 | |
---|
37 | nb_dom=$( find ${TEST_DIR} -name "${base_name}_${comp}_[0-9]*.nc" | wc -l | awk '{print $1}' ) |
---|
38 | if [ ${nb_dom} -gt 1 ]; then |
---|
39 | ${NEMO_TRBD}/rebuild_nemo -t $NPROC ${base_name}_$comp ${nb_dom} > /dev/null |
---|
40 | fi |
---|
41 | |
---|
42 | cmp ${TEST_DIR}/${base_name}_$comp.nc ${REFE_DIR}/${base_name}_$comp.nc |
---|
43 | if [ $? -ne 0 ]; then |
---|
44 | status='FAILED' |
---|
45 | printf "$CDO diffn\n" |
---|
46 | $CDO diffn ${TEST_DIR}/${base_name}_$comp.nc ${REFE_DIR}/${base_name}_$comp.nc 2> /dev/null |
---|
47 | else |
---|
48 | printf "identical\n" |
---|
49 | fi |
---|
50 | |
---|
51 | \rm ${TEST_DIR}/${base_name}_${comp}_[0-9]*.nc |
---|
52 | done |
---|
53 | |
---|
54 | else |
---|
55 | printf "no restart files found for comparison\n" |
---|
56 | fi |
---|
57 | } |
---|
58 | |
---|
59 | get_soft_rel() { |
---|
60 | for rel in $CDO $COMPILER $MPI $NETCDF; do |
---|
61 | arch_rel=$( echo $LOADEDMODULES | sed "s#.*$rel/\([^:]*\).*#\1#" ) |
---|
62 | [ -z "${arch_rel}" ] && arch_rel=$( echo $PATH | sed "s#.*$rel/\([^/]*\).*#\1#" ) |
---|
63 | [ $rel == $COMPILER ] && arch_rel=$( $rel --version | grep -m1 -o ' [0-9.]* ' ) |
---|
64 | echo $rel ${arch_rel} >> model.log |
---|
65 | done |
---|
66 | |
---|
67 | sed -n 2p model.log | sed 'i\Compiler' | tee mesg_05_compiler_${CFG_USER}_${CFG_ARCH}.txt |
---|
68 | sed -n 3p model.log | sed 'i\MPI libs' | tee mesg_06_mpi_${CFG_USER}_${CFG_ARCH}.txt |
---|
69 | sed -n 4p model.log | sed 'i\NetCDF libs' | tee mesg_07_netcdf_${CFG_USER}_${CFG_ARCH}.txt |
---|
70 | } |
---|
71 | |
---|
72 | get_nemo_rev() { |
---|
73 | rev=0 |
---|
74 | |
---|
75 | # If -v|--version option has been set, modify svn command |
---|
76 | if [ $( echo ${NEMO_VERS} | grep "up\|update" ) ]; then SVN_OPT='update' |
---|
77 | elif [ $( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) ]; then SVN_OPT='checkout' |
---|
78 | rev=$( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) |
---|
79 | fi |
---|
80 | |
---|
81 | for dir in ${DIR_XIOS} ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} ${NEMO_EIOI} \ |
---|
82 | ${NEMO_EFCM} ${NEMO_TRUS} ${NEMO_TCMP} ${NEMO_TRBD} ; do |
---|
83 | |
---|
84 | # For time being, just get revision number from XIOS |
---|
85 | if [ $dir == ${DIR_XIOS} ]; then |
---|
86 | echo 'XIOS '$( svn info $dir | awk '(NR == 9) {print $NF}' ) >> model.log |
---|
87 | continue |
---|
88 | fi |
---|
89 | |
---|
90 | # |
---|
91 | if [ $( echo ${SVN_OPT} | grep 'checkout' ) ]; then |
---|
92 | cd ../$dir |
---|
93 | url=$( svn info $dir | grep URL | awk '{print $NF}' ) |
---|
94 | svn ${SVN_OPT} $url -r $rev $dir && cd - |
---|
95 | else |
---|
96 | svn ${SVN_OPT} $dir |
---|
97 | fi |
---|
98 | |
---|
99 | rev_loc=$( svn info $dir | awk '(NR == 5) {print $NF}' ) |
---|
100 | (( ${rev_loc} >= $rev )) && rev=${rev_loc} |
---|
101 | done |
---|
102 | |
---|
103 | echo 'NEMO '$rev >> model.log |
---|
104 | sed -n 6p model.log | sed 'i\NEMOGCM rev.' | tee mesg_02_nemo_${CFG_USER}_${CFG_ARCH}.txt |
---|
105 | } |
---|
106 | |
---|
107 | mesg_make() { |
---|
108 | paste -d ';' mesg_*_${CFG_USER}_${CFG_ARCH}.txt > trusting_${DATE_STR}_${CFG_USER}_${CFG_ARCH}.txt |
---|
109 | cat trusting_${DATE_STR}_${CFG_USER}_${CFG_ARCH}.txt |
---|
110 | } |
---|
111 | |
---|
112 | mesg_publish() { |
---|
113 | |
---|
114 | if [ $PUBLISH -eq 1 ]; then |
---|
115 | |
---|
116 | if [ -f ${REFE_DIR}/trusting_${CFG_USER}_${CFG_ARCH}.txt ]; then cmd='sed -n 2p'; else cmd='cat'; fi |
---|
117 | |
---|
118 | $cmd trusting_${DATE_STR}_${CFG_USER}_${CFG_ARCH}.txt >> ${REFE_DIR}/trusting_${CFG_USER}_${CFG_ARCH}.txt |
---|
119 | |
---|
120 | # Send mail only when FAILED |
---|
121 | if [[ $( grep 'FAILED' mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt ) && ! -z $MAIL ]]; then |
---|
122 | cat << END_MAIL > trusting.mail |
---|
123 | XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX |
---|
124 | Dear $USER, |
---|
125 | |
---|
126 | The trusting sequence for ${CONF_REF} has failed at step: |
---|
127 | `tail -n 1 mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt` |
---|
128 | Directory : ${TEST_DIR} |
---|
129 | |
---|
130 | XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX |
---|
131 | |
---|
132 | END_MAIL |
---|
133 | cat mesg_*_${CFG_USER}_${CFG_ARCH}.txt >> trusting.mail |
---|
134 | mail -s "[trusting ${CONF_REF}] FAILED" $MAIL < trusting.mail |
---|
135 | fi |
---|
136 | |
---|
137 | fi |
---|
138 | } |
---|
139 | |
---|
140 | pstep() { |
---|
141 | printf "Step.....\n$1\n" |
---|
142 | } |
---|
143 | |
---|
144 | get_out() { |
---|
145 | step=$1; status=$2 |
---|
146 | mesg_make; mesg_publish |
---|
147 | |
---|
148 | # Save tested configuration if trusting failed |
---|
149 | if [ $status == 'FAILED' ]; then |
---|
150 | touch mesg_08_realcputime_${CFG_USER}_${CFG_ARCH}.txt mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt |
---|
151 | cd ${REFE_DIR} |
---|
152 | tar -czf trusting_${DATE_STR}_${CFG_USER}_${CFG_ARCH}.tar.gz ${TEST_DIR}/* |
---|
153 | cd - |
---|
154 | fi |
---|
155 | |
---|
156 | printf "Status\n$status\n" > mesg_03_status_${CFG_USER}_${CFG_ARCH}.txt |
---|
157 | printf "Step.....\n$step\n" > mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt |
---|
158 | |
---|
159 | exit 1 |
---|
160 | } |
---|