New URL for NEMO forge!   http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.
trusting_func.sh in branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST – NEMO

source: branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting_func.sh @ 5465

Last change on this file since 5465 was 5465, checked in by nicolasmartin, 9 years ago

dev_r5092_CNRS_SETTE End of main script trusting.sh size reduction

  • Property eol-style set to native
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Rev URL
File size: 7.3 KB
Line 
1#!/bin/bash
2
3
4comments() {
5    state=$1
6
7    if [ "$state" == 'E R R O R' ]; then
8   if [ -e ocean.output ]; then
9       last_time_step=$( cat ${TEST_DIR}/time.step | tr -d [:space:] )
10       STEP='nemo.exe crashed at '${last_time_step} && export STEP
11       line=$( grep -m1 -A5 "$state" ocean.output | tr -d '\n' )
12   else
13       STEP='nemo.exe crashed at initialization'    && export STEP
14       return
15   fi
16    fi
17
18    if [ $( diff -q model.log ${REFE_DIR}/model.log ) ]; then
19   line=$( diff model.log ${REFE_DIR}/model.log  | tr -d '\n' )
20    fi
21
22    printf "Comments\n$line\n" | tee mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
23}
24
25get_cpu_time() {
26    # Portability !
27    real_cpu_time=$( ${JOB_VIEW} ${JOB_ID} | awk 'END{print $9}' )
28    printf "Real CPU time\n${real_cpu_time}\n" | tee mesg_10_realcputime_${CFG_USER}_${CFG_ARCH}.txt
29}
30
31get_inputs() {
32    if [ ! -z "${FORC_TAR}" ]; then
33   cmd_iol="tar -tvf ${NEMO_FORC}/${FORC_TAR}"; cmd_iof="tar -vxf ${NEMO_FORC}/${FORC_TAR}"
34    else
35   cmd_iol="ls ${NEMO_FORC}/*"                ; cmd_iof="\cp ${NEMO_FORC}/* ."
36    fi
37
38    ${cmd_iol} > inputs_list.txt
39    ${cmd_iof} > /dev/null
40}
41
42diff_inputs() {
43    mesg='Same'
44
45    for file in inputs_list.txt $( ls namelist_* ) $( ls *.xml ); do
46   diff $file ${REFE_DIR}/$file
47   [ $? -ne 0 ] && mesg='Different'
48    done
49
50    printf "Input files\n$mesg\n" | tee mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt
51}
52
53diff_restart() {
54    last_time_step=$( cat ${REFE_DIR}/time.step | tr -d [:space:] )
55    echo 'Last time step of standard run: '${last_time_step}
56
57    if [ $( find ${TEST_DIR} -name "*${last_time_step}_restart*.nc" -print -quit ) ]; then
58   base_name=$( find ${TEST_DIR} -name "*${last_time_step}_restart*.nc" -print -quit           \
59                | awk -F/ '{print $NF}' | sed "s/^\(.*\)$last_time_step\_.*/\1$last_time_step/" )
60
61   for comp in restart restart_ice restart_trc; do
62       file=${base_name}'_'${comp} && printf "$file'.nc': "
63
64       nb_dom=$( find ${TEST_DIR} -name "${file}_[0-9]*.nc" | wc -l | awk '{print $1}' )
65       if [ ${nb_dom} -gt 1 ]; then
66      ${NEMO_TRBD}/rebuild_nemo -t $NPROC $file ${nb_dom} > /dev/null
67       fi
68
69       # UNIX `cmp` not suitable (filename & timestamp in .nc file)
70       nc_diff=$( $CDO diffn ${TEST_DIR}/$file'.nc' ${REFE_DIR}/$file'.nc' 2> /dev/null | sed -n '$p' )
71       if [ ! -z "${nc_diff}" ]; then
72      export STATUS='FAILED' STEP='Restart files different'
73      printf "$CDO diffn ${nc_diff}\n"
74       else
75      printf "identical\n"
76       fi
77
78       rm -f ${TEST_DIR}/${base_name}_${comp}_[0-9]*.nc &> /dev/null
79   done
80
81    else
82   export STATUS='FAILED'; STEP='No restart files found at last time step'
83   printf "no restart files found for comparison\n"
84    fi
85
86}
87
88diff_textfiles() {
89    for file in ocean.output $( ls *.stat ); do
90   diff $file ${REFE_DIR}/$file
91   [ $? -ne 0 ] && export STATUS='FAILED' STEP='ASCII output file diff'
92    done
93}
94
95get_soft_rel() {
96    for rel in $CDO $COMPILER $MPI $NETCDF; do
97                                 arch_rel=$( echo $LOADEDMODULES | sed  "s#.*$rel/\([^:]*\).*#\1#" )
98   [ -z "${arch_rel}"  ] && arch_rel=$( echo $PATH          | sed  "s#.*$rel/\([^/]*\).*#\1#" )
99   [ "$rel" == "$COMPILER" ] && arch_rel=$( $rel --version      | grep -m1 -o ' [0-9.]* '         )
100   echo $rel ${arch_rel} >> model.log
101    done
102
103    sed -n 4p model.log | sed 'i\Compiler'    > mesg_06_compiler_${CFG_USER}_${CFG_ARCH}.txt
104    sed -n 5p model.log | sed 'i\MPI libs'    > mesg_07_mpi_${CFG_USER}_${CFG_ARCH}.txt
105    sed -n 6p model.log | sed 'i\NetCDF libs' > mesg_08_netcdf_${CFG_USER}_${CFG_ARCH}.txt
106}
107
108get_nemo_rev() {
109    svn_opt='status'
110
111    # If -v|--version option has been set, modify svn command
112    if [[    $( echo ${NEMO_VERS} | grep  "HEAD\|up\|update"   ) \
113     || $( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' )  ]]; then
114   rev=$( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) || rev='HEAD'
115   svn_cmd='svn update -r '$rev 
116    else
117   svn_cmd='svn '${svn_opt}
118    fi
119
120    rev=0
121    for dir in ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} \
122               ${NEMO_EIOI} ${NEMO_EFCM}              \
123          ${NEMO_TCMP} ${NEMO_TRBD}              \
124          ${DIR_XIOS}                             ; do
125
126   # For time being, just get revision number from XIOS
127   if [ "$dir" == "${DIR_XIOS}" ]; then
128       rev_loc=$( svn info $dir | awk '(NR == 5) {print $NF}' )
129       echo   'XIOS '${rev_loc}         >> model.log
130       printf "XIOS rev.\n${rev_loc}\n" >  mesg_05_xios__${CFG_USER}_${CFG_ARCH}.txt
131       continue
132   fi
133
134   echo ${svn_cmd} $dir': '$( ${svn_cmd} $dir )
135   rev_loc=$( svn info $dir | awk '(NR == 5) {print $NF}' )
136   (( ${rev_loc} >= $rev )) && rev=${rev_loc}
137    done
138
139    echo   'NEMOGCM '$rev         >> model.log
140#   printf "NEMOGCM rev.\n$rev\n" >  mesg_04_nemogcm_${CFG_USER}_${CFG_ARCH}.txt
141    printf "NEMOGCM rev.\n\
142            <a href=\"https://forge.ipsl.jussieu.fr/nemo/changeset/$rev\" target=\"_blank\">$rev</a>" \
143    >  mesg_04_nemogcm_${CFG_USER}_${CFG_ARCH}.txt
144}
145
146mesg_make() {
147    paste -d ';' mesg_*_${CFG_USER}_${CFG_ARCH}.txt > trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt
148    cat                                               trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt
149}
150
151mesg_publish() {
152 
153if [ $PUBLISH -eq 1 ]; then
154
155  if [ -f ${REFE_DIR}/trusting_${CFG_USER}_${CFG_ARCH}.txt ]; then cmd='sed -n 2p'; else cmd='cat'; fi
156
157  $cmd trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt >> ${REFE_DIR}/trusting_${CFG_USER}_${CFG_ARCH}.txt
158
159  # Send mail only when FAILED
160  if [[ ! -z $EMAIL && "$STATUS" == 'FAILED' ]]; then
161      cat << END_MAIL > trusting.mail
162XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
163Dear $USER,
164
165The trusting sequence for ${CONF_REF} has failed at step:
166`tail -n 1 ${TEST_DIR}/mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt`
167Directory : ${TEST_DIR}
168
169XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
170
171END_MAIL
172      if [ -e ${TEST_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt ]; then
173     cat ${TEST_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt  >> trusting.mail
174      fi
175      mail -s "[trusting ${CONF_REF}] $STATUS $STEP" $EMAIL  <  trusting.mail
176  fi
177
178fi
179}
180
181print_step() {
182    [ ! -z "$1" ] && STEP=$1
183    export STEP && printf "Step.....\n$STEP\n"
184}
185
186get_out() {
187    # Save tested configuration if trusting failed
188    if [ "$STATUS" == 'FAILED' ]; then
189   cd ${TEST_DIR}
190   touch mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt mesg_10_realcputime_${CFG_USER}_${CFG_ARCH}.txt \
191         mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
192   #tar -czf ${REFE_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.tar.gz *
193    fi
194
195    printf "Status\n$STATUS\n"  > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt
196    printf "Step.....\n$STEP\n" > mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
197
198    mesg_make; mesg_publish
199
200    exit 1
201}
202
203job_pending() {
204    time_elapsed=0; time_increment=30
205    sleep ${time_increment}
206
207    while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do
208   printf "\n####################################################" >> computation.log
209   ${JOB_VIEW} ${JOB_ID}                                           >> computation.log
210   sleep ${time_increment}
211   let time_elapsed+=${time_increment}
212    done
213
214    sleep ${time_increment}
215
216    if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then
217   STEP='Exceeded time limit'
218
219   if [ $( ${JOB_LIST} | grep ${JOB_ID} ) ]; then
220       STEP='Job not finished on time: '$(( ${TIME_LIMI}/3600 ))'h'
221   fi
222
223   print_step
224   ${JOB_DELE} ${JOB_ID} &> /dev/null
225   get_out $STATUS $STEP
226    fi
227}
Note: See TracBrowser for help on using the repository browser.