New URL for NEMO forge!   http://forge.nemo-ocean.eu

Since March 2022 along with NEMO 4.2 release, the code development moved to a self-hosted GitLab.
This present forge is now archived and remained online for history.
trusting_func.sh in branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST – NEMO

source: branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting_func.sh @ 5633

Last change on this file since 5633 was 5633, checked in by nicolasmartin, 9 years ago

dev_r5092_CNRS_SETTE Right namelist_cfg for ORCA1 && simplify softs releases information

  • Property eol-style set to native
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Rev URL
File size: 9.0 KB
Line 
1#!/bin/bash
2
3
4comments() {
5    state=$1; LAST_TIME_STEP=0
6
7    if [ "$state" == 'E R R O R' ]; then
8   [ -e time.step ] && LAST_TIME_STEP=$( cat time.step | tr -d ' ' )
9   export LAST_TIME_STEP
10   [ -e ocean.output ] && line=$( grep -m1 -A4 "$state" ocean.output | tr -d '\n' )
11    else
12   [ -e ocean.output ] && line=$( grep -m1 -A3 "^ $state" ocean.output | tr -d '\n' )
13    fi
14
15#    if [[ -e ${REFE_DIR}/model.log && $( diff -q model.log ${REFE_DIR}/model.log ) ]]; then
16#  line=$( diff model.log ${REFE_DIR}/model.log  | grep '>' | sed '{N;s/\n/<BR>/}' )
17#    fi
18
19    printf "Comments\n$line\n" | tee mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
20}
21
22
23diff_inputs() {
24    mesg='Same'
25    for file in inputs_list.txt $( ls namelist_* ) $( ls *.xml ); do
26   diff -q $file ${REFE_DIR}/$file > /dev/null
27   [ $? -ne 0 ] && mesg='Different'
28    done
29
30    printf "Input files\n$mesg\n" | tee mesg_09_inputs_${CFG_USER}_${CFG_ARCH}.txt
31}
32
33diff_results() {
34    for file in ocean.output $( ls *.stat ); do
35   [ ! -e ${REFE_DIR}/$file ] && export ST='FAILED' && get_out 7
36   diff -q $file ${REFE_DIR}/$file
37   [ $? -ne 0 ] && export ST='FAILED'
38    done
39}
40
41diff_restart() {
42    [ ! -e  ${REFE_DIR}/time.step ] && export ST='FAILED' && get_out 9
43
44    export LAST_TIME_STEP=$( cat ${REFE_DIR}/time.step | tr -d [:space:] )
45    echo 'Last time step of standard run: '${LAST_TIME_STEP}
46
47    if [ $( find -regex ".*_0+${LAST_TIME_STEP}_restart.*\.nc" -print -quit ) ]; then
48   base_name=$( find -regex ".*_0+${LAST_TIME_STEP}_restart.*\.nc" -print -quit | awk -F/ '{print $NF}' \
49                | sed "s/^\(.*\)$LAST_TIME_STEP\_restart.*/\1$LAST_TIME_STEP\_/"                         )
50
51   for comp in restart restart_ice restart_trc; do
52       file=${base_name}${comp} && printf "$file.nc: "
53
54       nb_dom=$( find -name "${file}_[0-9]*.nc" | wc -l | awk '{print $1}' )
55       if   [ ${nb_dom} -gt 1 ]; then
56      ${NEMO_TRBD}/rebuild_nemo -t $NPROC $file ${nb_dom} > /dev/null
57      [ $? -eq 0 ] && rm -f ${base_name}${comp}_[0-9]*.nc > /dev/null
58       elif [ ${nb_dom} -eq 0 ]; then
59      export ST='FAILED' && get_out 10
60       fi
61
62       if [ -e ${REFE_DIR}/$file.nc ]; then
63       ## UNIX `cmp` not suitable (filename & timestamp in .nc file)
64      nc_diff=$( $CDO diffn $file.nc ${REFE_DIR}/$file.nc 2> /dev/null \
65                 | awk '/records/ {print $1}'                           )
66
67      if [[ ! -z ${nc_diff} && ${nc_diff} -ne 0 ]]; then
68          export ST='FAILED'
69          printf "$CDO ${nc_diff}\n"
70      else
71          printf "identical\n"
72      fi
73
74       else
75      export ST='FAILED' && get_out 9
76       fi
77
78   done
79
80    else
81   export ST='FAILED'
82    fi
83
84}
85
86get_cpu_time() {
87    real_cpu_time=$( eval ${JOB_TIME} )
88    printf "Real CPU time\n${real_cpu_time}\n" | tee mesg_10_cputime_${CFG_USER}_${CFG_ARCH}.txt
89}
90
91get_inputs() {
92    if [ ! -z "${FORC_TARF}" ]; then
93   cmd_iol="tar -tvf ${NEMO_FORC}/${FORC_TARF}"; cmd_iof="tar -vxf ${NEMO_FORC}/${FORC_TARF}"
94    else
95   cmd_iol="ls ${NEMO_FORC}/*"                 ; cmd_iof="\cp ${NEMO_FORC}/* ."
96    fi
97
98    ${cmd_iol} > inputs_list.txt && ${cmd_iof} > /dev/null
99}
100
101get_soft_rel() {
102    for soft in $CDO $COMPILER $MPI $NETCDF; do
103                                     soft_rel=$( echo $LOADEDMODULES | grep -o "$soft[^:]*"  )
104   [ -z "${soft_rel}"  ]     && soft_rel=$( echo $PATH          | grep -o "$soft/[^/]*" )
105   ## Not working inside CURIE cron (no modules command && cdo embedded in netcdf-*)
106   #[ "$soft" == "$COMPILER" ] && soft_rel=$( $soft --version      | grep -m1 -o ' [0-9.]* '         )
107   [[ "$soft" == "$CDO"    && -z "${soft_rel}" ]] && soft_rel=$( echo $PATH | grep -o "netcdf-utils[^/]*" )
108   [[ "$soft" == "$NETCDF" && -z "${soft_rel}" ]] && soft_rel=$( echo $PATH | grep -o "netcdf[^/]*"  | sort | head -1 )
109   echo ${soft_rel} >> model.log
110    done
111
112    sed -n 4p model.log | sed 'i\Fortran compiler' > mesg_06_compiler_${CFG_USER}_${CFG_ARCH}.txt
113    sed -n 5p model.log | sed 'i\MPI libs'         > mesg_07_mpi_${CFG_USER}_${CFG_ARCH}.txt
114    sed -n 6p model.log | sed 'i\NetCDF libs'      > mesg_08_netcdf_${CFG_USER}_${CFG_ARCH}.txt
115}
116
117get_nemo_rev() {
118    svn_opt='status'
119
120    ## If -v|--version option has been set, modify svn command
121    if   [ $( echo ${NEMO_VERS} | grep  "HEAD\|up\|update"                     ) ]; then
122   svn_cmd='svn update -r HEAD'
123    elif [ $( echo ${NEMO_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' ) ]; then
124   svn_cmd='svn update -r '$( echo ${NEMO_VERS} | grep -o '{[0-9]\{4\}-[0-9]\{2\}-[0-9]\{2\}}' )
125    elif [ $( echo ${NEMO_VERS} | grep -o '[0-9]*'                             ) ]; then
126   svn_cmd='svn update -r '$( echo ${NEMO_VERS} | grep -o '[0-9]*' )
127    else
128   svn_cmd='svn '${svn_opt}
129    fi
130
131    rev=0
132    for dir in ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_ENGI} \
133               ${NEMO_EIOI} ${NEMO_EFCM}              \
134          ${NEMO_TCMP} ${NEMO_TRBD}              \
135          ${DIR_XIOS}                             ; do
136
137   ## For time being, just get revision number from XIOS
138   if [ "$dir" == "${DIR_XIOS}" ]; then
139       rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' )
140       echo   'XIOS '${rev_loc} >> model.log
141       printf "XIOS rev.\n"     >  mesg_05_xios_${CFG_USER}_${CFG_ARCH}.txt
142       printf "<a href=\"https://forge.ipsl.jussieu.fr/ioserver/changeset/${rev_loc}\" target=\"_blank\">${rev_loc}</a>" \
143                                >> mesg_05_xios_${CFG_USER}_${CFG_ARCH}.txt
144       continue
145   fi
146
147   echo ${svn_cmd} $dir': '$( ${svn_cmd} $dir )
148   rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' )
149   (( ${rev_loc} >= $rev )) && rev=${rev_loc}
150    done
151
152    echo   'NEMOGCM '$rev   >> model.log
153    printf "NEMOGCM rev.\n" >  mesg_04_nemogcm_${CFG_USER}_${CFG_ARCH}.txt
154    printf "<a href=\"https://forge.ipsl.jussieu.fr/nemo/changeset/$rev\" target=\"_blank\">$rev</a>" \
155                       >> mesg_04_nemogcm_${CFG_USER}_${CFG_ARCH}.txt
156}
157
158mesg_make() {
159    paste -d ';' mesg_*_${CFG_USER}_${CFG_ARCH}.txt > trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt
160    cat                                               trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt
161}
162
163mesg_publish() {
164 
165if [ $PUBLISH -eq 1 ]; then
166
167  if [ -f ${REFE_DIR}/trusting_${CFG_USER}_${CFG_ARCH}.txt ]; then cmd='tail -1'; else cmd='cat'; fi
168
169  $cmd trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt >> ${REFE_DIR}/trusting_${CFG_USER}_${CFG_ARCH}.txt
170
171  ## Send mail only when FAILED
172  if [[ ! -z $EMAIL && "$ST" == 'FAILED' ]]; then
173      cat << END_MAIL > trusting.mail
174XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
175
176Dear $USER,
177
178The trusting sequence for ${REFE_CONF} has failed.
179Directory: ${TEST_DIR}
180Archive: trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.tar.gz
181
182XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX
183
184END_MAIL
185
186      if [ -e ${TEST_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt ]; then
187     cat ${TEST_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.txt  >> trusting.mail
188      fi
189      mail -s "[trusting ${REFE_CONF} $BRANCH] $ST $ERR" $EMAIL  <  trusting.mail
190  fi
191
192fi
193}
194
195print_step() { printf "Step.....\n$1\n"; }
196
197get_out() {
198    ERR=$1
199
200    cd ${TEST_DIR}
201    printf "Status\n$ST\n"  > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt
202
203    if [ "$ST" == 'FAILED' ]; then
204   file=mesg_09_inputs_${CFG_USER}_${CFG_ARCH}.txt
205   [ ! -e $file ] && printf "Input files\n\n"   > $file
206   file=mesg_10_cputime_${CFG_USER}_${CFG_ARCH}.txt
207   [ ! -e $file ] && printf "Real CPU time\n\n" > $file
208   file=mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
209   [ ! -e $file ] && printf "Comments\n\n"      > $file
210
211        ## Error identification
212   case $ERR in
213       ## Compilation
214       1) ERR='XIOS compilation failed'               ;;  2) ERR='NEMO compilation failed'            ;;
215       ## Submission
216       3) ERR='Missing input files'                   ;;  4) ERR='Job submission error'               ;;
217       ## Running
218       5) ERR='Crashed at time step '${LAST_TIME_STEP};;  6) ERR='Exceeded time limit '${TIME_LIMI}'h';;
219       ## Results
220       7) ERR='Missing previous outputs '             ;;  8) ERR='New outputs  differ/missing'        ;;
221       9) ERR='Missing previous restarts'             ;; 10) ERR='New restarts differ/missing'        ;;
222       ## Undefined
223       *) ERR='Unknown error'                         ;;
224   esac
225
226   export ERR
227    fi
228
229    printf "Result\n$ERR\n" > mesg_03_result_${CFG_USER}_${CFG_ARCH}.txt
230
231    ## Save tested configuration if trusting failed
232    [[ "$ST" == 'FAILED' && $DEBUG -eq 0 ]] && tar -czf ${REFE_DIR}/trusting_${DATESTR}_${CFG_USER}_${CFG_ARCH}.tar.gz *
233
234    mesg_make
235    mesg_publish
236
237    exit 0
238}
239
240job_pending() {
241    time_elapsed=0; time_increment=30
242
243    sleep ${time_increment}
244    while [[ $( ${JOB_LIST} | grep ${JOB_ID} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do
245   printf "\n####################################################\n" >> computation.log
246   ${JOB_INFO} ${JOB_ID}                                             >> computation.log
247   sleep ${time_increment}
248   time_elapsed=$(( ${time_elapsed} + ${time_increment} ))
249    done
250    sleep ${time_increment}
251
252    if [ ${time_elapsed} -eq ${TIME_LIMI} ]; then
253   ${JOB_DELE} ${JOB_ID} &> /dev/null
254   TIME_LIMIT=$(( ${TIME_LIMI} / 3600 ))
255   export ${TIME_LIMIT}
256   get_out $ST ERR=6
257    fi
258}
Note: See TracBrowser for help on using the repository browser.