source: branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh @ 5355

Last change on this file since 5355 was 5355, checked in by nicolasmartin, 5 years ago

dev_r5092_CNRS_SETTE Many improvments

  • Property eol-style set to native
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Rev URL
File size: 8.2 KB
Line 
1#!/bin/bash
2#
3# Author: Patrick Brockmann
4# Contact: Patrick.Brockmann@cea.fr
5# History:
6# Modification: Nicolas-thierry.Martin@locean-ipsl.upmc.fr
7#
8
9
10# Set defaults
11#---------------------------------------------------
12NEMO_TRUS=$( pwd $( dirname $0 ) )
13DEBUG=0; PUBLISH=0; status='OK'
14
15# Get options for replacing some initials settings
16#---------------------------------------------------
17while [ $# -ne 0 ]; do
18    case $1 in
19        -b|--branch ) NEMO_BRAN=$2         ; shift 2;; -d|--debug  ) set -vx; DEBUG=1     ; shift  ;;
20   -e|--email  ) MAIL=$2              ; shift 2;; -j|--job    ) NPROC=$2             ; shift 2;;
21   -h|--help   ) cat trusting_help.txt; exit  1;; -m|--machine) CFG_ARCH=$2          ; shift 2;;
22   -n|--newconf) TEST_CONF=$2         ; shift 2;; -r|--refconf) REFE_CONF=$2         ; shift 2;;
23   -t|--time   ) TIME_LIMI=$2         ; shift 2;; -p|--publish) PUBLISH=1            ; shift  ;;
24        -u|--user   ) CFG_USER=$2          ; shift 2;; -v|--version) NEMO_REVI=$2         ; shift 2;;
25                   *) cat trusting_help.txt; exit  1;;
26    esac
27done
28
29#export DEBUG PUBLISH
30
31
32# Initialization (super-computer & user environment)
33#---------------------------------------------------
34if [[ ! -z "${CFG_ARCH}" && ! -z "${CFG_USER}" ]]; then
35
36    if   [ ! $( find ${NEMO_TRUS}/cfg -name ${CFG_USER}.cfg ) ]; then
37   echo 'No configuration file in '${NEMO_TRUS}'/cfg for '${CFG_USER}; exit 1
38    elif [ ! $( find ${NEMO_TRUS}/cfg -name ${CFG_ARCH}.cfg ) ]; then
39   echo 'No configuration file in '${NEMO_TRUS}'/cfg for '${CFG_ARCH}; exit 1
40    else
41   . ${NEMO_TRUS}/trusting.env
42    fi
43
44else
45    cat ${NEMO_TRUS}/trusting_help.txt; exit 1
46fi
47
48datestr1=$( LC_ALL=C date -d"${DATE_STR}" +%Y-%m-%dT%H:%M )
49. ./trusting_func.sh
50
51mkdir -p ${TEST_DIR} ${REFE_DIR}
52echo ${TEST_DIR} && cd ${TEST_DIR}
53printf "Date\n%s\n"       $datestr1 > mesg_01_date_${CFG_USER}_${CFG_ARCH}.txt
54# Only at end status will change to OK
55printf "Status\nFAILED\n"           > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt
56
57
58# Check softwares versions
59#---------------------------------------------------
60printf "Step......................\nGet softwares release\n"                     | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
61for rel in $CDO $COMPILER $MPI $NETCDF; do
62                             arch_rel=$( echo $LOADEDMODULES | sed "s#.*:$rel/\([^:]*\).*#\1#" )
63    [ -z "${arch_rel}"  ] && arch_rel=$( echo $PATH          | sed "s#.*$rel/\([^/]*\).*#\1#"  )
64#    [ $( echo $rel | grep $COMPILER ) ] && arch_rel=$( $rel --version | grep -m1 -o ' [0-9.]* ' >/dev/null )
65    echo $rel ${arch_rel} >> model.log
66done
67
68sed -n 2p model.log | sed 'i\Fortran compiler' > mesg_04_compiler_${CFG_USER}_${CFG_ARCH}.txt
69sed -n 3p model.log | sed 'i\MPI libraries   ' > mesg_05_mpi_${CFG_USER}_${CFG_ARCH}.txt
70sed -n 4p model.log | sed 'i\NetCDF libraries' > mesg_06_netcdf_${CFG_USER}_${CFG_ARCH}.txt
71
72
73# SVN action on XIOS & NEMO essentials directories
74#---------------------------------------------------
75svn_cmd='svn status'
76# If -v|--version option has been set, modify svn command
77[ $( echo ${NEMO_VERS} | grep  "up\|update"         ) ] && svn_cmd='svn update'
78[ $( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) ] && svn_cmd='svn update -r '$( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' )
79
80printf "Step......................\nGet NEMO SVN revision with: ${svn_cmd}\n"    | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
81rev=0
82
83for dir in ${DIR_XIOS} ${NEMO_ARCH} ${NEMO_CONF} ${NEMO_EFCM} ${NEMO_ENGI} ${NEMO_IPSL} ${NEMO_TRUS} ${NEMO_TCMP}; do
84    # For time being, just get revision number from XIOS
85    [ $dir == ${DIR_XIOS} ] && echo 'XIOS '$( svn info $dir | awk '(NR == 9) {print $NF}' ) >> model.log && continue
86    ${svn_cmd} $dir
87    rev_loc=$( svn info $dir | awk '(NR == 9) {print $NF}' )
88    (( ${rev_loc} >= $rev )) && rev=${rev_loc}
89done
90echo 'NEMO '$rev >> model.log
91
92
93# NEMO config compiled from scratch
94#---------------------------------------------------
95printf "Step......................\nCompiling ${CONF_REF} configuration\n"       | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
96cd ${NEMO_CONF}
97. ${ARCH_ENV} > /dev/null    # Sourcing architecture environment
98#[ -d ${TEST_CONF} ] && ./makenemo -n ${TEST_CONF} clean
99./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m $ARCH -j $NPROC > /dev/null 2>&1
100[ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out
101
102
103# Get namelists, xml & forced files for running
104#---------------------------------------------------
105printf "Step......................\nPreparing job\n"                             | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
106cd ${TEST_DIR}
107find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(cfg\|opa\|ref\|xml\)' -exec cp {} . \;
108
109if [ ! -z "${NEMO_FORC}" ]; then
110    cp ${NEMO_FORC}/${FORC_TAR} .
111    tar -tvf ${FORC_TAR} > inputs_list.txt && tar -vxf ${FORC_TAR} > /dev/null
112    [ $? -ne 0 ] && get_out
113    gunzip *.gz
114else
115    touch inputs_list.txt
116fi
117
118
119# Job submission & computing
120#---------------------------------------------------
121printf "Step......................\nRunning\n"                                   | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt; echo ${JOB_SUBM}
122job_num=$( ${JOB_SUBM} | awk '{print $NF}' )
123[ $? -ne 0 ] && get_out
124
125time_elapsed=0; time_increment=30
126sleep ${time_increment}
127while [[ $( ${JOB_LIST} | grep $job_num ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do
128    printf "\n####################################################" >> computation.log
129    ${JOB_VIEW} $job_num                                            >> computation.log
130    sleep ${time_increment}
131    let time_elapsed+=${time_increment}
132done
133
134
135# Eventual crash during computing
136#---------------------------------------------------
137printf "Step......................\nTest if ASCII output files exist\n"          | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
138if   [ ! -f ocean.output                  ]; then
139    printf "Step......................\nOpa crashed at initialization\n"         | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
140    status='FAILED'
141elif [ $( grep 'E R R O R' ocean.output ) ]; then
142    printf "Step......................\nOpa crashed\n"                           | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
143    grep 'E R R O R'       ocean.output >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
144    sed 's/^/Time step :/' time.step    >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
145    status='FAILED'
146fi
147
148
149# Inspect output text files
150#---------------------------------------------------
151printf "Step......................\nTest ASCII output files difference\n"        | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
152for file in ocean.output solver.stat tracer.stat; do
153    diff $file ${REFE_DIR}/$file
154    [ $? -ne 0 ] && status='FAILED'
155done
156
157
158# Inspect output NetCDF files
159#---------------------------------------------------
160printf "Step......................\nTest last restart NetCDF files difference\n" | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
161diff_restart 2>/dev/null
162[ $? -ne 0 ] && status='FAILED'
163
164
165# Comments
166#---------------------------------------------------
167echo 'Comments'                    >  mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
168grep 'W A R N I N G:' ocean.output >> mesg_11_comments_${CFG_USER}_${CFG_ARCH}.txt
169
170
171# Finals tests
172#---------------------------------------------------
173if [ $status != 'FAILED' ]; then
174    printf "Step......................\nGet real CPU time\n"                     | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
175    real_cpu_time=$( ${JOB_VIEW} $job_num | awk 'END{print $9}' )
176    printf "Real CPU time\n%s\n" ${real_cpu_time} > mesg_08_realcputime_${CFG_USER}_${CFG_ARCH}.txt
177else
178    printf "Step......................\nTest input files difference\n"           | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
179    diff inputs_list.txt ${REFE_DIR}/inputs_list.txt
180
181    if [ $? -ne 0]; then
182   printf "Input files\nDifferent" > mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt
183    else
184   printf "Input files\nSame"      > mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt
185    fi
186
187    get_out
188fi
189
190
191# So far, everything is good
192#---------------------------------------------------
193printf "Status\nOK\n" > mesg_02_status_${CFG_USER}_${CFG_ARCH}.txt
194printf "Step......................\nCode is reliable\n"                          | tee mesg_03_step_${CFG_USER}_${CFG_ARCH}.txt
195
196
197# End
198#---------------------------------------------------
199get_out
Note: See TracBrowser for help on using the repository browser.