source: branches/2015/dev_r5092_CNRS_SETTE/NEMOGCM/TRUST/trusting.sh @ 5424

Last change on this file since 5424 was 5424, checked in by nicolasmartin, 5 years ago

dev_r5092_CNRS_SETTE Many improvments & bug corrections

  • Property eol-style set to native
  • Property svn:executable set to *
  • Property svn:keywords set to Author Date Id Rev URL
File size: 7.7 KB
Line 
1#!/bin/bash
2#
3# Author: Patrick Brockmann
4# Contact: Patrick.Brockmann@cea.fr
5# History:
6# Modification: Nicolas-thierry.Martin@locean-ipsl.upmc.fr
7#
8
9
10# Set defaults
11#---------------------------------------------------
12NEMO_TRUS=$( pwd $( dirname $0 ) )
13SVN_CMD='svn status'
14DEBUG=0; PUBLISH=0; status='OK'
15
16# Get options for replacing some initials settings
17#---------------------------------------------------
18while [ $# -ne 0 ]; do
19    case $1 in
20        -b|--branch ) NEMO_BRAN=$2         ; shift 2;; -d|--debug  ) set -vx; DEBUG=1     ; shift  ;;
21   -e|--email  ) MAIL=$2              ; shift 2;; -j|--job    ) NPROC=$2             ; shift 2;;
22   -h|--help   ) cat trusting_help.txt; exit  1;; -m|--machine) CFG_ARCH=$2          ; shift 2;;
23   -n|--newconf) TEST_CONF=$2         ; shift 2;; -r|--refconf) REFE_CONF=$2         ; shift 2;;
24   -t|--time   ) TIME_LIMI=$2         ; shift 2;; -p|--publish) PUBLISH=1            ; shift  ;;
25        -u|--user   ) CFG_USER=$2          ; shift 2;; -v|--version) NEMO_VERS=$2         ; shift 2;;
26                   *) cat trusting_help.txt; exit  1;;
27    esac
28done
29
30#export DEBUG PUBLISH
31
32
33# Initialization (super-computer & user environment)
34#---------------------------------------------------
35if [[ ! -z "${CFG_ARCH}" && ! -z "${CFG_USER}" ]]; then
36
37    if   [ ! $( find ${NEMO_TRUS}/cfg -name ${CFG_USER}.cfg ) ]; then
38   echo 'No configuration file in '${NEMO_TRUS}'/cfg for '${CFG_USER}; exit 1
39    elif [ ! $( find ${NEMO_TRUS}/cfg -name ${CFG_ARCH}.cfg ) ]; then
40   echo 'No configuration file in '${NEMO_TRUS}'/cfg for '${CFG_ARCH}; exit 1
41    else
42   . ${NEMO_TRUS}/trusting.env
43    fi
44
45else
46    cat ${NEMO_TRUS}/trusting_help.txt; exit 1
47fi
48
49# If -v|--version option has been set, modify svn command
50[ $( echo ${NEMO_VERS} | grep  "up\|update"         ) ] && SVN_CMD='svn update'
51[ $( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' ) ] && SVN_CMD='svn update -r '$( echo ${NEMO_VERS} | tr -d '[:alpha:][:punct:]' )
52
53. ./trusting_func.sh
54
55
56#
57#---------------------------------------------------
58mkdir -p ${TEST_DIR} ${REFE_DIR}
59echo ${TEST_DIR} && cd ${TEST_DIR}
60datestr1=$( LC_ALL=C date -d"${DATE_STR}" +%Y-%m-%dT%H:%M )
61printf "Date\n%s\n"       $datestr1 > mesg_01_date_${CFG_USER}_${CFG_ARCH}.txt
62# Only at end status will change to OK
63printf "Status\nFAILED\n"           > mesg_03_status_${CFG_USER}_${CFG_ARCH}.txt
64
65
66# Check softwares versions
67#---------------------------------------------------
68printf "Step......................\nGet softwares release\n"                     | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt
69. ${ARCH_ENV} > /dev/null    # Sourcing architecture environment
70get_soft_rel
71sed -n 2p model.log | sed 'i\Fortran compiler' | tee mesg_05_compiler_${CFG_USER}_${CFG_ARCH}.txt
72sed -n 3p model.log | sed 'i\MPI libraries   ' | tee mesg_06_mpi_${CFG_USER}_${CFG_ARCH}.txt
73sed -n 4p model.log | sed 'i\NetCDF libraries' | tee mesg_07_netcdf_${CFG_USER}_${CFG_ARCH}.txt
74
75
76# SVN action on XIOS & NEMO essentials directories
77#---------------------------------------------------
78printf "Step......................\nGet NEMO SVN revision with: ${SVN_CMD}\n"    | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt
79get_nemo_rev
80sed -n 6p model.log | sed 'i\NEMO revision'    | tee  mesg_02_nemo_${CFG_USER}_${CFG_ARCH}.txt
81
82
83# NEMO config compiled from scratch
84#---------------------------------------------------
85printf "Step......................\nCompiling ${CONF_REF} configuration\n"       | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt
86cd ${NEMO_CONF}
87#[ -d ${TEST_CONF} ] && ./makenemo -n ${TEST_CONF} clean
88./makenemo -n ${TEST_CONF} -r ${REFE_CONF} -m $ARCH -j $NPROC >& /dev/null
89[ ! -e ${TEST_CONF}/BLD/bin/nemo.exe ] && get_out
90
91
92# Get namelists, xml & forced files for running
93#---------------------------------------------------
94cd ${TEST_DIR}
95printf "Step......................\nPreparing job\n"                             | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt
96find ${NEMO_CONF}/${TEST_CONF}/EXP00 -regex '.*\(cfg\|opa\|ref\|xml\)' -exec cp {} . \;
97
98if [ ! -z "${FORC_TAR}" ]; then
99    cmd_iol="tar -tvf ${NEMO_FORC}/${FORC_TAR}"; cmd_iof="tar -vxf ${NEMO_FORC}/${FORC_TAR}"
100else
101    cmd_iol="ls ${NEMO_FORC}/*"                ; cmd_iof="\cp ${NEMO_FORC}/* ."
102fi
103
104${cmd_iol} > inputs_list.txt && ${cmd_iof} > /dev/null
105[ $? -ne 0 ] && get_out
106[ $( find . -name '*.gz' -print -quit ) ] && gunzip *.gz
107
108
109# Job submission & computing
110#---------------------------------------------------
111printf "Step......................\nRunning\n"                                   | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt; echo ${JOB_SUBM}
112job_num=$( ${JOB_SUBM} | awk '{print $NF}' )
113[ $? -ne 0 ] && get_out
114
115time_elapsed=0; time_increment=30
116sleep ${time_increment}
117while [[ $( ${JOB_LIST} | grep ${job_num} ) && ${time_elapsed} -lt ${TIME_LIMI} ]]; do
118    printf "\n####################################################" >> computation.log
119    ${JOB_VIEW} ${job_num}                                          >> computation.log
120    sleep ${time_increment}
121    let time_elapsed+=${time_increment}
122done
123sleep ${time_increment}
124
125# Eventual crash during computing
126#---------------------------------------------------
127printf "Step......................\nTest if ASCII output files exist\n"          | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt
128if   [ ! -f ocean.output                  ]; then
129    printf "Step......................\nOpa crashed at initialization\n"         | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt
130    ${JOB_DELE} ${job_num} >/dev/null 2>&1
131    status='FAILED'
132elif [ $( grep 'E R R O R' ocean.output ) ]; then
133    printf "Step......................\nOpa crashed\n"                           | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt
134    grep 'E R R O R'       ocean.output | tee    mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt
135    sed 's/^/Time step :/' time.step    | tee -a mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt
136    ${JOB_DELE} ${job_num} >/dev/null 2>&1
137    status='FAILED'
138fi
139
140
141# Inspect output text files
142#---------------------------------------------------
143printf "Step......................\nTest ASCII output files difference\n"        | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt
144for file in ocean.output solver.stat tracer.stat; do
145    diff $file ${REFE_DIR}/$file
146    [ $? -ne 0 ] && status='FAILED'
147done
148
149
150# Inspect output NetCDF files
151#---------------------------------------------------
152printf "Step......................\nTest last restart NetCDF files difference\n" | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt
153diff_restart 2>/dev/null
154[ $? -ne 0 ] && status='FAILED'
155
156
157# Finals tests
158#---------------------------------------------------
159if [ $status != 'FAILED' ]; then
160
161    # Comments
162    #---------------------------------------------------
163    echo 'Comments'                    | tee    mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt
164    grep 'W A R N I N G:' ocean.output | tee -a mesg_10_comments_${CFG_USER}_${CFG_ARCH}.txt
165
166    # Portability !
167    printf "Step......................\nGet real CPU time\n"                     | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt
168    real_cpu_time=$( ${JOB_VIEW} ${job_num} | awk 'END{print $9}' )
169    printf "Real CPU time\n%s\n" ${real_cpu_time} | tee mesg_08_realcputime_${CFG_USER}_${CFG_ARCH}.txt
170    printf "Input files\nNA"                      | tee mesg_09_inputfiles_${CFG_USER}_${CFG_ARCH}.txt
171else
172    printf "Step......................\nTest input files difference\n"           | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt
173    diff_inputs
174    get_out
175fi
176
177
178# So far, everything is good
179#---------------------------------------------------
180printf "Status\nOK\n" > mesg_03_status_${CFG_USER}_${CFG_ARCH}.txt
181printf "Step......................\nCode is reliable\n"                          | tee mesg_04_step_${CFG_USER}_${CFG_ARCH}.txt
182
183
184# End
185#---------------------------------------------------
186get_out
Note: See TracBrowser for help on using the repository browser.