1 | #!/bin/ksh |
---|
2 | set -u |
---|
3 | trap "echo Received signal ERR; exit 1" ERR |
---|
4 | trap "echo Received signal TERM; exit 1" TERM |
---|
5 | |
---|
6 | export MY_BIN=$(cd $(dirname $0) && pwd) |
---|
7 | PATH=$MY_BIN:$PATH |
---|
8 | |
---|
9 | export DEBUG=false |
---|
10 | FCM1=true |
---|
11 | FCM2=true |
---|
12 | while getopts ":c:t:flradgh12" opt |
---|
13 | do |
---|
14 | case $opt in |
---|
15 | c ) CONTROL_URL=$OPTARG ;; |
---|
16 | t ) TEST_URL=$OPTARG ;; |
---|
17 | f ) FUNC_TESTS=true ;; |
---|
18 | l ) LOCAL_TESTS=true ;; |
---|
19 | r ) REMOTE_TESTS=true ;; |
---|
20 | a ) FUNC_TESTS=true |
---|
21 | LOCAL_TESTS=true |
---|
22 | REMOTE_TESTS=true ;; |
---|
23 | d ) DELETE=true ;; |
---|
24 | g ) DEBUG=true ;; |
---|
25 | h ) HELP=true ;; |
---|
26 | 1 ) FCM2=false ;; |
---|
27 | 2 ) FCM1=false ;; |
---|
28 | \? ) echo "Invalid option" |
---|
29 | HELP=true |
---|
30 | break ;; |
---|
31 | esac |
---|
32 | done |
---|
33 | if [[ $# != $(($OPTIND - 1)) ]]; then |
---|
34 | echo "Invalid argument" |
---|
35 | HELP=true |
---|
36 | fi |
---|
37 | |
---|
38 | if [[ ${HELP:-} == true ]]; then |
---|
39 | echo 'Usage: run_tests [options]' |
---|
40 | echo 'Valid options:' |
---|
41 | echo '-c URL' |
---|
42 | echo ' Generate the control results using the version of FCM at "URL"' |
---|
43 | echo '-t URL' |
---|
44 | echo ' Generate the test results using the version of FCM at "URL"' |
---|
45 | echo '-f' |
---|
46 | echo ' Perform the functional tests' |
---|
47 | echo '-l' |
---|
48 | echo ' Perform the local performance tests' |
---|
49 | echo '-r' |
---|
50 | echo ' Perform the remote performance tests' |
---|
51 | echo '-a' |
---|
52 | echo ' Perform all the tests (equivalent to -flr)' |
---|
53 | echo '-d' |
---|
54 | echo ' Remove any previous test results before starting' |
---|
55 | echo '-g' |
---|
56 | echo ' Output additional diagnostics' |
---|
57 | echo '-h' |
---|
58 | echo ' Print this help message' |
---|
59 | echo '-1' |
---|
60 | echo ' Only run the FCM1 tests' |
---|
61 | echo '-2' |
---|
62 | echo ' Only run the FCM2 tests' |
---|
63 | exit 1 |
---|
64 | fi |
---|
65 | |
---|
66 | if [[ -n "${CONTROL_URL:-}" ]]; then |
---|
67 | TYPES="control" |
---|
68 | fi |
---|
69 | if [[ -n "${TEST_URL:-}" ]]; then |
---|
70 | TYPES="${TYPES:-} test" |
---|
71 | fi |
---|
72 | if [[ -z "${TYPES:-}" ]]; then |
---|
73 | echo "Either a control or a test URL must be specified" |
---|
74 | exit 1 |
---|
75 | fi |
---|
76 | |
---|
77 | if [[ ${REMOTE_TESTS:-} == true ]]; then |
---|
78 | export HPC=$(rose host-select -q hpc) |
---|
79 | export BASE_DIR_HPC=$(ssh $HPC 'echo $PWD')/working/fcm_test_suite |
---|
80 | fi |
---|
81 | |
---|
82 | export BASE_DIR=$LOCALTEMP/fcm_test_suite |
---|
83 | if [[ ${DELETE:-} == true ]]; then |
---|
84 | if $DEBUG; then |
---|
85 | echo "Removing any previous test directory ..." |
---|
86 | fi |
---|
87 | rm -rf $BASE_DIR |
---|
88 | if [[ ${REMOTE_TESTS:-} == true ]]; then |
---|
89 | ssh $HPC "rm -rf $BASE_DIR_HPC" |
---|
90 | fi |
---|
91 | fi |
---|
92 | mkdir -p $BASE_DIR |
---|
93 | |
---|
94 | export REPOS_DIR=$BASE_DIR/test_svn |
---|
95 | export REPOS_URL="file://$REPOS_DIR" |
---|
96 | if [[ ! -d $REPOS_DIR ]]; then |
---|
97 | echo "$(date): Creating test repository ..." |
---|
98 | $MY_BIN/create_repos > $BASE_DIR/repos.stdout 2> $BASE_DIR/repos.stderr |
---|
99 | fi |
---|
100 | |
---|
101 | cp $MY_BIN/compare_*_fcm* $BASE_DIR |
---|
102 | PATH_BASE=$MY_BIN/wrapper_scripts:$PATH:~opsrc/ops0/mpi/mpich2-1.4-ukmo-v1/ifort-12/bin |
---|
103 | |
---|
104 | trap "" ERR |
---|
105 | export TEST |
---|
106 | export TYPE |
---|
107 | for TYPE in $TYPES |
---|
108 | do |
---|
109 | if [[ $TYPE == test ]]; then |
---|
110 | URL=$TEST_URL |
---|
111 | else |
---|
112 | URL=$CONTROL_URL |
---|
113 | fi |
---|
114 | REV=$(git describe $URL) || exit $RC |
---|
115 | echo "FCM version to be used: $REV" |
---|
116 | |
---|
117 | export RUN_DIR=$BASE_DIR/$TYPE |
---|
118 | rm -rf $RUN_DIR |
---|
119 | mkdir $RUN_DIR |
---|
120 | |
---|
121 | if $DEBUG; then |
---|
122 | echo "Creating local copy of FCM ..." |
---|
123 | fi |
---|
124 | (cd $MY_BIN/.. && git archive --format=tar --prefix=fcm/ $REV) | (cd $RUN_DIR && tar xf -) |
---|
125 | if [[ -a $RUN_DIR/fcm/etc/fcm.cfg ]]; then |
---|
126 | echo "set::url::test_suite $REPOS_URL" >>$RUN_DIR/fcm/etc/fcm.cfg |
---|
127 | else |
---|
128 | echo "location{primary}[test_suite] = $REPOS_URL" >>$RUN_DIR/fcm/etc/fcm/keyword.cfg |
---|
129 | fi |
---|
130 | export PATH=$RUN_DIR/fcm/bin:$PATH_BASE |
---|
131 | |
---|
132 | if [[ ${FUNC_TESTS:-} == true ]]; then |
---|
133 | . $MY_BIN/tests_functional.list |
---|
134 | export COMPARE_TIMES=false |
---|
135 | let failed=0 |
---|
136 | if [[ $FCM1 == true ]]; then |
---|
137 | for TEST in $TESTS_FCM1 |
---|
138 | do |
---|
139 | $MY_BIN/perform_test_fcm1 |
---|
140 | if [[ $? != 0 ]]; then |
---|
141 | let failed=failed+1 |
---|
142 | fi |
---|
143 | done |
---|
144 | fi |
---|
145 | if [[ $FCM2 == true ]]; then |
---|
146 | for TEST in $TESTS_FCM2 |
---|
147 | do |
---|
148 | $MY_BIN/perform_test_fcm2 |
---|
149 | if [[ $? != 0 ]]; then |
---|
150 | let failed=failed+1 |
---|
151 | fi |
---|
152 | done |
---|
153 | fi |
---|
154 | |
---|
155 | echo "$(date): Functional tests finished" |
---|
156 | if [[ $failed == 0 ]]; then |
---|
157 | echo "SUMMARY: All functional tests succeeded" |
---|
158 | else |
---|
159 | echo "SUMMARY: $failed functional tests failed" |
---|
160 | fi |
---|
161 | fi |
---|
162 | |
---|
163 | if [[ ${LOCAL_TESTS:-} == true ]]; then |
---|
164 | . $MY_BIN/tests_perf_local.list |
---|
165 | export COMPARE_TIMES=true |
---|
166 | export run_1=no |
---|
167 | export run_2=no |
---|
168 | let failed=0 |
---|
169 | if [[ $FCM1 == true ]]; then |
---|
170 | for TEST in $TESTS_FCM1 |
---|
171 | do |
---|
172 | $MY_BIN/perform_test_fcm1 |
---|
173 | if [[ $? != 0 ]]; then |
---|
174 | let failed=failed+1 |
---|
175 | fi |
---|
176 | done |
---|
177 | fi |
---|
178 | if [[ $FCM2 == true ]]; then |
---|
179 | for TEST in $TESTS_FCM2 |
---|
180 | do |
---|
181 | $MY_BIN/perform_test_fcm2 |
---|
182 | if [[ $? != 0 ]]; then |
---|
183 | let failed=failed+1 |
---|
184 | fi |
---|
185 | done |
---|
186 | fi |
---|
187 | unset run_1 run_2 |
---|
188 | |
---|
189 | echo "$(date): Local performance tests finished" |
---|
190 | if [[ $failed == 0 ]]; then |
---|
191 | echo "SUMMARY: All local performance tests succeeded" |
---|
192 | else |
---|
193 | echo "SUMMARY: $failed local performance tests failed" |
---|
194 | fi |
---|
195 | fi |
---|
196 | |
---|
197 | if [[ ${REMOTE_TESTS:-} == true ]]; then |
---|
198 | if $DEBUG; then |
---|
199 | echo "Copying files to HPC platform ..." |
---|
200 | fi |
---|
201 | export RUN_DIR_HPC=$BASE_DIR_HPC/$TYPE |
---|
202 | ssh $HPC "rm -rf $RUN_DIR_HPC" |
---|
203 | ssh $HPC "mkdir -p $RUN_DIR_HPC" |
---|
204 | rsync -a --rsh="ssh" $RUN_DIR/fcm $HPC:$RUN_DIR_HPC |
---|
205 | rsync -a --rsh="ssh" compare_*_fcm* report_hpc_results $HPC:$BASE_DIR_HPC |
---|
206 | |
---|
207 | BATCH_SCRIPT_NAME=hpc_batch.sh |
---|
208 | export BATCH_DIRS_NAME=hpc_dirs.sh |
---|
209 | export BATCH_SCRIPT=$RUN_DIR/$BATCH_SCRIPT_NAME |
---|
210 | $MY_BIN/create_hpc_batch_script |
---|
211 | export BATCH_DIRS=$RUN_DIR/$BATCH_DIRS_NAME |
---|
212 | |
---|
213 | . $MY_BIN/tests_perf_remote.list |
---|
214 | export COMPARE_TIMES=true |
---|
215 | export mirror=remote |
---|
216 | let failed=0 |
---|
217 | echo 'TESTS_FCM1="' >$BATCH_DIRS |
---|
218 | if [[ $FCM1 == true ]]; then |
---|
219 | for TEST in $TESTS_FCM1 |
---|
220 | do |
---|
221 | $MY_BIN/perform_test_fcm1 |
---|
222 | if [[ $? != 0 ]]; then |
---|
223 | let failed=failed+1 |
---|
224 | else |
---|
225 | SUBMIT_REMOTE=true |
---|
226 | fi |
---|
227 | done |
---|
228 | fi |
---|
229 | echo '"' >>$BATCH_DIRS |
---|
230 | echo 'TESTS_FCM2="' >>$BATCH_DIRS |
---|
231 | if [[ $FCM2 == true ]]; then |
---|
232 | export NPROC=6 |
---|
233 | for TEST in $TESTS_FCM2 |
---|
234 | do |
---|
235 | $MY_BIN/perform_test_fcm2 |
---|
236 | if [[ $? != 0 ]]; then |
---|
237 | let failed=failed+1 |
---|
238 | else |
---|
239 | SUBMIT_REMOTE=true |
---|
240 | fi |
---|
241 | done |
---|
242 | fi |
---|
243 | echo '"' >>$BATCH_DIRS |
---|
244 | unset mirror NPROC |
---|
245 | |
---|
246 | if [[ ${SUBMIT_REMOTE:-} == true ]]; then |
---|
247 | echo "$(date): Submitting HPC build job ..." |
---|
248 | rsync -a --rsh="ssh" $BATCH_SCRIPT $BATCH_DIRS $HPC:$RUN_DIR_HPC |
---|
249 | ssh $HPC "llsubmit $RUN_DIR_HPC/$BATCH_SCRIPT_NAME" |
---|
250 | fi |
---|
251 | |
---|
252 | echo "$(date): HPC performance tests finished" |
---|
253 | if [[ $failed == 0 ]]; then |
---|
254 | echo "SUMMARY: All HPC performance tests succeeded" |
---|
255 | else |
---|
256 | echo "SUMMARY: $failed HPC performance tests failed" |
---|
257 | fi |
---|
258 | fi |
---|
259 | done |
---|