[10672] | 1 | #!/bin/ksh |
---|
| 2 | set -u |
---|
| 3 | trap "echo Received signal ERR; exit 1" ERR |
---|
| 4 | trap "echo Received signal TERM; exit 1" TERM |
---|
| 5 | |
---|
| 6 | export MY_BIN=$(cd $(dirname $0) && pwd) |
---|
| 7 | PATH=$MY_BIN:$PATH |
---|
| 8 | |
---|
| 9 | export DEBUG=false |
---|
| 10 | FCM1=true |
---|
| 11 | FCM2=true |
---|
| 12 | while getopts ":c:t:flradgh12" opt |
---|
| 13 | do |
---|
| 14 | case $opt in |
---|
| 15 | c ) CONTROL_URL=$OPTARG ;; |
---|
| 16 | t ) TEST_URL=$OPTARG ;; |
---|
| 17 | f ) FUNC_TESTS=true ;; |
---|
| 18 | l ) LOCAL_TESTS=true ;; |
---|
| 19 | r ) REMOTE_TESTS=true ;; |
---|
| 20 | a ) FUNC_TESTS=true |
---|
| 21 | LOCAL_TESTS=true |
---|
| 22 | REMOTE_TESTS=true ;; |
---|
| 23 | d ) DELETE=true ;; |
---|
| 24 | g ) DEBUG=true ;; |
---|
| 25 | h ) HELP=true ;; |
---|
| 26 | 1 ) FCM2=false ;; |
---|
| 27 | 2 ) FCM1=false ;; |
---|
| 28 | \? ) echo "Invalid option" |
---|
| 29 | HELP=true |
---|
| 30 | break ;; |
---|
| 31 | esac |
---|
| 32 | done |
---|
| 33 | if [[ $# != $(($OPTIND - 1)) ]]; then |
---|
| 34 | echo "Invalid argument" |
---|
| 35 | HELP=true |
---|
| 36 | fi |
---|
| 37 | |
---|
| 38 | if [[ ${HELP:-} == true ]]; then |
---|
| 39 | echo 'Usage: run_tests [options]' |
---|
| 40 | echo 'Valid options:' |
---|
| 41 | echo '-c URL' |
---|
| 42 | echo ' Generate the control results using the version of FCM at "URL"' |
---|
| 43 | echo '-t URL' |
---|
| 44 | echo ' Generate the test results using the version of FCM at "URL"' |
---|
| 45 | echo '-f' |
---|
| 46 | echo ' Perform the functional tests' |
---|
| 47 | echo '-l' |
---|
| 48 | echo ' Perform the local performance tests' |
---|
| 49 | echo '-r' |
---|
| 50 | echo ' Perform the remote performance tests' |
---|
| 51 | echo '-a' |
---|
| 52 | echo ' Perform all the tests (equivalent to -flr)' |
---|
| 53 | echo '-d' |
---|
| 54 | echo ' Remove any previous test results before starting' |
---|
| 55 | echo '-g' |
---|
| 56 | echo ' Output additional diagnostics' |
---|
| 57 | echo '-h' |
---|
| 58 | echo ' Print this help message' |
---|
| 59 | echo '-1' |
---|
| 60 | echo ' Only run the FCM1 tests' |
---|
| 61 | echo '-2' |
---|
| 62 | echo ' Only run the FCM2 tests' |
---|
| 63 | exit 1 |
---|
| 64 | fi |
---|
| 65 | |
---|
| 66 | if [[ -n "${CONTROL_URL:-}" ]]; then |
---|
| 67 | TYPES="control" |
---|
| 68 | fi |
---|
| 69 | if [[ -n "${TEST_URL:-}" ]]; then |
---|
| 70 | TYPES="${TYPES:-} test" |
---|
| 71 | fi |
---|
| 72 | if [[ -z "${TYPES:-}" ]]; then |
---|
| 73 | echo "Either a control or a test URL must be specified" |
---|
| 74 | exit 1 |
---|
| 75 | fi |
---|
| 76 | |
---|
| 77 | if [[ ${REMOTE_TESTS:-} == true ]]; then |
---|
| 78 | export HPC=$(rose host-select -q hpc) |
---|
| 79 | export BASE_DIR_HPC=$(ssh $HPC 'echo $PWD')/working/fcm_test_suite |
---|
| 80 | fi |
---|
| 81 | |
---|
| 82 | export BASE_DIR=$LOCALTEMP/fcm_test_suite |
---|
| 83 | if [[ ${DELETE:-} == true ]]; then |
---|
| 84 | if $DEBUG; then |
---|
| 85 | echo "Removing any previous test directory ..." |
---|
| 86 | fi |
---|
| 87 | rm -rf $BASE_DIR |
---|
| 88 | if [[ ${REMOTE_TESTS:-} == true ]]; then |
---|
| 89 | ssh $HPC "rm -rf $BASE_DIR_HPC" |
---|
| 90 | fi |
---|
| 91 | fi |
---|
| 92 | mkdir -p $BASE_DIR |
---|
| 93 | |
---|
| 94 | export REPOS_DIR=$BASE_DIR/test_svn |
---|
| 95 | export REPOS_URL="file://$REPOS_DIR" |
---|
| 96 | if [[ ! -d $REPOS_DIR ]]; then |
---|
| 97 | echo "$(date): Creating test repository ..." |
---|
| 98 | $MY_BIN/create_repos > $BASE_DIR/repos.stdout 2> $BASE_DIR/repos.stderr |
---|
| 99 | fi |
---|
| 100 | |
---|
| 101 | cp $MY_BIN/compare_*_fcm* $BASE_DIR |
---|
| 102 | PATH_BASE=$MY_BIN/wrapper_scripts:$PATH:~opsrc/ops0/mpi/mpich2-1.4-ukmo-v1/ifort-12/bin |
---|
| 103 | |
---|
| 104 | trap "" ERR |
---|
| 105 | export TEST |
---|
| 106 | export TYPE |
---|
| 107 | for TYPE in $TYPES |
---|
| 108 | do |
---|
| 109 | if [[ $TYPE == test ]]; then |
---|
| 110 | URL=$TEST_URL |
---|
| 111 | else |
---|
| 112 | URL=$CONTROL_URL |
---|
| 113 | fi |
---|
| 114 | REV=$(git describe $URL) || exit $RC |
---|
| 115 | echo "FCM version to be used: $REV" |
---|
| 116 | |
---|
| 117 | export RUN_DIR=$BASE_DIR/$TYPE |
---|
| 118 | rm -rf $RUN_DIR |
---|
| 119 | mkdir $RUN_DIR |
---|
| 120 | |
---|
| 121 | if $DEBUG; then |
---|
| 122 | echo "Creating local copy of FCM ..." |
---|
| 123 | fi |
---|
| 124 | (cd $MY_BIN/.. && git archive --format=tar --prefix=fcm/ $REV) | (cd $RUN_DIR && tar xf -) |
---|
| 125 | if [[ -a $RUN_DIR/fcm/etc/fcm.cfg ]]; then |
---|
| 126 | echo "set::url::test_suite $REPOS_URL" >>$RUN_DIR/fcm/etc/fcm.cfg |
---|
| 127 | else |
---|
| 128 | echo "location{primary}[test_suite] = $REPOS_URL" >>$RUN_DIR/fcm/etc/fcm/keyword.cfg |
---|
| 129 | fi |
---|
| 130 | export PATH=$RUN_DIR/fcm/bin:$PATH_BASE |
---|
| 131 | |
---|
| 132 | if [[ ${FUNC_TESTS:-} == true ]]; then |
---|
| 133 | . $MY_BIN/tests_functional.list |
---|
| 134 | export COMPARE_TIMES=false |
---|
| 135 | let failed=0 |
---|
| 136 | if [[ $FCM1 == true ]]; then |
---|
| 137 | for TEST in $TESTS_FCM1 |
---|
| 138 | do |
---|
| 139 | $MY_BIN/perform_test_fcm1 |
---|
| 140 | if [[ $? != 0 ]]; then |
---|
| 141 | let failed=failed+1 |
---|
| 142 | fi |
---|
| 143 | done |
---|
| 144 | fi |
---|
| 145 | if [[ $FCM2 == true ]]; then |
---|
| 146 | for TEST in $TESTS_FCM2 |
---|
| 147 | do |
---|
| 148 | $MY_BIN/perform_test_fcm2 |
---|
| 149 | if [[ $? != 0 ]]; then |
---|
| 150 | let failed=failed+1 |
---|
| 151 | fi |
---|
| 152 | done |
---|
| 153 | fi |
---|
| 154 | |
---|
| 155 | echo "$(date): Functional tests finished" |
---|
| 156 | if [[ $failed == 0 ]]; then |
---|
| 157 | echo "SUMMARY: All functional tests succeeded" |
---|
| 158 | else |
---|
| 159 | echo "SUMMARY: $failed functional tests failed" |
---|
| 160 | fi |
---|
| 161 | fi |
---|
| 162 | |
---|
| 163 | if [[ ${LOCAL_TESTS:-} == true ]]; then |
---|
| 164 | . $MY_BIN/tests_perf_local.list |
---|
| 165 | export COMPARE_TIMES=true |
---|
| 166 | export run_1=no |
---|
| 167 | export run_2=no |
---|
| 168 | let failed=0 |
---|
| 169 | if [[ $FCM1 == true ]]; then |
---|
| 170 | for TEST in $TESTS_FCM1 |
---|
| 171 | do |
---|
| 172 | $MY_BIN/perform_test_fcm1 |
---|
| 173 | if [[ $? != 0 ]]; then |
---|
| 174 | let failed=failed+1 |
---|
| 175 | fi |
---|
| 176 | done |
---|
| 177 | fi |
---|
| 178 | if [[ $FCM2 == true ]]; then |
---|
| 179 | for TEST in $TESTS_FCM2 |
---|
| 180 | do |
---|
| 181 | $MY_BIN/perform_test_fcm2 |
---|
| 182 | if [[ $? != 0 ]]; then |
---|
| 183 | let failed=failed+1 |
---|
| 184 | fi |
---|
| 185 | done |
---|
| 186 | fi |
---|
| 187 | unset run_1 run_2 |
---|
| 188 | |
---|
| 189 | echo "$(date): Local performance tests finished" |
---|
| 190 | if [[ $failed == 0 ]]; then |
---|
| 191 | echo "SUMMARY: All local performance tests succeeded" |
---|
| 192 | else |
---|
| 193 | echo "SUMMARY: $failed local performance tests failed" |
---|
| 194 | fi |
---|
| 195 | fi |
---|
| 196 | |
---|
| 197 | if [[ ${REMOTE_TESTS:-} == true ]]; then |
---|
| 198 | if $DEBUG; then |
---|
| 199 | echo "Copying files to HPC platform ..." |
---|
| 200 | fi |
---|
| 201 | export RUN_DIR_HPC=$BASE_DIR_HPC/$TYPE |
---|
| 202 | ssh $HPC "rm -rf $RUN_DIR_HPC" |
---|
| 203 | ssh $HPC "mkdir -p $RUN_DIR_HPC" |
---|
| 204 | rsync -a --rsh="ssh" $RUN_DIR/fcm $HPC:$RUN_DIR_HPC |
---|
| 205 | rsync -a --rsh="ssh" compare_*_fcm* report_hpc_results $HPC:$BASE_DIR_HPC |
---|
| 206 | |
---|
| 207 | BATCH_SCRIPT_NAME=hpc_batch.sh |
---|
| 208 | export BATCH_DIRS_NAME=hpc_dirs.sh |
---|
| 209 | export BATCH_SCRIPT=$RUN_DIR/$BATCH_SCRIPT_NAME |
---|
| 210 | $MY_BIN/create_hpc_batch_script |
---|
| 211 | export BATCH_DIRS=$RUN_DIR/$BATCH_DIRS_NAME |
---|
| 212 | |
---|
| 213 | . $MY_BIN/tests_perf_remote.list |
---|
| 214 | export COMPARE_TIMES=true |
---|
| 215 | export mirror=remote |
---|
| 216 | let failed=0 |
---|
| 217 | echo 'TESTS_FCM1="' >$BATCH_DIRS |
---|
| 218 | if [[ $FCM1 == true ]]; then |
---|
| 219 | for TEST in $TESTS_FCM1 |
---|
| 220 | do |
---|
| 221 | $MY_BIN/perform_test_fcm1 |
---|
| 222 | if [[ $? != 0 ]]; then |
---|
| 223 | let failed=failed+1 |
---|
| 224 | else |
---|
| 225 | SUBMIT_REMOTE=true |
---|
| 226 | fi |
---|
| 227 | done |
---|
| 228 | fi |
---|
| 229 | echo '"' >>$BATCH_DIRS |
---|
| 230 | echo 'TESTS_FCM2="' >>$BATCH_DIRS |
---|
| 231 | if [[ $FCM2 == true ]]; then |
---|
| 232 | export NPROC=6 |
---|
| 233 | for TEST in $TESTS_FCM2 |
---|
| 234 | do |
---|
| 235 | $MY_BIN/perform_test_fcm2 |
---|
| 236 | if [[ $? != 0 ]]; then |
---|
| 237 | let failed=failed+1 |
---|
| 238 | else |
---|
| 239 | SUBMIT_REMOTE=true |
---|
| 240 | fi |
---|
| 241 | done |
---|
| 242 | fi |
---|
| 243 | echo '"' >>$BATCH_DIRS |
---|
| 244 | unset mirror NPROC |
---|
| 245 | |
---|
| 246 | if [[ ${SUBMIT_REMOTE:-} == true ]]; then |
---|
| 247 | echo "$(date): Submitting HPC build job ..." |
---|
| 248 | rsync -a --rsh="ssh" $BATCH_SCRIPT $BATCH_DIRS $HPC:$RUN_DIR_HPC |
---|
| 249 | ssh $HPC "llsubmit $RUN_DIR_HPC/$BATCH_SCRIPT_NAME" |
---|
| 250 | fi |
---|
| 251 | |
---|
| 252 | echo "$(date): HPC performance tests finished" |
---|
| 253 | if [[ $failed == 0 ]]; then |
---|
| 254 | echo "SUMMARY: All HPC performance tests succeeded" |
---|
| 255 | else |
---|
| 256 | echo "SUMMARY: $failed HPC performance tests failed" |
---|
| 257 | fi |
---|
| 258 | fi |
---|
| 259 | done |
---|