source: vendors/test/run_tests @ 10669

#!/bin/ksh
set -u
trap "echo Received signal ERR; exit 1"  ERR
trap "echo Received signal TERM; exit 1" TERM

export MY_BIN=$(cd $(dirname $0) && pwd)
PATH=$MY_BIN:$PATH

export DEBUG=false
FCM1=true
FCM2=true
while getopts ":c:t:flradgh12" opt
do
  case $opt in
    c ) CONTROL_URL=$OPTARG ;;
    t ) TEST_URL=$OPTARG ;;
    f ) FUNC_TESTS=true ;;
    l ) LOCAL_TESTS=true ;;
    r ) REMOTE_TESTS=true ;;
    a ) FUNC_TESTS=true
        LOCAL_TESTS=true
        REMOTE_TESTS=true ;;
    d ) DELETE=true ;;
    g ) DEBUG=true ;;
    h ) HELP=true ;;
    1 ) FCM2=false ;;
    2 ) FCM1=false ;;
    \? ) echo "Invalid option"
        HELP=true
        break ;;
  esac
done
if [[ $# != $(($OPTIND - 1)) ]]; then
  echo "Invalid argument"
  HELP=true
fi

if [[ ${HELP:-} == true ]]; then
 echo 'Usage: run_tests [options]'
 echo 'Valid options:'
 echo '-c URL'
 echo '   Generate the control results using the version of FCM at "URL"'
 echo '-t URL'
 echo '   Generate the test results using the version of FCM at "URL"'
 echo '-f'
 echo '   Perform the functional tests'
 echo '-l'
 echo '   Perform the local performance tests'
 echo '-r'
 echo '   Perform the remote performance tests'
 echo '-a'
 echo '   Perform all the tests (equivalent to -flr)'
 echo '-d'
 echo '   Remove any previous test results before starting'
 echo '-g'
 echo '   Output additional diagnostics'
 echo '-h'
 echo '   Print this help message'
 echo '-1'
 echo '   Only run the FCM1 tests'
 echo '-2'
 echo '   Only run the FCM2 tests'
 exit 1
fi

if [[ -n "${CONTROL_URL:-}" ]]; then
  TYPES="control"
fi
if [[ -n "${TEST_URL:-}" ]]; then
  TYPES="${TYPES:-} test"
fi
if [[ -z "${TYPES:-}" ]]; then
  echo "Either a control or a test URL must be specified"
  exit 1
fi

if [[ ${REMOTE_TESTS:-} == true ]]; then
  export HPC=$(rose host-select -q hpc)
  export BASE_DIR_HPC=$(ssh $HPC 'echo $PWD')/working/fcm_test_suite
fi

export BASE_DIR=$LOCALTEMP/fcm_test_suite
if [[ ${DELETE:-} == true ]]; then
  if $DEBUG; then
    echo "Removing any previous test directory ..."
  fi
  rm -rf $BASE_DIR
  if [[ ${REMOTE_TESTS:-} == true ]]; then
    ssh $HPC "rm -rf $BASE_DIR_HPC"
  fi
fi
mkdir -p $BASE_DIR

export REPOS_DIR=$BASE_DIR/test_svn
export REPOS_URL="file://$REPOS_DIR"
if [[ ! -d $REPOS_DIR ]]; then
  echo "$(date): Creating test repository ..."
  $MY_BIN/create_repos > $BASE_DIR/repos.stdout 2> $BASE_DIR/repos.stderr
fi

cp $MY_BIN/compare_*_fcm* $BASE_DIR
PATH_BASE=$MY_BIN/wrapper_scripts:$PATH:~opsrc/ops0/mpi/mpich2-1.4-ukmo-v1/ifort-12/bin

trap ""  ERR
export TEST
export TYPE
for TYPE in $TYPES
do
  if [[ $TYPE == test ]]; then
    URL=$TEST_URL
  else
    URL=$CONTROL_URL
  fi
  REV=$(git describe $URL) || exit $RC
  echo "FCM version to be used: $REV"

  export RUN_DIR=$BASE_DIR/$TYPE
  rm -rf $RUN_DIR
  mkdir $RUN_DIR

  if $DEBUG; then
    echo "Creating local copy of FCM ..."
  fi
  (cd $MY_BIN/.. && git archive --format=tar --prefix=fcm/ $REV) | (cd $RUN_DIR && tar xf -)
  if [[ -a $RUN_DIR/fcm/etc/fcm.cfg ]]; then
    echo "set::url::test_suite $REPOS_URL" >>$RUN_DIR/fcm/etc/fcm.cfg
  else
    echo "location{primary}[test_suite] = $REPOS_URL" >>$RUN_DIR/fcm/etc/fcm/keyword.cfg
  fi
  export PATH=$RUN_DIR/fcm/bin:$PATH_BASE

  if [[ ${FUNC_TESTS:-} == true ]]; then
    . $MY_BIN/tests_functional.list
    export COMPARE_TIMES=false
    let failed=0
    if [[ $FCM1 == true ]]; then
      for TEST in $TESTS_FCM1
      do
        $MY_BIN/perform_test_fcm1
        if [[ $? != 0 ]]; then
          let failed=failed+1
        fi
      done
    fi
    if [[ $FCM2 == true ]]; then
      for TEST in $TESTS_FCM2
      do
        $MY_BIN/perform_test_fcm2
        if [[ $? != 0 ]]; then
          let failed=failed+1
        fi
      done
    fi

    echo "$(date): Functional tests finished"
    if [[ $failed == 0 ]]; then
      echo "SUMMARY: All functional tests succeeded"
    else
      echo "SUMMARY: $failed functional tests failed"
    fi
  fi

  if [[ ${LOCAL_TESTS:-} == true ]]; then
    . $MY_BIN/tests_perf_local.list
    export COMPARE_TIMES=true
    export run_1=no
    export run_2=no
    let failed=0
    if [[ $FCM1 == true ]]; then
      for TEST in $TESTS_FCM1
      do
        $MY_BIN/perform_test_fcm1
        if [[ $? != 0 ]]; then
          let failed=failed+1
        fi
      done
    fi
    if [[ $FCM2 == true ]]; then
      for TEST in $TESTS_FCM2
      do
        $MY_BIN/perform_test_fcm2
        if [[ $? != 0 ]]; then
          let failed=failed+1
        fi
      done
    fi
    unset run_1 run_2

    echo "$(date): Local performance tests finished"
    if [[ $failed == 0 ]]; then
      echo "SUMMARY: All local performance tests succeeded"
    else
      echo "SUMMARY: $failed local performance tests failed"
    fi
  fi

  if [[ ${REMOTE_TESTS:-} == true ]]; then
    if $DEBUG; then
      echo "Copying files to HPC platform ..."
    fi
    export RUN_DIR_HPC=$BASE_DIR_HPC/$TYPE
    ssh $HPC "rm -rf $RUN_DIR_HPC"
    ssh $HPC "mkdir -p $RUN_DIR_HPC"
    rsync -a --rsh="ssh" $RUN_DIR/fcm $HPC:$RUN_DIR_HPC
    rsync -a --rsh="ssh" compare_*_fcm* report_hpc_results $HPC:$BASE_DIR_HPC

    BATCH_SCRIPT_NAME=hpc_batch.sh
    export BATCH_DIRS_NAME=hpc_dirs.sh
    export BATCH_SCRIPT=$RUN_DIR/$BATCH_SCRIPT_NAME
    $MY_BIN/create_hpc_batch_script
    export BATCH_DIRS=$RUN_DIR/$BATCH_DIRS_NAME

    . $MY_BIN/tests_perf_remote.list
    export COMPARE_TIMES=true
    export mirror=remote
    let failed=0
    echo 'TESTS_FCM1="' >$BATCH_DIRS
    if [[ $FCM1 == true ]]; then
      for TEST in $TESTS_FCM1
      do
        $MY_BIN/perform_test_fcm1
        if [[ $? != 0 ]]; then
          let failed=failed+1
        else
          SUBMIT_REMOTE=true
        fi
      done
    fi
    echo '"' >>$BATCH_DIRS
    echo 'TESTS_FCM2="' >>$BATCH_DIRS
    if [[ $FCM2 == true ]]; then
      export NPROC=6
      for TEST in $TESTS_FCM2
      do
        $MY_BIN/perform_test_fcm2
        if [[ $? != 0 ]]; then
          let failed=failed+1
        else
          SUBMIT_REMOTE=true
        fi
      done
    fi
    echo '"' >>$BATCH_DIRS
    unset mirror NPROC

    if [[ ${SUBMIT_REMOTE:-} == true ]]; then
      echo "$(date): Submitting HPC build job ..."
      rsync -a --rsh="ssh" $BATCH_SCRIPT $BATCH_DIRS $HPC:$RUN_DIR_HPC
      ssh $HPC "llsubmit $RUN_DIR_HPC/$BATCH_SCRIPT_NAME"
    fi

    echo "$(date): HPC performance tests finished"
    if [[ $failed == 0 ]]; then
      echo "SUMMARY: All HPC performance tests succeeded"
    else
      echo "SUMMARY: $failed HPC performance tests failed"
    fi
  fi
done