linux-test-kb-long-run

#!/bin/bash
#
# Copyright (C) Extensible Service Proxy Authors
# All rights reserved.
#
# Redistribution and use in source and binary forms, with or without
# modification, are permitted provided that the following conditions
# are met:
# 1. Redistributions of source code must retain the above copyright
#    notice, this list of conditions and the following disclaimer.
# 2. Redistributions in binary form must reproduce the above copyright
#    notice, this list of conditions and the following disclaimer in the
#    documentation and/or other materials provided with the distribution.
#
# THIS SOFTWARE IS PROVIDED BY THE AUTHOR AND CONTRIBUTORS ``AS IS'' AND
# ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
# IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
# ARE DISCLAIMED.  IN NO EVENT SHALL THE AUTHOR OR CONTRIBUTORS BE LIABLE
# FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
# DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
# OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
# HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
# LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
# OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
# SUCH DAMAGE.
#
################################################################################
#

# This script runs a long-running test against it.

ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/.." && pwd)"
. ${ROOT}/script/all-utilities \
  || { echo "Cannot load Bash utilities" ; exit 1 ; }

API_KEY=''
SERVICE_NAME='testing-dot-endpoints-jenkins.appspot.com'
HOST=''
DURATION_IN_HOUR=0

while getopts :a:h:l:s: arg; do
  case ${arg} in
    a) API_KEY="${OPTARG}";;
    h) HOST="${OPTARG}";;
    l) DURATION_IN_HOUR="${OPTARG}";;
    s) SERVICE_NAME="${OPTARG}";;
    *) echo "Invalid option: -${OPTARG}";;
  esac
done

[[ -n "${HOST}" ]] || error_exit 'Please specify a host with -h option.'

if ! [[ -n "${API_KEY}" ]]; then
  set_api_keys;
  API_KEY="${ENDPOINTS_JENKINS_API_KEY}"
fi

# Download api Keys with restrictions from Cloud storage.
TEMP_DIR="$(mktemp -d)"
API_RESTRICTION_KEYS_FILE="${TEMP_DIR}/esp-e2e-key-restriction.json"
gsutil cp gs://esp-testing-secret-files/esp-e2e-key-restriction.json \
      "${API_RESTRICTION_KEYS_FILE}" \
        || error_exit "Failed to download API key with restrictions file."

POST_FILE="${ROOT}/test/data/35k.json"

pushd ${ROOT}/test/client > /dev/null

END_TIME=$(date +"%s")
END_TIME=$((END_TIME + DURATION_IN_HOUR * 60 * 60))
RUN_COUNT=0
STRESS_FAILURES=0
BOOKSTORE_FAILURES=0

detect_memory_leak_init ${HOST}

while true; do
  ((RUN_COUNT++))

  #######################
  # Insert tests here
  #######################

  echo "Starting test run ${RUN_COUNT} at $(date)."
  echo "Failures so far: Stress: ${STRESS_FAILURES}, Bookstore: ${BOOKSTORE_FAILURES}."

  # Generating token for each run, that they expire in 1 hour.
  JWT_TOKEN=`${ROOT}/client/custom/gen-auth-token.sh -a ${SERVICE_NAME}`
  echo "Auth token is: ${JWT_TOKEN}"

  echo "Starting bookstore test at $(date)."
  ./esp_bookstore_test.py \
      --host=${HOST} \
      --api_key=${API_KEY} \
      --auth_token=${JWT_TOKEN} \
      --allow_unverified_cert=true \
    || ((BOOKSTORE_FAILURES++))

  echo "Starting bookstore API Key restriction test at $(date)."
  ./esp_bookstore_key_restriction_test.py \
      --host=${HOST} \
      --allow_unverified_cert=true \
      --key_restriction_tests=${ROOT}/test/bookstore/test/key_restriction_test.json.template \
      --key_restriction_keys_file=${API_RESTRICTION_KEYS_FILE} \
    || ((BOOKSTORE_FAILURES++))

# temporary disabled for unstable test
#  echo "Starting bookstore quota control test at $(date)."
#  ./esp_bookstore_quota_test.py \
#      --host=${HOST} \
#      --api_key=${API_KEY} \
#      --auth_token=${JWT_TOKEN} \
#      --allow_unverified_cert=true \
#    || ((BOOKSTORE_FAILURES++))

  echo "Starting stress test at $(date)."
  ./esp_client.py \
    --test=stress \
    --host=${HOST} \
    --api_key=${API_KEY} \
    --auth_token=${JWT_TOKEN} \
    --post_file=${POST_FILE} \
    || ((STRESS_FAILURES++))

  echo "Starting negative stress test."
  ./esp_client.py \
    --test=negative \
    --test_data=negative_test_data.json.temp \
    --host=${HOST} \
    --api_key=${API_KEY} \
    --auth_token=${JWT_TOKEN} \
    --post_file=${POST_FILE} \

  #######################
  # End of test suite
  #######################

  detect_memory_leak_check ${RUN_COUNT}

  # Break if test has run long enough.
  [[ $(date +"%s") -lt ${END_TIME} ]] || break
done

echo "Finished ${RUN_COUNT} test runs."
echo "Failures: Stress: ${STRESS_FAILURES}, Bookstore: ${BOOKSTORE_FAILURES}."

RESULT=0
# We fail the test if all bookstore runs failed.
[[ ${BOOKSTORE_FAILURES} == ${RUN_COUNT} ]] \
  && RESULT=1

# We fail the test if memory increase is large.
detect_memory_leak_final && MEMORY_LEAK=0 || MEMORY_LEAK=1

exit $((${RESULT} + ${MEMORY_LEAK}))