Make it easier to install different Knative versions in E2E tests (#2494

)

* Make it easier to install different Knative versions in E2E tests

Bonuses:
* remove dead/unnecessary code in presubmits
* update upgrade tests to Knative 0.2.2
* update test-infra to the latest version to get
  * --gcp-project for E2E tests
  * --run-test for presubmit tests
  * assorted minor enhancements/fixes

* Add missing failure handling.

* Rename env var.

test/cluster.sh

@@ -18,12 +18,16 @@
 
 source $(dirname $0)/../vendor/github.com/knative/test-infra/scripts/e2e-tests.sh
 
-CLUSTER_SH_CREATED_MANIFESTS=0
+# Current YAMLs used to install Knative Serving.
+INSTALL_ISTIO_CRD_YAML=""
+INSTALL_ISTIO_YAML=""
+INSTALL_RELEASE_YAML=""
 
 # Create all manifests required to install Knative Serving.
-function create_manifests() {
-  # Don't generate twice.
-  (( CLUSTER_SH_CREATED_MANIFESTS )) && return 0
+# This will build everything from the current source.
+# All generated YAMLs will be available and pointed by the corresponding
+# environment variables as set in /hack/generate-yamls.sh.
+function build_knative_from_source() {
   local YAML_LIST="$(mktemp)"
   # Generate manifests, capture environment variables pointing to the YAML files.
   local FULL_OUTPUT="$( \
@@ -36,24 +40,34 @@ function create_manifests() {
   echo "${LOG_OUTPUT}"
   echo -e "Generated manifests:\n${ENV_OUTPUT}"
   eval "${ENV_OUTPUT}"
-  CLUSTER_SH_CREATED_MANIFESTS=1
 }
 
 # Installs Knative Serving in the current cluster, and waits for it to be ready.
+# If no parameters are passed, installs the current source-based build.
+# Parameters: $1 - Istio CRD YAML file
+#             $2 - Istio YAML file
+#             $3 - Knative Serving YAML file
 function install_knative_serving() {
-  export KO_DOCKER_REPO=${DOCKER_REPO_OVERRIDE}
-  create_manifests
+  INSTALL_ISTIO_CRD_YAML=$1
+  INSTALL_ISTIO_YAML=$2
+  INSTALL_RELEASE_YAML=$3
+  if [[ -z "${INSTALL_ISTIO_CRD_YAML}" ]]; then
+    build_knative_from_source
+    INSTALL_ISTIO_CRD_YAML="${ISTIO_CRD_YAML}"
+    INSTALL_ISTIO_YAML="${ISTIO_YAML}"
+    # TODO(#2122): Use RELEASE_YAML once we have monitoring e2e.
+    INSTALL_RELEASE_YAML="${RELEASE_NO_MON_YAML}"
+  fi
+  echo ">> Installing Knative serving"
+  echo "Istio CRD YAML: ${INSTALL_ISTIO_CRD_YAML}"
+  echo "Istio YAML: ${INSTALL_ISTIO_YAML}"
+  echo "Knative YAML: ${INSTALL_RELEASE_YAML}"
   echo ">> Bringing up Istio"
-  kubectl apply -f "${ISTIO_CRD_YAML}" || return 1
-  kubectl apply -f "${ISTIO_YAML}" || return 1
+  kubectl apply -f "${INSTALL_ISTIO_CRD_YAML}" || return 1
+  kubectl apply -f "${INSTALL_ISTIO_YAML}" || return 1
 
   echo ">> Bringing up Serving"
-  if [[ -z "${RELEASE_YAML_OVERRIDE}" ]]; then
-    # TODO(#2122): Use RELEASE_YAML once we have monitoring e2e.
-    kubectl apply -f "${RELEASE_NO_MON_YAML}" || return 1
-  else
-    kubectl apply -f "${RELEASE_YAML_OVERRIDE}" || return 1
-  fi
+  kubectl apply -f "${INSTALL_RELEASE_YAML}" || return 1
 
   # Due to the lack of Status in Istio, we have to ignore failures in initial requests.
   #
@@ -78,32 +92,28 @@ function install_knative_serving() {
 
 # Uninstalls Knative Serving from the current cluster.
 function uninstall_knative_serving() {
-  create_manifests
+  if [[ -z "${INSTALL_RELEASE_YAML}" ]]; then
+    echo "install_knative_serving() was not called, nothing to uninstall"
+    return 0
+  fi
+  echo ">> Uninstalling Knative serving"
+  echo "Istio YAML: ${INSTALL_ISTIO_YAML}"
+  echo "Knative YAML: ${INSTALL_RELEASE_YAML}"
   echo ">> Removing test resources (test/config/)"
   ko delete --ignore-not-found=true -f test/config/ || return 1
 
   echo ">> Bringing down Serving"
-  # TODO(#2122): Use RELEASE_YAML once we have monitoring e2e.
-  if [[ -z "${RELEASE_YAML_OVERRIDE}" ]]; then
-    ko delete --ignore-not-found=true -f "${RELEASE_NO_MON_YAML}" || return 1
-  else
-    ko delete --ignore-not-found=true -f "${RELEASE_YAML_OVERRIDE}" || return 1
-  fi
+  ko delete --ignore-not-found=true -f "${INSTALL_RELEASE_YAML}" || return 1
 
   echo ">> Bringing down Istio"
-  kubectl delete --ignore-not-found=true -f ${ISTIO_YAML} || return 1
+  kubectl delete --ignore-not-found=true -f "${INSTALL_ISTIO_YAML}" || return 1
   kubectl delete --ignore-not-found=true clusterrolebinding cluster-admin-binding
 }
 
-# Create test namespace
-function create_namespace() {
-  echo ">> Creating namespace serving-tests"
-  kubectl create namespace serving-tests
-}
-
 # Publish all e2e test images in ${REPO_ROOT_DIR}/test/test_images/
 function publish_test_images() {
   echo ">> Publishing test images"
+  kubectl create namespace serving-tests
   local image_dirs="$(find ${REPO_ROOT_DIR}/test/test_images -mindepth 1 -maxdepth 1 -type d)"
   for image_dir in ${image_dirs}; do
     ko publish -P "github.com/knative/serving/test/test_images/$(basename ${image_dir})" || return 1

test/e2e-tests.sh

@@ -71,7 +71,6 @@ publish_test_images || fail_test "one or more test images weren't published"
 # Run the tests
 
 header "Running tests"
-create_namespace
 go_test_e2e -timeout=20m ./test/conformance ./test/e2e || fail_test
 
 success

test/performance-tests.sh

@@ -21,8 +21,7 @@ source $(dirname $0)/cluster.sh
 
 # Deletes everything created on the cluster including all knative and istio components.
 function teardown() {
-  # Delete the service now that the test is done
-  RELEASE_YAML_OVERRIDE=$RELEASE_YAML uninstall_knative_serving
+  uninstall_knative_serving
 }
 
 initialize $@
@@ -33,13 +32,13 @@ header "Setting up environment"
 set +o errexit
 set +o pipefail
 
-create_manifests
-
-RELEASE_YAML_OVERRIDE=$RELEASE_YAML install_knative_serving || fail_test "Knative Serving installation failed"
+# Build Knative, but don't install the default "no monitoring" version
+build_knative_from_source
+install_knative_serving "${ISTIO_CRD_YAML}" "${ISTIO_YAML}" "${RELEASE_YAML}" \
+    || fail_test "Knative Serving installation failed"
 publish_test_images || fail_test "one or more test images weren't published"
 
-create_namespace || fail_test "cannot create test namespace"
-
+# Run the tests
 go_test_e2e -tags=performance -timeout=5m ./test/performance || fail_test
 
 success

test/presubmit-tests.sh

@@ -44,16 +44,6 @@ function unit_tests() {
   report_go_test ./...
 }
 
-function integration_tests() {
-  local options=""
-  (( EMIT_METRICS )) && options="--emit-metrics"
-  ./test/e2e-tests.sh ${options}
-}
-
-function upgrade_tests() {
-  local options=""
-  (( EMIT_METRICS )) && options="--emit-metrics"
-  ./test/upgrade-tests.sh ${options}
-}
+# We use the default integration test runner.
 
 main $@

test/upgrade-tests.sh

@@ -37,48 +37,33 @@ source $(dirname $0)/cluster.sh
 # version will make tests either:
 # 1. Still pass, meaning we can upgrade from earlier than latest release (good).
 # 2. Fail, which might be remedied by bumping this version.
-readonly SERVING_RELEASE_YAML=https://github.com/knative/serving/releases/download/v0.2.1/release-no-mon.yaml
-
-# Cribbed from e2e-tests.sh
-# TODO(#2320): Remove this.
-function publish_test_images() {
-  echo ">> Publishing test images"
-  image_dirs="$(find ${REPO_ROOT_DIR}/test/test_images -mindepth 1 -maxdepth 1 -type d)"
-  for image_dir in ${image_dirs}; do
-    ko publish -P "github.com/knative/serving/test/test_images/$(basename ${image_dir})"
-  done
-}
+readonly LATEST_SERVING_RELEASE_VERSION=0.2.2
 
 function install_latest_release() {
-  header "Installing latest release"
-  set -o errexit
-  set -o pipefail
-  RELEASE_YAML_OVERRIDE=${SERVING_RELEASE_YAML} install_knative_serving
-  set +o errexit
-  set +o pipefail
+  header "Installing Knative latest public release"
+  local url="https://github.com/knative/serving/releases/download/v${LATEST_SERVING_RELEASE_VERSION}"
+  install_knative_serving \
+    "${url}/istio-crds.yaml" \
+    "${url}/istio.yaml" \
+    "${url}/release-no-mon.yaml" \
+    || fail_test "Knative latest release installation failed"
 }
 
 function install_head() {
-  header "Installing HEAD"
-  set -o errexit
-  set -o pipefail
-  install_knative_serving
-  set +o errexit
-  set +o pipefail
+  header "Installing Knative head release"
+  install_knative_serving || fail_test "Knative head release installation failed"
 }
 
 # Deletes everything created on the cluster including all knative and istio components.
 function teardown() {
   uninstall_knative_serving
-  RELEASE_YAML_OVERRIDE=${SERVING_RELEASE_YAML} uninstall_knative_serving
 }
 
 # Script entry point.
 
 initialize $@
 
 header "Setting up environment"
-kubectl create namespace serving-tests
 publish_test_images
 
 install_latest_release