Add testing folder for deploying test infrastructure and running tests (

#34)

* Created folder for test infrastructure and set up Terraform remote state configuration

* Added Terraform config for setting up a testing cluster

* Added docker image and script for running unit tests

* Added Prow setup guide and configuration

.gitignore

@@ -44,3 +44,6 @@ vendor
 ## direnv
 .envrc
 .direnv
+
+.terraform/
+*.tfvars

Makefile

@@ -1,6 +1,9 @@
 VERSION_FILE=VERSION
 FEAST_VERSION=`cat $(VERSION_FILE)`
 
+test:
+	echo testing not implemented
+
 build-deps:
 	$(MAKE) -C protos gen-go
 	dep ensure
@@ -19,5 +22,4 @@ build-docker:
 build-push-docker:
 	@$(MAKE) build-docker registry=$(registry) version=$(version)
 	docker push $(registry)/feast-core:$(version)
-	docker push $(registry)/feast-serving:$(version)
-
+	docker push $(registry)/feast-serving:$(version)

testing/Makefile

@@ -0,0 +1,37 @@
+# 
+#  Copyright 2019 The Feast Authors
+# 
+#  Licensed under the Apache License, Version 2.0 (the "License");
+#  you may not use this file except in compliance with the License.
+#  You may obtain a copy of the License at
+# 
+#      https://www.apache.org/licenses/LICENSE-2.0
+# 
+#  Unless required by applicable law or agreed to in writing, software
+#  distributed under the License is distributed on an "AS IS" BASIS,
+#  WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+#  See the License for the specific language governing permissions and
+#  limitations under the License.
+# 
+
+PHONY: get-cluster-credentials update-config update-plugins build-push
+
+PROJECT       				?= kf-feast
+ZONE          				?= us-central1-a
+CLUSTER       				?= primary-test-cluster
+REGISTRY      				?= us.gcr.io
+VERSION       				?= 1.0.0
+PUSH     					?= docker push
+
+get-cluster-credentials:
+	gcloud container clusters get-credentials "$(CLUSTER)" --project="$(PROJECT)" --zone="$(ZONE)"
+
+update-config: get-cluster-credentials
+	kubectl create configmap config --from-file=config.yaml=prow/config.yaml --dry-run -o yaml | kubectl replace configmap config -f -
+
+update-plugins: get-cluster-credentials
+	kubectl create configmap plugins --from-file=plugins.yaml=prow/plugins.yaml --dry-run -o yaml | kubectl replace configmap plugins -f -
+
+build-push:
+	docker build test-image/ -t $(REGISTRY)/$(PROJECT)/test-image:$(VERSION)
+	$(PUSH) "$(REGISTRY)/$(PROJECT)/test-image:$(VERSION)"

testing/README.md

@@ -0,0 +1,57 @@
+# Feast Test Infrastructure
+
+This folder contains the Feast test infrastructure.
+
+## Components
+
+* test-image/ - Base docker image and script for running tests.
+* prow/ - Prow configuration (plugins and jobs)
+* tf/ - Terraform modules to provision the base testing infrastructure on GCP
+
+## Set up
+
+These steps will allow you to provision the test infrastructure in an empty GCP project. It is not necessary to rerun these steps once the infrastructure has been provisioned.
+
+1. Make sure you have access to your GCP project and that you Terraform installed.
+
+2. Create a bucket for maintaining Terraform state in your GCP project. This can be done manually. Update all `backend.tf` files to point to this bucket.
+
+3. Import the bucket to manage it's own state
+
+```
+mv tf/gcs
+terraform import google_storage_bucket.kf-feast-terraform-state kf-feast-terraform-state
+```
+
+4. Ensure that all variables are set correctly in `tf/terraform.tfvars`. It is likely that the GCP project will need to be updated.
+
+5. Create the primary Kubernetes cluster which will host Prow and Argo
+
+```
+mv ../k8s-cluster
+terraform apply -var-file="../terraform.tfvars"
+```
+
+6. Create a ci-bot account on GitHub with owner access to your organization. Create and save a personal access token for the bot.
+
+7. Install and run Tackle
+
+```
+go get -u k8s.io/test-infra/prow/cmd/tackle
+tackle
+```
+
+Follow the steps to install Prow in your cluster. If any step fails, follow this [guide](https://github.com/kubernetes/test-infra/blob/master/prow/getting_started_deploy.md) to complete the steps manually. 
+
+Prow should now be receiving events from the Feast repository. You should be able to access Prow on an Ingress IP that the cluster exposes.
+
+## Updating Prow Jobs
+
+To update Prow jobs, plugins, or the Docker image used for testing, modify one or more of the following: 
+
+- `prow/config.yaml`
+- `prow/plugins.yaml`
+- `test-image/Dockerfile`
+- `test-image/run.sh`
+
+After making modifications, run `make`. This will update the Prow configuration, build a new test image, and push it to the container registry.

testing/prow/config.yaml

@@ -0,0 +1,18 @@
+presubmits:
+  gojek/feast:
+  - name: presubmit-unit-tests
+    always_run: true
+    spec:
+      containers:
+      - image: us.gcr.io/kf-feast/test-image:1.0.0
+        args: ["make", "test"]
+        imagePullPolicy: Always
+postsubmits:
+  gojek/feast:
+  - name: postsubmit-unit-tests
+    always_run: true
+    spec:
+      containers:
+      - image: us.gcr.io/kf-feast/test-image:1.0.0
+        args: ["make", "test"]
+        imagePullPolicy: Always

testing/prow/plugins.yaml

@@ -0,0 +1,31 @@
+plugins:
+  gojek/feast:
+  - approve   # Enable /approve and /assign commands.
+  - assign
+  - help
+  - hold
+  - label
+  - lgtm
+  - lifecycle   # Lets PRs & issues be flagged as stale
+  - size
+  - verify-owners   # Validates OWNERS file changes in PRs.
+  - wip   # Applies a label to PRs with wip in the title to block merge
+  - trigger
+
+tide:
+  queries:
+  - repos:
+    - gojek/feast
+    labels:
+    - lgtm
+    - approved
+    missingLabels:
+    - do-not-merge
+    - do-not-merge/hold
+    - do-not-merge/invalid-owners-file
+    - do-not-merge/work-in-progress
+    - needs-rebase
+  merge_method:
+    gojek/feast: squash
+  blocker_label: merge-blocker
+  squash_label: tide/squash

testing/test-image/Dockerfile

@@ -0,0 +1,13 @@
+FROM openjdk:8-slim
+
+RUN apt-get update && apt-get install -y git maven
+
+RUN git clone https://github.com/gojek/feast \
+    && cd feast && mvn test || true \
+    && cd .. && rm -rf feast
+
+RUN apt-get install -y make
+
+COPY run.sh /usr/local/bin/
+
+ENTRYPOINT ["run.sh"]

testing/test-image/run.sh

@@ -0,0 +1,40 @@
+#!/usr/bin/env bash
+
+set -e
+printenv
+
+if [ $# -eq 0 ]
+then
+    echo ""
+    echo "please specify commands as arguments, for example \"make test\""
+    echo ""
+    exit 1
+fi
+
+echo ""
+echo "cloning Feast repository..."
+echo ""
+
+git clone https://github.com/gojek/feast
+cd feast
+
+if [ -n "${PULL_NUMBER}" ] && [ "$JOB_TYPE" = "presubmit" ] 
+then
+    echo ""
+    echo "fetching PR ${PULL_NUMBER}..."
+    echo ""
+    git fetch origin pull/"${PULL_NUMBER}"/head:pull_"${PULL_NUMBER}"
+
+    echo ""
+    echo "checking out PR ${PULL_NUMBER}..."
+    echo ""
+    git checkout pull_"${PULL_NUMBER}"
+fi
+
+echo "sha:"
+git rev-parse HEAD
+echo ""
+echo "running unit tests"
+echo ""
+
+$@

testing/tf/gcs/backend.tf

@@ -0,0 +1,6 @@
+terraform {
+  backend "gcs" {
+    bucket  = "kf-feast-terraform-state"
+    prefix  = "tf/gcs"
+  }
+}

testing/tf/gcs/main.tf

@@ -0,0 +1,4 @@
+resource "google_storage_bucket" "kf-feast-terraform-state" {
+  name     = "kf-feast-terraform-state"
+  location = "US"
+}

testing/tf/k8s-cluster/backend.tf

@@ -0,0 +1,6 @@
+terraform {
+  backend "gcs" {
+    bucket  = "kf-feast-terraform-state"
+    prefix  = "tf/k8s-cluster"
+  }
+}

testing/tf/k8s-cluster/main.tf

@@ -0,0 +1,23 @@
+resource "google_container_cluster" "primary-test-cluster" {
+  name       = "primary-test-cluster"
+  zone       = "${var.default_region}-a"
+  network    = "${var.default_network}"
+  subnetwork = "${var.default_subnet}"
+
+  initial_node_count = 1
+  min_master_version = "1.11.5-gke.5"
+  node_version       = "1.11.5-gke.5"
+
+  additional_zones = [
+    "${var.default_region}-b",
+    "${var.default_region}-c",
+  ]
+
+  node_config {
+    oauth_scopes = [
+      "https://www.googleapis.com/auth/cloud-platform",
+    ]
+
+    machine_type = "n1-standard-4"
+  }
+}

testing/tf/k8s-cluster/output.tf

@@ -0,0 +1,12 @@
+# The following outputs allow authentication and connectivity to the GKE Cluster.
+output "client_certificate" {
+  value = "${google_container_cluster.primary-test-cluster.master_auth.0.client_certificate}"
+}
+
+output "client_key" {
+  value = "${google_container_cluster.primary-test-cluster.master_auth.0.client_key}"
+}
+
+output "cluster_ca_certificate" {
+  value = "${google_container_cluster.primary-test-cluster.master_auth.0.cluster_ca_certificate}"
+}

testing/tf/k8s-cluster/provider.tf

@@ -0,0 +1,4 @@
+provider "google" {
+  version     = "~> 1.16"
+  project     = "${var.gcp_project}"
+}

testing/tf/k8s-cluster/variables.tf

@@ -0,0 +1,15 @@
+variable "gcp_project" {
+  description = "GCP Project name"
+}
+
+variable "default_region" {
+  description = "Kubernetes cluster region"
+}
+
+variable "default_network" {
+  description = "GCP Project name"
+}
+
+variable "default_subnet" {
+  description = "Kubernetes cluster subnet name"
+}

testing/tf/terraform.tfvars

@@ -0,0 +1,7 @@
+gcp_project = "kf-feast"
+
+default_network = "default"
+
+default_subnet = "default"
+
+default_region = "us-central1"