Implement CLI and E2E testing using the KUDO test harness.

Fixes #368 (support CLI commands)
Fixes #369 (support install kudo frameworks)
Fixes #520 (panic when executing test harness)

This adds a `kubectl` setting to the KUDO test harness TestSuite and TestStep configurations to support running kubectl commands
as a part of tests.

Makefile

@@ -26,7 +26,7 @@ test:
 
 .PHONY: integration-test
 # Run integration tests
-integration-test:
+integration-test: cli-test
 	go test -tags integration ./pkg/... ./cmd/... -v -mod=readonly -coverprofile cover-integration.out
 	go run ./cmd/kubectl-kudo test
 
@@ -117,6 +117,11 @@ generate:
 generate-clean:
 	rm -rf hack/code-gen
 
+.PHONY: cli-test
+# Build CLI for tests
+cli-test:
+	go build -o bin/${CLI} cmd/kubectl-kudo/main.go
+
 .PHONY: cli
 # Build CLI
 cli: prebuild

go.mod

@@ -15,14 +15,12 @@ require (
 	github.com/go-playground/universal-translator v0.16.0 // indirect
 	github.com/go-test/deep v1.0.1
 	github.com/gobuffalo/envy v1.6.15 // indirect
-	github.com/gogo/protobuf v1.2.1
-	github.com/golang/lint v0.0.0-20180702182130-06c8688daad7 // indirect
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf
 	github.com/hashicorp/golang-lru v0.5.1 // indirect
 	github.com/huandu/xstrings v1.2.0 // indirect
 	github.com/imdario/mergo v0.3.7 // indirect
-	github.com/jteeuwen/go-bindata v0.0.0-20180305030458-6025e8de665b // indirect
 	github.com/leodido/go-urn v1.1.0 // indirect
 	github.com/markbates/inflect v1.0.4 // indirect
 	github.com/masterminds/sprig v2.18.0+incompatible
@@ -44,8 +42,6 @@ require (
 	golang.org/x/text v0.3.2 // indirect
 	golang.org/x/tools v0.0.0-20190312170243-e65039ee4138
 	google.golang.org/appengine v1.5.0 // indirect
-	gopkg.in/airbrake/gobrake.v2 v2.0.9 // indirect
-	gopkg.in/gemnasium/logrus-airbrake-hook.v2 v2.1.2 // indirect
 	gopkg.in/go-playground/assert.v1 v1.2.1 // indirect
 	gopkg.in/go-playground/validator.v9 v9.27.0
 	gopkg.in/yaml.v2 v2.2.2
@@ -55,8 +51,6 @@ require (
 	k8s.io/apimachinery v0.0.0-20190404173353-6a84e37a896d
 	k8s.io/client-go v11.0.1-0.20190409021438-1a26190bd76a+incompatible
 	k8s.io/code-generator v0.0.0-20181117043124-c2090bec4d9b
-	k8s.io/gengo v0.0.0-20190128074634-0689ccc1d7d6
-	k8s.io/klog v0.3.0
 	sigs.k8s.io/controller-runtime v0.2.0-beta.1.0.20190619203651-3bc157084f53
 	sigs.k8s.io/controller-tools v0.1.11
 	sigs.k8s.io/kind v0.4.0

pkg/apis/kudo/v1alpha1/test_types.go

@@ -40,6 +40,8 @@ type TestSuite struct {
 	Parallel int `json:"parallel"`
 	// The directory to output artifacts to (current working directory if not specified).
 	ArtifactsDir string `json:"artifactsDir"`
+	// Kubectl commands to run before running any tests.
+	Kubectl []string `json:"kubectl"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -58,6 +60,9 @@ type TestStep struct {
 	// Indicates that this is a unit test - safe to run without a real Kubernetes cluster.
 	UnitTest bool `json:"unitTest"`
 
+	// Kubectl commands to run at the start of the test
+	Kubectl []string `json:"kubectl"`
+
 	// Allowed environment labels
 	// Disallowed environment labels
 }

pkg/test/case.go

@@ -125,6 +125,10 @@ func (t *Case) CollectTestStepFiles() (map[int64][]string, error) {
 	for _, file := range files {
 		matches := testStepRegex.FindStringSubmatch(file.Name())
 
+		if len(matches) < 2 {
+			continue
+		}
+
 		index, err := strconv.ParseInt(matches[1], 10, 32)
 		if err != nil {
 			return nil, err
@@ -168,6 +172,7 @@ func (t *Case) LoadTestSteps() error {
 		testStep := &Step{
 			Timeout: t.Timeout,
 			Index:   int(index),
+			Dir:     t.Dir,
 			Asserts: []runtime.Object{},
 			Apply:   []runtime.Object{},
 			Errors:  []runtime.Object{},

pkg/test/case_test.go

@@ -234,10 +234,12 @@ func TestLoadTestSteps(t *testing.T) {
 
 			assert.Equal(t, len(tt.testSteps), len(testStepsVal))
 			for index := range tt.testSteps {
+				tt.testSteps[index].Dir = tt.path
 				assert.Equal(t, tt.testSteps[index].Apply, testStepsVal[index].Apply)
 				assert.Equal(t, tt.testSteps[index].Asserts, testStepsVal[index].Asserts)
 				assert.Equal(t, tt.testSteps[index].Errors, testStepsVal[index].Errors)
 				assert.Equal(t, tt.testSteps[index].Step, testStepsVal[index].Step)
+				assert.Equal(t, tt.testSteps[index].Dir, testStepsVal[index].Dir)
 				assert.Equal(t, tt.testSteps[index], testStepsVal[index])
 			}
 		})

pkg/test/harness.go

@@ -1,10 +1,12 @@
 package test
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"os"
 	"path/filepath"
 	"sync"
 	"testing"
@@ -14,6 +16,7 @@ import (
 	"github.com/kudobuilder/kudo/pkg/controller"
 	testutils "github.com/kudobuilder/kudo/pkg/test/utils"
 	"github.com/kudobuilder/kudo/pkg/webhook"
+	"github.com/pkg/errors"
 	"k8s.io/client-go/discovery"
 	"k8s.io/client-go/rest"
 	"k8s.io/client-go/tools/clientcmd"
@@ -162,7 +165,18 @@ func (h *Harness) Config() (*rest.Config, error) {
 		h.config, err = config.GetConfig()
 	}
 
-	return h.config, err
+	if err != nil {
+		return h.config, err
+	}
+
+	f, err := os.Create("kubeconfig")
+	if err != nil {
+		return h.config, err
+	}
+
+	defer f.Close()
+
+	return h.config, testutils.Kubeconfig(h.config, f)
 }
 
 // RunKUDO starts the KUDO controllers and installs the required CRDs.
@@ -231,6 +245,28 @@ func (h *Harness) DiscoveryClient() (discovery.DiscoveryInterface, error) {
 	return h.dclient, err
 }
 
+// DoKubectl runs all kubectl commands defined for the test suite.
+func (h *Harness) DoKubectl() error {
+	if h.TestSuite.Kubectl == nil {
+		return nil
+	}
+
+	for _, cmd := range h.TestSuite.Kubectl {
+		stdout := &bytes.Buffer{}
+		stderr := &bytes.Buffer{}
+
+		h.T.Log("Running kubectl:", cmd)
+
+		if err := testutils.Kubectl(context.TODO(), "default", cmd, "", stdout, stderr); err != nil {
+			return errors.Wrap(err, stderr.String())
+		}
+
+		h.T.Log(stdout.String())
+	}
+
+	return nil
+}
+
 // RunTests should be called from within a Go test (t) and launches all of the KUDO integration
 // tests at dir.
 func (h *Harness) RunTests() {
@@ -299,6 +335,10 @@ func (h *Harness) Run() {
 		}
 	}
 
+	if err := h.DoKubectl(); err != nil {
+		h.T.Fatal(err)
+	}
+
 	if h.TestSuite.StartKUDO {
 		if err := h.RunKUDO(); err != nil {
 			h.T.Fatal(err)
@@ -326,7 +366,12 @@ func (h *Harness) Stop() {
 	}
 
 	if h.TestSuite.SkipClusterDelete || h.TestSuite.SkipDelete {
-		// TODO: figure out how to write the Kubernetes client configuration to disk.
+		cwd, _ := os.Getwd()
+		kubeconfig := filepath.Join(cwd, "kubeconfig")
+
+		h.T.Log("skipping cluster tear down")
+		h.T.Log(fmt.Sprintf("to connect to the cluster, run: export KUBECONFIG=\"%s\"", kubeconfig))
+
 		return
 	}
 

pkg/test/step.go

@@ -1,6 +1,7 @@
 package test
 
 import (
+	"bytes"
 	"context"
 	"fmt"
 	"path/filepath"
@@ -26,6 +27,8 @@ type Step struct {
 	Name  string
 	Index int
 
+	Dir string
+
 	Step   *kudo.TestStep
 	Assert *kudo.TestAssert
 
@@ -183,6 +186,32 @@ func (s *Step) Create(namespace string) []error {
 	return errors
 }
 
+// DoKubectl runs all kubectl commands defined for the test step.
+func (s *Step) DoKubectl(namespace string) []error {
+	errs := []error{}
+
+	if s.Step == nil || s.Step.Kubectl == nil {
+		return errs
+	}
+
+	for _, cmd := range s.Step.Kubectl {
+		stdout := &bytes.Buffer{}
+		stderr := &bytes.Buffer{}
+
+		s.Logger.Log("Running kubectl:", cmd)
+
+		err := testutils.Kubectl(context.TODO(), namespace, cmd, s.Dir, stdout, stderr)
+		if err != nil {
+			errs = append(errs, errors.New(stderr.String()))
+			errs = append(errs, err)
+		}
+
+		s.Logger.Log(stdout.String())
+	}
+
+	return errs
+}
+
 // GetTimeout gets the timeout defined for the test step.
 func (s *Step) GetTimeout() int {
 	timeout := s.Timeout
@@ -299,7 +328,8 @@ func (s *Step) Run(namespace string) []error {
 		return []error{err}
 	}
 
-	testErrors := s.Create(namespace)
+	testErrors := s.DoKubectl(namespace)
+	testErrors = append(testErrors, s.Create(namespace)...)
 
 	if len(testErrors) != 0 {
 		return testErrors

pkg/test/test_data/cli-test/00-assert.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cli-test-pod

pkg/test/test_data/cli-test/00-create-pod.yaml

@@ -0,0 +1,4 @@
+apiVersion: kudo.dev/v1alpha1
+kind: TestStep
+kubectl:
+- apply -f ./test_data/pod.yaml

pkg/test/test_data/cli-test/test_data/pod.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cli-test-pod
+spec:
+  containers:
+  - name: cli-test
+    image: alpine
+  restartPolicy: Never