Implement CLI and E2E testing using the KUDO test harness. (#656)

* Implement CLI and E2E testing using the KUDO test harness.

Fixes #368 (support CLI commands)
Fixes #369 (support install kudo frameworks)
Fixes #520 (panic when executing test harness)

This adds a `kubectl` setting to the KUDO test harness TestSuite and TestStep configurations to support running kubectl commands
as a part of tests.

* Log files that are ignored by test harness.

* factor DoKubectl -> testutils.RunKubectlCommands

* Clean up kubectl argument parsing.

* Fix broken logging and log stderr instead of writing it to an error.

* make cli-test -> make cli-fast

* make cli-test -> make cli-fast

Makefile

@@ -26,7 +26,7 @@ test:
 
 .PHONY: integration-test
 # Run integration tests
-integration-test:
+integration-test: cli-fast
 	go test -tags integration ./pkg/... ./cmd/... -v -mod=readonly -coverprofile cover-integration.out
 	go run ./cmd/kubectl-kudo test
 
@@ -117,11 +117,14 @@ generate:
 generate-clean:
 	rm -rf hack/code-gen
 
+.PHONY: cli-fast
+# Build CLI but don't lint or run code generation first.
+cli-fast:
+	go build -ldflags "${LDFLAGS}" -o bin/${CLI} cmd/kubectl-kudo/main.go
+
 .PHONY: cli
 # Build CLI
-cli: prebuild
-	# developer convince for platform they are running
-	go build -ldflags "${LDFLAGS}" -o bin/${CLI} cmd/kubectl-kudo/main.go
+cli: prebuild cli-fast
 
 .PHONY: cli-clean
 # Clean CLI build

go.mod

@@ -18,6 +18,7 @@ require (
 	github.com/gobuffalo/envy v1.6.15 // indirect
 	github.com/google/go-github v17.0.0+incompatible
 	github.com/google/go-querystring v1.0.0 // indirect
+	github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf
 	github.com/gophercloud/gophercloud v0.2.0 // indirect
 	github.com/hashicorp/golang-lru v0.5.1 // indirect
 	github.com/huandu/xstrings v1.2.0 // indirect

go.sum

@@ -112,7 +112,6 @@ github.com/gobuffalo/envy v1.6.5/go.mod h1:N+GkhhZ/93bGZc6ZKhJLP6+m+tCNPKwgSpH9k
 github.com/gobuffalo/envy v1.6.15 h1:OsV5vOpHYUpP7ZLS6sem1y40/lNX1BZj+ynMiRi21lQ=
 github.com/gobuffalo/envy v1.6.15/go.mod h1:n7DRkBerg/aorDM8kbduw5dN3oXGswK5liaSCx4T5NI=
 github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
-github.com/gogo/protobuf v1.2.0 h1:xU6/SpYbvkNYiptHJYEDRseDLvYE7wSqhYYNy0QSUzI=
 github.com/gogo/protobuf v1.2.0/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ=
 github.com/gogo/protobuf v1.2.1 h1:/s5zKNz0uPFCZ5hddgPdo2TK2TVrUNMn0OOX8/aZMTE=
 github.com/gogo/protobuf v1.2.1/go.mod h1:hp+jE20tsWTFYpLwKvXlhS1hjn+gTNwPg2I6zVXpSg4=
@@ -144,6 +143,8 @@ github.com/google/gofuzz v1.0.0 h1:A8PeW59pxE9IoFRqBp37U+mSNaQoZ46F1f0f863XSXw=
 github.com/google/gofuzz v1.0.0/go.mod h1:dBl0BpW6vV/+mYPU4Po3pmUjxk6FQPldtuIdl/M65Eg=
 github.com/google/martian v2.1.0+incompatible/go.mod h1:9I4somxYTbIHy5NJKHRl3wXiIaQGbYVAs8BPL6v8lEs=
 github.com/google/pprof v0.0.0-20181206194817-3ea8567a2e57/go.mod h1:zfwlbNMJ+OItoe0UupaVj+oy1omPYYDuagoSzA8v9mc=
+github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf h1:7+FW5aGwISbqUtkfmIpZJGRgNFg2ioYPvFaUxdqpDsg=
+github.com/google/shlex v0.0.0-20181106134648-c34317bd91bf/go.mod h1:RpwtwJQFrIEPstU94h88MWPXP2ektJZ8cZ0YntAmXiE=
 github.com/google/uuid v1.1.1 h1:Gkbcsh/GbpXz7lPftLA3P6TYMwjCLYm83jiFQZF/3gY=
 github.com/google/uuid v1.1.1/go.mod h1:TIyPZe4MgqvfeYDBFedMoGGpEw/LqOeaOT+nhxU+yHo=
 github.com/googleapis/gax-go/v2 v2.0.4/go.mod h1:0Wqv26UfaUD9n4G6kQubkQ+KchISgw+vpHVxEJEs9eg=

pkg/apis/kudo/v1alpha1/test_types.go

@@ -40,6 +40,8 @@ type TestSuite struct {
 	Parallel int `json:"parallel"`
 	// The directory to output artifacts to (current working directory if not specified).
 	ArtifactsDir string `json:"artifactsDir"`
+	// Kubectl commands to run before running any tests.
+	Kubectl []string `json:"kubectl"`
 }
 
 // +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
@@ -58,6 +60,9 @@ type TestStep struct {
 	// Indicates that this is a unit test - safe to run without a real Kubernetes cluster.
 	UnitTest bool `json:"unitTest"`
 
+	// Kubectl commands to run at the start of the test
+	Kubectl []string `json:"kubectl"`
+
 	// Allowed environment labels
 	// Disallowed environment labels
 }

pkg/test/case.go

@@ -74,40 +74,36 @@ func (t *Case) CreateNamespace(namespace string) error {
 	})
 }
 
-// TestCaseFactory creates a new Go test that runs a set of test steps.
-func (t *Case) TestCaseFactory() func(*testing.T) {
-	return func(test *testing.T) {
-		t.Logger = testutils.NewTestLogger(test, t.Name)
+// Run runs a test case including all of its steps.
+func (t *Case) Run(test *testing.T) {
+	test.Parallel()
 
-		test.Parallel()
+	ns := fmt.Sprintf("kudo-test-%s", petname.Generate(2, "-"))
 
-		ns := fmt.Sprintf("kudo-test-%s", petname.Generate(2, "-"))
+	if err := t.CreateNamespace(ns); err != nil {
+		test.Fatal(err)
+	}
 
-		if err := t.CreateNamespace(ns); err != nil {
-			test.Fatal(err)
-		}
+	if !t.SkipDelete {
+		defer t.DeleteNamespace(ns)
+	}
+
+	for _, testStep := range t.Steps {
+		testStep.Client = t.Client
+		testStep.DiscoveryClient = t.DiscoveryClient
+		testStep.Logger = t.Logger.WithPrefix(testStep.String())
 
 		if !t.SkipDelete {
-			defer t.DeleteNamespace(ns)
+			defer testStep.Clean(ns)
 		}
 
-		for _, testStep := range t.Steps {
-			testStep.Client = t.Client
-			testStep.DiscoveryClient = t.DiscoveryClient
-			testStep.Logger = t.Logger.WithPrefix(testStep.String())
-
-			if !t.SkipDelete {
-				defer testStep.Clean(ns)
+		if errs := testStep.Run(ns); len(errs) > 0 {
+			for _, err := range errs {
+				test.Error(err)
 			}
 
-			if errs := testStep.Run(ns); len(errs) > 0 {
-				for _, err := range errs {
-					test.Error(err)
-				}
-
-				test.Error(fmt.Errorf("failed in step %s", testStep.String()))
-				break
-			}
+			test.Error(fmt.Errorf("failed in step %s", testStep.String()))
+			break
 		}
 	}
 }
@@ -125,6 +121,11 @@ func (t *Case) CollectTestStepFiles() (map[int64][]string, error) {
 	for _, file := range files {
 		matches := testStepRegex.FindStringSubmatch(file.Name())
 
+		if len(matches) < 2 {
+			t.Logger.Log("Ignoring", file.Name(), "as it does not match file name regexp:", testStepRegex.String())
+			continue
+		}
+
 		index, err := strconv.ParseInt(matches[1], 10, 32)
 		if err != nil {
 			return nil, err
@@ -168,6 +169,7 @@ func (t *Case) LoadTestSteps() error {
 		testStep := &Step{
 			Timeout: t.Timeout,
 			Index:   int(index),
+			Dir:     t.Dir,
 			Asserts: []runtime.Object{},
 			Apply:   []runtime.Object{},
 			Errors:  []runtime.Object{},

pkg/test/case_test.go

@@ -222,7 +222,7 @@ func TestLoadTestSteps(t *testing.T) {
 		},
 	} {
 		t.Run(tt.path, func(t *testing.T) {
-			test := &Case{Dir: tt.path}
+			test := &Case{Dir: tt.path, Logger: testutils.NewTestLogger(t, tt.path)}
 
 			err := test.LoadTestSteps()
 			assert.Nil(t, err)
@@ -234,10 +234,12 @@ func TestLoadTestSteps(t *testing.T) {
 
 			assert.Equal(t, len(tt.testSteps), len(testStepsVal))
 			for index := range tt.testSteps {
+				tt.testSteps[index].Dir = tt.path
 				assert.Equal(t, tt.testSteps[index].Apply, testStepsVal[index].Apply)
 				assert.Equal(t, tt.testSteps[index].Asserts, testStepsVal[index].Asserts)
 				assert.Equal(t, tt.testSteps[index].Errors, testStepsVal[index].Errors)
 				assert.Equal(t, tt.testSteps[index].Step, testStepsVal[index].Step)
+				assert.Equal(t, tt.testSteps[index].Dir, testStepsVal[index].Dir)
 				assert.Equal(t, tt.testSteps[index], testStepsVal[index])
 			}
 		})
@@ -283,7 +285,7 @@ func TestCollectTestStepFiles(t *testing.T) {
 		},
 	} {
 		t.Run(tt.path, func(t *testing.T) {
-			test := &Case{Dir: tt.path}
+			test := &Case{Dir: tt.path, Logger: testutils.NewTestLogger(t, tt.path)}
 			testStepFiles, err := test.CollectTestStepFiles()
 			assert.Nil(t, err)
 			assert.Equal(t, tt.expected, testStepFiles)

pkg/test/harness.go

@@ -5,6 +5,7 @@ import (
 	"fmt"
 	"io/ioutil"
 	"math/rand"
+	"os"
 	"path/filepath"
 	"sync"
 	"testing"
@@ -30,6 +31,7 @@ type Harness struct {
 	TestSuite kudo.TestSuite
 	T         *testing.T
 
+	logger        testutils.Logger
 	managerStopCh chan struct{}
 	config        *rest.Config
 	client        client.Client
@@ -71,6 +73,15 @@ func (h *Harness) LoadTests(dir string) ([]*Case, error) {
 	return tests, nil
 }
 
+// GetLogger returns an initialized test logger.
+func (h *Harness) GetLogger() testutils.Logger {
+	if h.logger == nil {
+		h.logger = testutils.NewTestLogger(h.T, "")
+	}
+
+	return h.logger
+}
+
 // GetTimeout returns the configured timeout for the test suite.
 func (h *Harness) GetTimeout() int {
 	timeout := 30
@@ -162,7 +173,18 @@ func (h *Harness) Config() (*rest.Config, error) {
 		h.config, err = config.GetConfig()
 	}
 
-	return h.config, err
+	if err != nil {
+		return h.config, err
+	}
+
+	f, err := os.Create("kubeconfig")
+	if err != nil {
+		return h.config, err
+	}
+
+	defer f.Close()
+
+	return h.config, testutils.Kubeconfig(h.config, f)
 }
 
 // RunKUDO starts the KUDO controllers and installs the required CRDs.
@@ -249,11 +271,15 @@ func (h *Harness) RunTests() {
 			test.Client = h.Client
 			test.DiscoveryClient = h.DiscoveryClient
 
-			if err := test.LoadTestSteps(); err != nil {
-				t.Fatal(err)
-			}
+			t.Run(test.Name, func(t *testing.T) {
+				test.Logger = testutils.NewTestLogger(t, test.Name)
 
-			t.Run(test.Name, test.TestCaseFactory())
+				if err := test.LoadTestSteps(); err != nil {
+					t.Fatal(err)
+				}
+
+				test.Run(t)
+			})
 		}
 	})
 }
@@ -299,6 +325,10 @@ func (h *Harness) Run() {
 		}
 	}
 
+	if err := testutils.RunKubectlCommands(h.GetLogger(), "default", h.TestSuite.Kubectl, ""); err != nil {
+		h.T.Fatal(err)
+	}
+
 	if h.TestSuite.StartKUDO {
 		if err := h.RunKUDO(); err != nil {
 			h.T.Fatal(err)
@@ -326,7 +356,12 @@ func (h *Harness) Stop() {
 	}
 
 	if h.TestSuite.SkipClusterDelete || h.TestSuite.SkipDelete {
-		// TODO: figure out how to write the Kubernetes client configuration to disk.
+		cwd, _ := os.Getwd()
+		kubeconfig := filepath.Join(cwd, "kubeconfig")
+
+		h.T.Log("skipping cluster tear down")
+		h.T.Log(fmt.Sprintf("to connect to the cluster, run: export KUBECONFIG=\"%s\"", kubeconfig))
+
 		return
 	}
 

pkg/test/step.go

@@ -26,6 +26,8 @@ type Step struct {
 	Name  string
 	Index int
 
+	Dir string
+
 	Step   *kudo.TestStep
 	Assert *kudo.TestAssert
 
@@ -299,7 +301,15 @@ func (s *Step) Run(namespace string) []error {
 		return []error{err}
 	}
 
-	testErrors := s.Create(namespace)
+	testErrors := []error{}
+
+	if s.Step != nil {
+		if errors := testutils.RunKubectlCommands(s.Logger, namespace, s.Step.Kubectl, s.Dir); errors != nil {
+			testErrors = append(testErrors, errors...)
+		}
+	}
+
+	testErrors = append(testErrors, s.Create(namespace)...)
 
 	if len(testErrors) != 0 {
 		return testErrors

pkg/test/test_data/cli-test/00-assert.yaml

@@ -0,0 +1,4 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cli-test-pod

pkg/test/test_data/cli-test/00-create-pod.yaml

@@ -0,0 +1,4 @@
+apiVersion: kudo.dev/v1alpha1
+kind: TestStep
+kubectl:
+- apply -f ./test_data/pod.yaml

pkg/test/test_data/cli-test/test_data/pod.yaml

@@ -0,0 +1,9 @@
+apiVersion: v1
+kind: Pod
+metadata:
+  name: cli-test-pod
+spec:
+  containers:
+  - name: cli-test
+    image: alpine
+  restartPolicy: Never