Implement object deletion in test steps. (#493)

* Implement object deletion in test steps.

* fix docs table of contents

* fix test comment

* Support deleting objects by label.

* fix context usage inside of retry

docs/testing.md

@@ -16,6 +16,7 @@ KUDO uses a declarative integration testing harness for testing itself and Opera
 * [Writing test cases](#writing-test-cases)
    * [Test case directory structure](#test-case-directory-structure)
    * [Test steps](#test-steps)
+     * [Deleting resources](#deleting-resources)
      * [Test assertions](#test-assertions)
        * [Listing objects](#listing-objects)
        * [Advanced test assertions](#advanced-test-assertions)
@@ -116,6 +117,27 @@ spec:
 
 This test step will create a Zookeeper `Instance`. The namespace should not be specified in the resources as a namespace is created for the test case to run in.
 
+#### Deleting resources
+
+It is possible to delete existing resources at the beginning of a test step. Create a `TestStep` object in your step to configure it:
+
+```
+apiVersion: kudo.k8s.io/v1alpha1
+kind: TestStep
+delete:
+- name: my-pod
+  kind: Pod
+  version: v1
+- kind: Pod
+  version: v1
+  labels:
+    app: nginx
+```
+
+The test harness will delete for each resource referenced in the delete list and wait for them to disappear from the API. If the object fails to delete, the test step will fail.
+
+In the first delete example, the `Pod` called `my-pod` will be deleted. In the second, all `Pods` matching the `app=nginx` label will be deleted.
+
 #### Test assertions
 
 Test assert files contain any number of Kubernetes resources that are expected to be created. Each resource must specify the `apiVersion`, `kind`, and `metadata`. The test harness watches each defined resource and waits for the state defined to match the state in Kubernetes. Once all resources have the correct state simultaneously, the test is considered successful.

keps/0008-operator-testing.md

@@ -229,18 +229,29 @@ type TestStep struct {
     ObjectMeta
 
     // Objects to delete at the beginning of the test step.
-    Delete []corev1.ObjectReference
+    Delete []ObjectReference
 
     // Indicates that this is a unit test - safe to run without a real Kubernetes cluster.
     UnitTest bool
 
     // Allowed environment labels
     // Disallowed environment labels
 }
+
+// ObjectReference is a Kubernetes object reference with added labels to allow referencing
+// objects by label.
+type ObjectReference struct {
+	corev1.ObjectReference `json:",inline"`
+	// Labels to match on.
+	Labels map[string]string
+}
 ```
 
 Using a `TestStep`, it is possible to skip certain test steps if conditions are not met, e.g., only run a test step on GKE or on clusters with more than three nodes.
 
+The `Delete` list can be used to specify objects to delete prior to running the tests. If `Labels` are set in an ObjectReference,
+all resources matching the labels and specified kind will be deleted.
+
 #### Assertion files
 
 The assertion file contains one or more Kubernetes resources to watch the Kubernetes API for. For each object, it checks the API for an object with the same kind, name, and metadata and waits for it to have a state matching what is defined in the assertion files.

pkg/apis/kudo/v1alpha1/test_types.go

@@ -39,7 +39,7 @@ type TestStep struct {
 
 	Index int `json:"index,omitempty"`
 	// Objects to delete at the beginning of the test step.
-	Delete []corev1.ObjectReference `json:"delete,omitempty"`
+	Delete []ObjectReference `json:"delete,omitempty"`
 
 	// Indicates that this is a unit test - safe to run without a real Kubernetes cluster.
 	UnitTest bool `json:"unitTest"`
@@ -57,3 +57,11 @@ type TestAssert struct {
 	// Override the default timeout of 300 seconds (in seconds).
 	Timeout int `json:"timeout"`
 }
+
+// ObjectReference is a Kubernetes object reference with added labels to allow referencing
+// objects by label.
+type ObjectReference struct {
+	corev1.ObjectReference `json:",inline"`
+	// Labels to match on.
+	Labels map[string]string
+}

pkg/test/case_test.go

@@ -62,10 +62,12 @@ func TestLoadTestSteps(t *testing.T) {
 							APIVersion: "kudo.k8s.io/v1alpha1",
 						},
 						Index: 1,
-						Delete: []corev1.ObjectReference{
+						Delete: []kudo.ObjectReference{
 							{
-								Kind: "Pod",
-								Name: "test",
+								ObjectReference: corev1.ObjectReference{
+									Kind: "Pod",
+									Name: "test",
+								},
 							},
 						},
 					},

pkg/test/step.go

@@ -2,17 +2,18 @@ package test
 
 import (
 	"context"
-	"errors"
 	"fmt"
 	"path/filepath"
 	"regexp"
 	"time"
 
 	kudo "github.com/kudobuilder/kudo/pkg/apis/kudo/v1alpha1"
 	testutils "github.com/kudobuilder/kudo/pkg/test/utils"
+	"github.com/pkg/errors"
 	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
+	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/discovery"
 	"sigs.k8s.io/controller-runtime/pkg/client"
 )
@@ -53,6 +54,84 @@ func (s *Step) Clean(namespace string) error {
 	return nil
 }
 
+// DeleteExisting deletes any resources in the TestStep.Delete list prior to running the tests.
+func (s *Step) DeleteExisting(namespace string) error {
+	toDelete := []runtime.Object{}
+
+	if s.Step == nil {
+		return nil
+	}
+
+	for _, ref := range s.Step.Delete {
+		gvk := ref.GroupVersionKind()
+
+		obj := testutils.NewResource(gvk.GroupVersion().String(), gvk.Kind, ref.Name, "")
+
+		objNs := namespace
+		if ref.Namespace != "" {
+			objNs = ref.Namespace
+		}
+
+		_, objNs, err := testutils.Namespaced(s.DiscoveryClient, obj, objNs)
+		if err != nil {
+			return err
+		}
+
+		if ref.Labels != nil && len(ref.Labels) != 0 {
+			// If the reference has a label selector, List all objects that match
+			if err := testutils.Retry(context.TODO(), func(ctx context.Context) error {
+				u := &unstructured.UnstructuredList{}
+				u.SetGroupVersionKind(gvk)
+
+				listOptions := []client.ListOptionFunc{client.MatchingLabels(ref.Labels)}
+				if objNs != "" {
+					listOptions = append(listOptions, client.InNamespace(objNs))
+				}
+
+				err := s.Client.List(ctx, u, listOptions...)
+				if err != nil {
+					return errors.Wrap(err, "listing matching resources")
+				}
+
+				for index := range u.Items {
+					toDelete = append(toDelete, &u.Items[index])
+				}
+
+				return nil
+			}, testutils.IsJSONSyntaxError); err != nil {
+				return err
+			}
+		} else {
+			// Otherwise just append the object specified.
+			toDelete = append(toDelete, obj.DeepCopyObject())
+		}
+	}
+
+	for _, obj := range toDelete {
+		if err := testutils.Retry(context.TODO(), func(ctx context.Context) error {
+			err := s.Client.Delete(context.TODO(), obj.DeepCopyObject())
+			if err != nil && k8serrors.IsNotFound(err) {
+				return nil
+			}
+			return err
+		}, testutils.IsJSONSyntaxError); err != nil {
+			return err
+		}
+	}
+
+	// Wait for resources to be deleted.
+	return wait.PollImmediate(100*time.Millisecond, 10*time.Second, func() (done bool, err error) {
+		for _, obj := range toDelete {
+			err = s.Client.Get(context.TODO(), testutils.ObjectKey(obj), obj.DeepCopyObject())
+			if err == nil || !k8serrors.IsNotFound(err) {
+				return false, err
+			}
+		}
+
+		return true, nil
+	})
+}
+
 // Create applies all resources defined in the Apply list.
 func (s *Step) Create(namespace string) []error {
 	errors := []error{}
@@ -176,6 +255,10 @@ func (s *Step) Check(namespace string) []error {
 func (s *Step) Run(namespace string) []error {
 	s.Logger.Log("starting test step", s.String())
 
+	if err := s.DeleteExisting(namespace); err != nil {
+		return []error{err}
+	}
+
 	testErrors := s.Create(namespace)
 
 	if len(testErrors) != 0 {

pkg/test/step_integration_test.go

@@ -9,8 +9,11 @@ import (
 
 	petname "github.com/dustinkirkland/golang-petname"
 
+	kudo "github.com/kudobuilder/kudo/pkg/apis/kudo/v1alpha1"
 	testutils "github.com/kudobuilder/kudo/pkg/test/utils"
 	"github.com/stretchr/testify/assert"
+	corev1 "k8s.io/api/core/v1"
+	k8serrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/discovery"
@@ -226,3 +229,76 @@ func TestCheckResourceIntegration(t *testing.T) {
 		})
 	}
 }
+
+// Verify that the DeleteExisting method properly cleans up resources that are matched on labels during a test step.
+func TestStepDeleteExistingLabelMatch(t *testing.T) {
+	env := &envtest.Environment{}
+
+	config, err := env.Start()
+	assert.Nil(t, err)
+
+	defer env.Stop()
+
+	cl, err := client.New(config, client.Options{
+		Scheme: testutils.Scheme(),
+	})
+	assert.Nil(t, err)
+	dClient, err := discovery.NewDiscoveryClientForConfig(config)
+	assert.Nil(t, err)
+
+	namespace := "world"
+
+	podSpec := map[string]interface{}{
+		"containers": []interface{}{
+			map[string]interface{}{
+				"image": "otherimage:latest",
+				"name":  "nginx",
+			},
+		},
+	}
+
+	podToDelete := testutils.WithSpec(testutils.WithLabels(testutils.NewPod("aa-delete-me", "world"), map[string]string{
+		"hello": "world",
+	}), podSpec)
+
+	podToKeep := testutils.WithSpec(testutils.WithLabels(testutils.NewPod("bb-dont-delete-me", "world"), map[string]string{
+		"bye": "moon",
+	}), podSpec)
+
+	podToDelete2 := testutils.WithSpec(testutils.WithLabels(testutils.NewPod("cc-delete-me", "world"), map[string]string{
+		"hello": "world",
+	}), podSpec)
+
+	step := Step{
+		Logger: testutils.NewTestLogger(t, ""),
+		Step: &kudo.TestStep{
+			Delete: []kudo.ObjectReference{
+				{
+					ObjectReference: corev1.ObjectReference{
+						Kind:       "Pod",
+						APIVersion: "v1",
+					},
+					Labels: map[string]string{
+						"hello": "world",
+					},
+				},
+			},
+		},
+		Client:          cl,
+		DiscoveryClient: dClient,
+	}
+
+	assert.Nil(t, step.Client.Create(context.TODO(), podToKeep))
+	assert.Nil(t, step.Client.Create(context.TODO(), podToDelete))
+	assert.Nil(t, step.Client.Create(context.TODO(), podToDelete2))
+
+	assert.Nil(t, step.Client.Get(context.TODO(), testutils.ObjectKey(podToKeep), podToKeep))
+	assert.Nil(t, step.Client.Get(context.TODO(), testutils.ObjectKey(podToDelete), podToDelete))
+	assert.Nil(t, step.Client.Get(context.TODO(), testutils.ObjectKey(podToDelete2), podToDelete2))
+
+	assert.Nil(t, step.DeleteExisting(namespace))
+
+	assert.Nil(t, step.Client.Get(context.TODO(), testutils.ObjectKey(podToKeep), podToKeep))
+	assert.True(t, k8serrors.IsNotFound(step.Client.Get(context.TODO(), testutils.ObjectKey(podToDelete), podToDelete)))
+	assert.True(t, k8serrors.IsNotFound(step.Client.Get(context.TODO(), testutils.ObjectKey(podToDelete2), podToDelete2)))
+}