fix(e2e): Refactor e2e utils and resource actions (#832)

* fix(e2e): Refactor e2e utils and resource actions

 brings commonly re-used utilities into lib/test so they can be resused
 and e2e can be run selectively for eventing/serving.

* Refactor service and revision e2e test utils into test lib

* Docstrings for exported e2e test utils

lib/test/revision.go

@@ -0,0 +1,141 @@
+// Copyright 2020 The Knative Authors
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"fmt"
+	"strconv"
+	"strings"
+
+	"gotest.tools/assert"
+	"knative.dev/client/pkg/util"
+)
+
+// RevisionListForService list revisions of given service and verifies if their status is True
+func RevisionListForService(r *KnRunResultCollector, serviceName string) {
+	out := r.KnTest().Kn().Run("revision", "list", "-s", serviceName)
+	r.AssertNoError(out)
+	outputLines := strings.Split(out.Stdout, "\n")
+	// Ignore the last line because it is an empty string caused by splitting a line break
+	// at the end of the output string
+	for _, line := range outputLines[1 : len(outputLines)-1] {
+		// The last item is the revision status, which should be ready
+		assert.Check(r.T(), util.ContainsAll(line, " "+serviceName+" ", "True"))
+	}
+}
+
+// RevisionDescribe verifies revision describe output for given service's revision
+func RevisionDescribe(r *KnRunResultCollector, serviceName string) {
+	revName := FindRevision(r, serviceName)
+
+	out := r.KnTest().Kn().Run("revision", "describe", revName)
+	r.AssertNoError(out)
+	assert.Check(r.T(), util.ContainsAll(out.Stdout, revName, r.KnTest().Kn().Namespace(), serviceName, "++ Ready", "TARGET=kn"))
+}
+
+// RevisionDelete verifies deleting given revision in sync mode
+func RevisionDelete(r *KnRunResultCollector, revName string) {
+	out := r.KnTest().Kn().Run("revision", "delete", "--wait", revName)
+	assert.Check(r.T(), util.ContainsAll(out.Stdout, "Revision", revName, "deleted", "namespace", r.KnTest().Kn().Namespace()))
+	r.AssertNoError(out)
+}
+
+// RevisionMultipleDelete verifies deleting multiple revisions
+func RevisionMultipleDelete(r *KnRunResultCollector, existRevision1, existRevision2, nonexistRevision string) {
+	out := r.KnTest().Kn().Run("revision", "list")
+	r.AssertNoError(out)
+	assert.Check(r.T(), strings.Contains(out.Stdout, existRevision1), "Required revision1 does not exist")
+	assert.Check(r.T(), strings.Contains(out.Stdout, existRevision2), "Required revision2 does not exist")
+
+	out = r.KnTest().Kn().Run("revision", "delete", existRevision1, existRevision2, nonexistRevision)
+	r.AssertNoError(out)
+
+	assert.Check(r.T(), util.ContainsAll(out.Stdout, "Revision", existRevision1, "deleted", "namespace", r.KnTest().Kn().Namespace()), "Failed to get 'deleted' first revision message")
+	assert.Check(r.T(), util.ContainsAll(out.Stdout, "Revision", existRevision2, "deleted", "namespace", r.KnTest().Kn().Namespace()), "Failed to get 'deleted' second revision message")
+	assert.Check(r.T(), util.ContainsAll(out.Stdout, "revisions.serving.knative.dev", nonexistRevision, "not found"), "Failed to get 'not found' error")
+}
+
+// RevisionDescribeWithPrintFlags verifies describing given revision using print flag '--output=name'
+func RevisionDescribeWithPrintFlags(r *KnRunResultCollector, revName string) {
+	out := r.KnTest().Kn().Run("revision", "describe", revName, "-o=name")
+	r.AssertNoError(out)
+	expectedName := fmt.Sprintf("revision.serving.knative.dev/%s", revName)
+	assert.Equal(r.T(), strings.TrimSpace(out.Stdout), expectedName)
+}
+
+// FindRevision returns a revision name (at index 0) for given service
+func FindRevision(r *KnRunResultCollector, serviceName string) string {
+	out := r.KnTest().Kn().Run("revision", "list", "-s", serviceName, "-o=jsonpath={.items[0].metadata.name}")
+	r.AssertNoError(out)
+	if strings.Contains(out.Stdout, "No resources") {
+		r.T().Errorf("Could not find revision name.")
+	}
+	return out.Stdout
+}
+
+// FindRevisionByGeneration returns a revision name for given revision at given generation number
+func FindRevisionByGeneration(r *KnRunResultCollector, serviceName string, generation int) string {
+	maxGen := FindConfigurationGeneration(r, serviceName)
+	out := r.KnTest().Kn().Run("revision", "list", "-s", serviceName,
+		fmt.Sprintf("-o=jsonpath={.items[%d].metadata.name}", maxGen-generation))
+	r.AssertNoError(out)
+	if strings.Contains(out.Stdout, "No resources found.") {
+		r.T().Errorf("Could not find revision name.")
+	}
+	return out.Stdout
+}
+
+// FindConfigurationGeneration returns the configuration generation number of given service
+func FindConfigurationGeneration(r *KnRunResultCollector, serviceName string) int {
+	out := r.KnTest().Kn().Run("revision", "list", "-s", serviceName, "-o=jsonpath={.items[0].metadata.labels.serving\\.knative\\.dev/configurationGeneration}")
+	r.AssertNoError(out)
+	if out.Stdout == "" {
+		r.T().Errorf("Could not find configuration generation.")
+	}
+	confGen, err := strconv.Atoi(out.Stdout)
+	if err != nil {
+		r.T().Errorf("Invalid type of configuration generation: %s", err)
+	}
+
+	return confGen
+}
+
+// RevisionListOutputName verifies listing given revision using print flag '--output name'
+func RevisionListOutputName(r *KnRunResultCollector, revisionName string) {
+	out := r.KnTest().Kn().Run("revision", "list", "--output", "name")
+	r.AssertNoError(out)
+	assert.Check(r.T(), util.ContainsAll(out.Stdout, revisionName, "revision.serving.knative.dev"))
+}
+
+// RevisionListWithService verifies listing revisions per service from each given service names
+func RevisionListWithService(r *KnRunResultCollector, serviceNames ...string) {
+	for _, svcName := range serviceNames {
+		confGen := FindConfigurationGeneration(r, svcName)
+		out := r.KnTest().Kn().Run("revision", "list", "-s", svcName)
+		r.AssertNoError(out)
+
+		outputLines := strings.Split(out.Stdout, "\n")
+		// Ignore the last line because it is an empty string caused by splitting a line break
+		// at the end of the output string
+		for _, line := range outputLines[1 : len(outputLines)-1] {
+			revName := FindRevisionByGeneration(r, svcName, confGen)
+			assert.Check(r.T(), util.ContainsAll(line, revName, svcName, strconv.Itoa(confGen)))
+			confGen--
+		}
+		if r.T().Failed() {
+			r.AddDump("service", svcName, r.KnTest().Kn().Namespace())
+		}
+	}
+}

lib/test/service.go

@@ -0,0 +1,80 @@
+// Copyright 2020 The Knative Authors
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import (
+	"gotest.tools/assert"
+	"knative.dev/client/pkg/util"
+)
+
+// ServiceCreate verifies given service creation in sync mode and also verifies output
+func ServiceCreate(r *KnRunResultCollector, serviceName string) {
+	out := r.KnTest().Kn().Run("service", "create", serviceName, "--image", KnDefaultTestImage)
+	r.AssertNoError(out)
+	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "service", serviceName, "creating", "namespace", r.KnTest().Kn().Namespace(), "ready"))
+}
+
+// ServiceListEmpty verifies that there are no services present
+func ServiceListEmpty(r *KnRunResultCollector) {
+	out := r.KnTest().Kn().Run("service", "list")
+	r.AssertNoError(out)
+	assert.Check(r.T(), util.ContainsAll(out.Stdout, "No services found."))
+}
+
+// ServiceList verifies if given service exists
+func ServiceList(r *KnRunResultCollector, serviceName string) {
+	out := r.KnTest().Kn().Run("service", "list", serviceName)
+	r.AssertNoError(out)
+	assert.Check(r.T(), util.ContainsAll(out.Stdout, serviceName))
+}
+
+// ServiceDescribe describes given service and verifies the keys in the output
+func ServiceDescribe(r *KnRunResultCollector, serviceName string) {
+	out := r.KnTest().Kn().Run("service", "describe", serviceName)
+	r.AssertNoError(out)
+	assert.Assert(r.T(), util.ContainsAll(out.Stdout, serviceName, r.KnTest().Kn().Namespace(), KnDefaultTestImage))
+	assert.Assert(r.T(), util.ContainsAll(out.Stdout, "Conditions", "ConfigurationsReady", "Ready", "RoutesReady"))
+	assert.Assert(r.T(), util.ContainsAll(out.Stdout, "Name", "Namespace", "URL", "Age", "Revisions"))
+}
+
+// ServiceListOutput verifies listing given service using '--output name' flag
+func ServiceListOutput(r *KnRunResultCollector, serviceName string) {
+	out := r.KnTest().Kn().Run("service", "list", serviceName, "--output", "name")
+	r.AssertNoError(out)
+	assert.Check(r.T(), util.ContainsAll(out.Stdout, serviceName, "service.serving.knative.dev"))
+}
+
+// ServiceUpdate verifies service update operation with given arguments in sync mode
+func ServiceUpdate(r *KnRunResultCollector, serviceName string, args ...string) {
+	fullArgs := append([]string{}, "service", "update", serviceName)
+	fullArgs = append(fullArgs, args...)
+	out := r.KnTest().Kn().Run(fullArgs...)
+	r.AssertNoError(out)
+	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "updating", "service", serviceName, "ready"))
+}
+
+// ServiceDelete verifies service deletion in sync mode
+func ServiceDelete(r *KnRunResultCollector, serviceName string) {
+	out := r.KnTest().Kn().Run("service", "delete", "--wait", serviceName)
+	r.AssertNoError(out)
+	assert.Check(r.T(), util.ContainsAll(out.Stdout, "Service", serviceName, "successfully deleted in namespace", r.KnTest().Kn().Namespace()))
+}
+
+// ServiceDescribeWithJSONPath returns output of given JSON path by describing the service
+func ServiceDescribeWithJSONPath(r *KnRunResultCollector, serviceName, jsonpath string) string {
+	out := r.KnTest().Kn().Run("service", "describe", serviceName, "-o", jsonpath)
+	r.AssertNoError(out)
+	return out.Stdout
+}

lib/test/utils.go

@@ -0,0 +1,27 @@
+// Copyright 2020 The Knative Authors
+
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+
+//     http://www.apache.org/licenses/LICENSE-2.0
+
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package test
+
+import "testing"
+
+// GetResourceFieldsWithJSONPath returns output of given JSON path for given resource using kubectl and error if any
+func GetResourceFieldsWithJSONPath(t *testing.T, it *KnTest, resource, name, jsonpath string) (string, error) {
+	out, err := NewKubectl(it.Kn().Namespace()).Run("get", resource, name, "-o", jsonpath, "-n", it.Kn().Namespace())
+	if err != nil {
+		return "", err
+	}
+
+	return out, nil
+}

test/e2e/basic_workflow_test.go

@@ -18,7 +18,6 @@
 package e2e
 
 import (
-	"strings"
 	"testing"
 
 	"gotest.tools/assert"
@@ -39,37 +38,37 @@ func TestBasicWorkflow(t *testing.T) {
 	defer r.DumpIfFailed()
 
 	t.Log("returns no service before running tests")
-	serviceListEmpty(r)
+	test.ServiceListEmpty(r)
 
 	t.Log("create hello service and return no error")
-	serviceCreate(r, "hello")
+	test.ServiceCreate(r, "hello")
 
 	t.Log("return valid info about hello service")
-	serviceList(r, "hello")
-	serviceDescribe(r, "hello")
+	test.ServiceList(r, "hello")
+	test.ServiceDescribe(r, "hello")
 
 	t.Log("return list --output name about hello service")
-	serviceListOutput(r, "hello")
+	test.ServiceListOutput(r, "hello")
 
 	t.Log("update hello service's configuration and return no error")
-	serviceUpdate(r, "hello", "--env", "TARGET=kn", "--port", "8888")
+	test.ServiceUpdate(r, "hello", "--env", "TARGET=kn", "--port", "8888")
 
 	t.Log("create another service and return no error")
-	serviceCreate(r, "svc2")
+	test.ServiceCreate(r, "svc2")
 
 	t.Log("return a list of revisions associated with hello and svc2 services")
-	revisionListForService(r, "hello")
-	revisionListForService(r, "svc2")
+	test.RevisionListForService(r, "hello")
+	test.RevisionListForService(r, "svc2")
 
 	t.Log("describe revision from hello service")
-	revisionDescribe(r, "hello")
+	test.RevisionDescribe(r, "hello")
 
 	t.Log("delete hello and svc2 services and return no error")
-	serviceDelete(r, "hello")
-	serviceDelete(r, "svc2")
+	test.ServiceDelete(r, "hello")
+	test.ServiceDelete(r, "svc2")
 
 	t.Log("return no service after completing tests")
-	serviceListEmpty(r)
+	test.ServiceListEmpty(r)
 }
 
 func TestWrongCommand(t *testing.T) {
@@ -87,71 +86,3 @@ func TestWrongCommand(t *testing.T) {
 	r.AssertError(out)
 
 }
-
-// ==========================================================================
-
-func serviceListEmpty(r *test.KnRunResultCollector) {
-	out := r.KnTest().Kn().Run("service", "list")
-	r.AssertNoError(out)
-	assert.Check(r.T(), util.ContainsAll(out.Stdout, "No services found."))
-}
-
-func serviceCreate(r *test.KnRunResultCollector, serviceName string) {
-	out := r.KnTest().Kn().Run("service", "create", serviceName, "--image", test.KnDefaultTestImage)
-	r.AssertNoError(out)
-	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "service", serviceName, "creating", "namespace", r.KnTest().Kn().Namespace(), "ready"))
-}
-
-func serviceList(r *test.KnRunResultCollector, serviceName string) {
-	out := r.KnTest().Kn().Run("service", "list", serviceName)
-	r.AssertNoError(out)
-	assert.Check(r.T(), util.ContainsAll(out.Stdout, serviceName))
-}
-
-func serviceDescribe(r *test.KnRunResultCollector, serviceName string) {
-	out := r.KnTest().Kn().Run("service", "describe", serviceName)
-	r.AssertNoError(out)
-	assert.Assert(r.T(), util.ContainsAll(out.Stdout, serviceName, r.KnTest().Kn().Namespace(), test.KnDefaultTestImage))
-	assert.Assert(r.T(), util.ContainsAll(out.Stdout, "Conditions", "ConfigurationsReady", "Ready", "RoutesReady"))
-	assert.Assert(r.T(), util.ContainsAll(out.Stdout, "Name", "Namespace", "URL", "Age", "Revisions"))
-}
-
-func serviceListOutput(r *test.KnRunResultCollector, serviceName string) {
-	out := r.KnTest().Kn().Run("service", "list", serviceName, "--output", "name")
-	r.AssertNoError(out)
-	assert.Check(r.T(), util.ContainsAll(out.Stdout, serviceName, "service.serving.knative.dev"))
-}
-
-func serviceUpdate(r *test.KnRunResultCollector, serviceName string, args ...string) {
-	fullArgs := append([]string{}, "service", "update", serviceName)
-	fullArgs = append(fullArgs, args...)
-	out := r.KnTest().Kn().Run(fullArgs...)
-	r.AssertNoError(out)
-	assert.Check(r.T(), util.ContainsAllIgnoreCase(out.Stdout, "updating", "service", serviceName, "ready"))
-}
-
-func serviceDelete(r *test.KnRunResultCollector, serviceName string) {
-	out := r.KnTest().Kn().Run("service", "delete", "--wait", serviceName)
-	r.AssertNoError(out)
-	assert.Check(r.T(), util.ContainsAll(out.Stdout, "Service", serviceName, "successfully deleted in namespace", r.KnTest().Kn().Namespace()))
-}
-
-func revisionListForService(r *test.KnRunResultCollector, serviceName string) {
-	out := r.KnTest().Kn().Run("revision", "list", "-s", serviceName)
-	r.AssertNoError(out)
-	outputLines := strings.Split(out.Stdout, "\n")
-	// Ignore the last line because it is an empty string caused by splitting a line break
-	// at the end of the output string
-	for _, line := range outputLines[1 : len(outputLines)-1] {
-		// The last item is the revision status, which should be ready
-		assert.Check(r.T(), util.ContainsAll(line, " "+serviceName+" ", "True"))
-	}
-}
-
-func revisionDescribe(r *test.KnRunResultCollector, serviceName string) {
-	revName := findRevision(r, serviceName)
-
-	out := r.KnTest().Kn().Run("revision", "describe", revName)
-	r.AssertNoError(out)
-	assert.Check(r.T(), util.ContainsAll(out.Stdout, revName, r.KnTest().Kn().Namespace(), serviceName, "++ Ready", "TARGET=kn"))
-}