Print logs and events on test failure (#2347)

Problem: When the tests fail we don't have any outputs from the
containers or the k8s events.

Solution: Print logs and outputs on failure.

tests/Makefile

@@ -109,7 +109,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 
 .PHONY: .vm-nfr-test
 .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`)
-	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v \
+	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --force-newlines $(ifeq $(CI),true,--github-output) \
 	    --label-filter "nfr" $(GINKGO_FLAGS) --timeout 3h ./suite --  --gateway-api-version=$(GW_API_VERSION) \
 		--gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \
 		--plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \
@@ -118,7 +118,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results
 
 .PHONY: test
 test: ## Runs the functional tests on your default k8s cluster
-	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v \
+	go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --force-newlines \
 		--label-filter "functional" $(GINKGO_FLAGS) ./suite -- \
 		--gateway-api-version=$(GW_API_VERSION) --gateway-api-prev-version=$(GW_API_PREV_VERSION) \
 		--image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \

tests/framework/info.go

@@ -0,0 +1,55 @@
+package framework
+
+import (
+	"fmt"
+
+	core "k8s.io/api/core/v1"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+)
+
+func GetLogs(rm ResourceManager, namespace string, releaseName string) string {
+	var returnLogs string
+	pods, err := rm.GetPods(namespace, client.MatchingLabels{
+		"app.kubernetes.io/instance": releaseName,
+	})
+	if err != nil {
+		return fmt.Sprintf("failed to get pods: %v", err)
+	}
+
+	for _, pod := range pods {
+		for _, container := range pod.Spec.Containers {
+			returnLogs += fmt.Sprintf("Logs for container %s:\n", container.Name)
+			logs, err := rm.GetPodLogs(pod.Namespace, pod.Name, &core.PodLogOptions{
+				Container: container.Name,
+			})
+			if err != nil {
+				returnLogs += fmt.Sprintf("  failed to get logs: %v\n", err)
+				continue
+			}
+			returnLogs += fmt.Sprintf("  %s\n", logs)
+		}
+	}
+	return returnLogs
+}
+
+func GetEvents(rm ResourceManager, namespace string) string {
+	var returnEvents string
+	events, err := rm.GetEvents(namespace)
+	if err != nil {
+		return fmt.Sprintf("failed to get events: %v", err)
+	}
+
+	eventGroups := make(map[string][]core.Event)
+	for _, event := range events.Items {
+		eventGroups[event.InvolvedObject.Name] = append(eventGroups[event.InvolvedObject.Name], event)
+	}
+
+	for name, events := range eventGroups {
+		returnEvents += fmt.Sprintf("Events for %s:\n", name)
+		for _, event := range events {
+			returnEvents += fmt.Sprintf("  %s\n", event.Message)
+		}
+		returnEvents += "\n"
+	}
+	return returnEvents
+}

tests/framework/resourcemanager.go

@@ -622,6 +622,23 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app
 	return &deployment, nil
 }
 
+// GetEvents returns all Events in the specified namespace.
+func (rm *ResourceManager) GetEvents(namespace string) (*core.EventList, error) {
+	ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout)
+	defer cancel()
+
+	var eventList core.EventList
+	if err := rm.K8sClient.List(
+		ctx,
+		&eventList,
+		client.InNamespace(namespace),
+	); err != nil {
+		return &core.EventList{}, fmt.Errorf("error getting list of Events: %w", err)
+	}
+
+	return &eventList, nil
+}
+
 // ScaleDeployment scales the Deployment to the specified number of replicas.
 func (rm *ResourceManager) ScaleDeployment(namespace, name string, replicas int32) error {
 	ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.UpdateTimeout)

tests/scripts/cleanup-router.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -eo pipefail
+set -o pipefail
 
 source scripts/vars.env
 

tests/scripts/cleanup-vm.sh

@@ -1,6 +1,6 @@
 #!/usr/bin/env bash
 
-set -eo pipefail
+set -o pipefail
 
 source scripts/vars.env
 

tests/suite/system_suite_test.go

@@ -21,7 +21,7 @@ import (
 	apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	k8sRuntime "k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/apimachinery/pkg/types"
+	k8sTypes "k8s.io/apimachinery/pkg/types"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/kubernetes"
 	ctlr "sigs.k8s.io/controller-runtime"
@@ -76,6 +76,7 @@ var (
 	chartVersion      string
 	clusterInfo       framework.ClusterInfo
 	skipNFRTests      bool
+	logs              string
 )
 
 const (
@@ -235,7 +236,7 @@ func teardown(relName string) {
 		500*time.Millisecond,
 		true, /* poll immediately */
 		func(ctx context.Context) (bool, error) {
-			key := types.NamespacedName{Name: ngfNamespace}
+			key := k8sTypes.NamespacedName{Name: ngfNamespace}
 			if err := k8sClient.Get(ctx, key, &core.Namespace{}); err != nil && apierrors.IsNotFound(err) {
 				return true, nil
 			}
@@ -292,6 +293,11 @@ var _ = AfterSuite(func() {
 	if skipNFRTests {
 		Skip("")
 	}
+	events := framework.GetEvents(resourceManager, ngfNamespace)
+	AddReportEntry("Events", events, ReportEntryVisibilityNever)
+
+	logs = framework.GetLogs(resourceManager, ngfNamespace, releaseName)
+	AddReportEntry("Logs", logs, ReportEntryVisibilityNever)
 
 	labelFilter := GinkgoLabelFilter()
 	if !strings.Contains(labelFilter, "longevity-setup") {
@@ -311,3 +317,13 @@ func isNFR(labelFilter string) bool {
 		strings.Contains(labelFilter, "upgrade") ||
 		strings.Contains(labelFilter, "scale")
 }
+
+var _ = ReportAfterSuite("Print info on failure", func(report Report) {
+	if !report.SuiteSucceeded {
+		for _, specReport := range report.SpecReports {
+			for _, entry := range specReport.ReportEntries {
+				fmt.Println(entry.GetRawValue())
+			}
+		}
+	}
+})