From a7b7672dff03e00329151bdba62aa5778bb0890b Mon Sep 17 00:00:00 2001 From: Luca Comellini Date: Wed, 7 Aug 2024 19:38:17 -0700 Subject: [PATCH 1/4] Print logs and events on test failure Problem: When the tests fail we don't have any outputs from the containers or the k8s events. Solution: Print logs and outputs on failure. --- tests/Makefile | 6 ++-- tests/framework/info.go | 52 ++++++++++++++++++++++++++++++ tests/framework/resourcemanager.go | 17 ++++++++++ tests/suite/system_suite_test.go | 20 ++++++++++-- 4 files changed, 90 insertions(+), 5 deletions(-) create mode 100644 tests/framework/info.go diff --git a/tests/Makefile b/tests/Makefile index 9ee6b307c..e83fc7fac 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -70,17 +70,17 @@ ifeq ($(PLUS_ENABLED),true) endif .PHONY: setup-gcp-and-run-tests -setup-gcp-and-run-tests: create-gke-router create-and-setup-vm run-tests-on-vm ## Create and setup a GKE router and GCP VM for tests and run the functional tests +setup-gcp-and-run-tests: create-and-setup-vm run-tests-on-vm ## Create and setup a GCP VM for tests and run the functional tests .PHONY: setup-gcp-and-run-nfr-tests -setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests +setup-gcp-and-run-nfr-tests: create-and-setup-vm nfr-test ## Create and setup a GCP VM for tests and run the NFR tests .PHONY: create-gke-cluster create-gke-cluster: ## Create a GKE cluster ./scripts/create-gke-cluster.sh $(CI) .PHONY: create-and-setup-vm -create-and-setup-vm: ## Create and setup a GCP VM for tests +create-and-setup-vm: create-gke-router ## Create and setup a GKE router and GCP VM for tests ./scripts/create-and-setup-gcp-vm.sh .PHONY: create-gke-router diff --git a/tests/framework/info.go b/tests/framework/info.go new file mode 100644 index 000000000..f258ba595 --- /dev/null +++ b/tests/framework/info.go @@ -0,0 +1,52 @@ +package framework + +import ( + "fmt" + + core "k8s.io/api/core/v1" + "sigs.k8s.io/controller-runtime/pkg/client" +) + +func GetLogs(rm ResourceManager, namespace string, releaseName string) string { + var returnLogs string + pods, err := rm.GetPods(namespace, client.MatchingLabels{ + "app.kubernetes.io/instance": releaseName, + }) + if err != nil { + return fmt.Sprintf("failed to get pods: %v", err) + } + for _, pod := range pods { + for _, container := range pod.Spec.Containers { + returnLogs += fmt.Sprintf("Logs for container %s:\n", container.Name) + logs, err := rm.GetPodLogs(pod.Namespace, pod.Name, &core.PodLogOptions{ + Container: container.Name, + }) + if err != nil { + returnLogs += fmt.Sprintf(" failed to get logs: %v\n", err) + continue + } + returnLogs += fmt.Sprintf(" %s\n", logs) + } + } + return returnLogs +} + +func GetEvents(rm ResourceManager, namespace string) string { + var returnEvents string + events, err := rm.GetEvents(namespace) + if err != nil { + return fmt.Sprintf("failed to get events: %v", err) + } + eventGroups := make(map[string][]core.Event) + for _, event := range events.Items { + eventGroups[event.InvolvedObject.Name] = append(eventGroups[event.InvolvedObject.Name], event) + } + for name, events := range eventGroups { + returnEvents += fmt.Sprintf("Events for %s:\n", name) + for _, event := range events { + returnEvents += fmt.Sprintf(" %s\n", event.Message) + } + returnEvents += "\n" + } + return returnEvents +} diff --git a/tests/framework/resourcemanager.go b/tests/framework/resourcemanager.go index cf9e6d3fc..e7a585cf7 100644 --- a/tests/framework/resourcemanager.go +++ b/tests/framework/resourcemanager.go @@ -622,6 +622,23 @@ func (rm *ResourceManager) GetNGFDeployment(namespace, releaseName string) (*app return &deployment, nil } +// GetEvents returns all Events in the specified namespace. +func (rm *ResourceManager) GetEvents(namespace string) (*core.EventList, error) { + ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.GetTimeout) + defer cancel() + + var eventList core.EventList + if err := rm.K8sClient.List( + ctx, + &eventList, + client.InNamespace(namespace), + ); err != nil { + return &core.EventList{}, fmt.Errorf("error getting list of Events: %w", err) + } + + return &eventList, nil +} + // ScaleDeployment scales the Deployment to the specified number of replicas. func (rm *ResourceManager) ScaleDeployment(namespace, name string, replicas int32) error { ctx, cancel := context.WithTimeout(context.Background(), rm.TimeoutConfig.UpdateTimeout) diff --git a/tests/suite/system_suite_test.go b/tests/suite/system_suite_test.go index f6913935a..2ab0c4d45 100644 --- a/tests/suite/system_suite_test.go +++ b/tests/suite/system_suite_test.go @@ -21,7 +21,7 @@ import ( apiext "k8s.io/apiextensions-apiserver/pkg/apis/apiextensions/v1" apierrors "k8s.io/apimachinery/pkg/api/errors" k8sRuntime "k8s.io/apimachinery/pkg/runtime" - "k8s.io/apimachinery/pkg/types" + k8sTypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/wait" "k8s.io/client-go/kubernetes" ctlr "sigs.k8s.io/controller-runtime" @@ -76,6 +76,7 @@ var ( chartVersion string clusterInfo framework.ClusterInfo skipNFRTests bool + logs string ) const ( @@ -235,7 +236,7 @@ func teardown(relName string) { 500*time.Millisecond, true, /* poll immediately */ func(ctx context.Context) (bool, error) { - key := types.NamespacedName{Name: ngfNamespace} + key := k8sTypes.NamespacedName{Name: ngfNamespace} if err := k8sClient.Get(ctx, key, &core.Namespace{}); err != nil && apierrors.IsNotFound(err) { return true, nil } @@ -292,6 +293,11 @@ var _ = AfterSuite(func() { if skipNFRTests { Skip("") } + events := framework.GetEvents(resourceManager, ngfNamespace) + AddReportEntry("Events", events, ReportEntryVisibilityNever) + + logs = framework.GetLogs(resourceManager, ngfNamespace, releaseName) + AddReportEntry("Logs", logs, ReportEntryVisibilityNever) labelFilter := GinkgoLabelFilter() if !strings.Contains(labelFilter, "longevity-setup") { @@ -311,3 +317,13 @@ func isNFR(labelFilter string) bool { strings.Contains(labelFilter, "upgrade") || strings.Contains(labelFilter, "scale") } + +var _ = ReportAfterSuite("Print info on failure", func(report Report) { + if !report.SuiteSucceeded { + for _, specReport := range report.SpecReports { + for _, entry := range specReport.ReportEntries { + fmt.Println(entry.GetRawValue()) + } + } + } +}) From a345fc0e2d0ccf0bbaf4a8e5444486dec7006eb4 Mon Sep 17 00:00:00 2001 From: Luca Comellini Date: Wed, 7 Aug 2024 19:55:21 -0700 Subject: [PATCH 2/4] remove router --- tests/Makefile | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index e83fc7fac..9ee6b307c 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -70,17 +70,17 @@ ifeq ($(PLUS_ENABLED),true) endif .PHONY: setup-gcp-and-run-tests -setup-gcp-and-run-tests: create-and-setup-vm run-tests-on-vm ## Create and setup a GCP VM for tests and run the functional tests +setup-gcp-and-run-tests: create-gke-router create-and-setup-vm run-tests-on-vm ## Create and setup a GKE router and GCP VM for tests and run the functional tests .PHONY: setup-gcp-and-run-nfr-tests -setup-gcp-and-run-nfr-tests: create-and-setup-vm nfr-test ## Create and setup a GCP VM for tests and run the NFR tests +setup-gcp-and-run-nfr-tests: create-gke-router create-and-setup-vm nfr-test ## Create and setup a GKE router and GCP VM for tests and run the NFR tests .PHONY: create-gke-cluster create-gke-cluster: ## Create a GKE cluster ./scripts/create-gke-cluster.sh $(CI) .PHONY: create-and-setup-vm -create-and-setup-vm: create-gke-router ## Create and setup a GKE router and GCP VM for tests +create-and-setup-vm: ## Create and setup a GCP VM for tests ./scripts/create-and-setup-gcp-vm.sh .PHONY: create-gke-router From 2638fff4e64709c1fa280a01746ecda19d3ff512 Mon Sep 17 00:00:00 2001 From: Luca Comellini Date: Wed, 7 Aug 2024 21:41:23 -0700 Subject: [PATCH 3/4] output --- tests/Makefile | 4 ++-- tests/scripts/cleanup-router.sh | 2 +- tests/scripts/cleanup-vm.sh | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/tests/Makefile b/tests/Makefile index 9ee6b307c..e9cc59004 100644 --- a/tests/Makefile +++ b/tests/Makefile @@ -109,7 +109,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results .PHONY: .vm-nfr-test .vm-nfr-test: ## Runs the NFR tests on the GCP VM (called by `nfr-test`) - go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v \ + go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --force-newlines $(ifeq $(CI),true,--github-output) \ --label-filter "nfr" $(GINKGO_FLAGS) --timeout 3h ./suite -- --gateway-api-version=$(GW_API_VERSION) \ --gateway-api-prev-version=$(GW_API_PREV_VERSION) --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \ --plus-enabled=$(PLUS_ENABLED) --ngf-image-repo=$(PREFIX) --nginx-image-repo=$(NGINX_PREFIX) --nginx-plus-image-repo=$(NGINX_PLUS_PREFIX) \ @@ -118,7 +118,7 @@ stop-longevity-test: nfr-test ## Stop the longevity test and collects results .PHONY: test test: ## Runs the functional tests on your default k8s cluster - go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v \ + go run github.com/onsi/ginkgo/v2/ginkgo --randomize-all --randomize-suites --keep-going --fail-on-pending --trace -r -v --force-newlines \ --label-filter "functional" $(GINKGO_FLAGS) ./suite -- \ --gateway-api-version=$(GW_API_VERSION) --gateway-api-prev-version=$(GW_API_PREV_VERSION) \ --image-tag=$(TAG) --version-under-test=$(NGF_VERSION) \ diff --git a/tests/scripts/cleanup-router.sh b/tests/scripts/cleanup-router.sh index fe7c215a6..ee3ea524b 100755 --- a/tests/scripts/cleanup-router.sh +++ b/tests/scripts/cleanup-router.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -eo pipefail +set -o pipefail source scripts/vars.env diff --git a/tests/scripts/cleanup-vm.sh b/tests/scripts/cleanup-vm.sh index b98589e46..ecb0420d1 100755 --- a/tests/scripts/cleanup-vm.sh +++ b/tests/scripts/cleanup-vm.sh @@ -1,6 +1,6 @@ #!/usr/bin/env bash -set -eo pipefail +set -o pipefail source scripts/vars.env From c2a8322c3b10b1b2568316874504e0b8a3624441 Mon Sep 17 00:00:00 2001 From: Luca Comellini Date: Thu, 8 Aug 2024 09:26:29 -0700 Subject: [PATCH 4/4] Apply suggestions from code review Co-authored-by: Saylor Berman --- tests/framework/info.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/tests/framework/info.go b/tests/framework/info.go index f258ba595..e83a1be18 100644 --- a/tests/framework/info.go +++ b/tests/framework/info.go @@ -15,6 +15,7 @@ func GetLogs(rm ResourceManager, namespace string, releaseName string) string { if err != nil { return fmt.Sprintf("failed to get pods: %v", err) } + for _, pod := range pods { for _, container := range pod.Spec.Containers { returnLogs += fmt.Sprintf("Logs for container %s:\n", container.Name) @@ -37,10 +38,12 @@ func GetEvents(rm ResourceManager, namespace string) string { if err != nil { return fmt.Sprintf("failed to get events: %v", err) } + eventGroups := make(map[string][]core.Event) for _, event := range events.Items { eventGroups[event.InvolvedObject.Name] = append(eventGroups[event.InvolvedObject.Name], event) } + for name, events := range eventGroups { returnEvents += fmt.Sprintf("Events for %s:\n", name) for _, event := range events {