From 0309d9d309d07afae5b4c154a53e7feb86c0490d Mon Sep 17 00:00:00 2001 From: Huy Mai Date: Tue, 13 Feb 2024 09:09:22 +0000 Subject: [PATCH] Add ironic and bmo cleanup for upgrade BMO E2E tests Signed-off-by: Huy Mai --- test/e2e/common.go | 40 +++++++++++++++++++++++++++++++++----- test/e2e/e2e_suite_test.go | 4 ++-- test/e2e/upgrade_test.go | 40 +++++++++++++++++++++++++------------- 3 files changed, 63 insertions(+), 21 deletions(-) diff --git a/test/e2e/common.go b/test/e2e/common.go index af84df27b6..023f254ccf 100644 --- a/test/e2e/common.go +++ b/test/e2e/common.go @@ -1,6 +1,7 @@ package e2e import ( + "bytes" "context" "fmt" "os" @@ -21,6 +22,7 @@ import ( metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" metal3api "github.com/metal3-io/baremetal-operator/apis/metal3.io/v1alpha1" + testexec "sigs.k8s.io/cluster-api/test/framework/exec" capm3_e2e "github.com/metal3-io/cluster-api-provider-metal3/test/e2e" @@ -409,10 +411,10 @@ func PerformSSHBootCheck(e2eConfig *Config, expectedBootMode string, auth ssh.Au Expect(isExpectedBootMode).To(BeTrue(), fmt.Sprintf("Expected booting from %s, but found different mode", expectedBootMode)) } -// BuildAndApplyKustomizeInput provides input for BuildAndApplyKustomize(). +// BuildAndApplyKustomizationInput provides input for BuildAndApplyKustomize(). // If WaitForDeployment and/or WatchDeploymentLogs is set to true, then DeploymentName // and DeploymentNamespace are expected. -type BuildAndApplyKustomizeInput struct { +type BuildAndApplyKustomizationInput struct { // Path to the kustomization to build Kustomization string @@ -437,7 +439,7 @@ type BuildAndApplyKustomizeInput struct { WaitIntervals []interface{} } -func (input *BuildAndApplyKustomizeInput) validate() error { +func (input *BuildAndApplyKustomizationInput) validate() error { // If neither WaitForDeployment nor WatchDeploymentLogs is true, we don't need to validate the input if !input.WaitForDeployment && !input.WatchDeploymentLogs { return nil @@ -454,9 +456,9 @@ func (input *BuildAndApplyKustomizeInput) validate() error { return nil } -// BuildAndApplyKustomize takes input from BuildAndApplyKustomizeInput. It builds the provided kustomization +// BuildAndApplyKustomization takes input from BuildAndApplyKustomizationInput. It builds the provided kustomization // and apply it to the cluster provided by clusterProxy. -func BuildAndApplyKustomize(ctx context.Context, input *BuildAndApplyKustomizeInput) error { +func BuildAndApplyKustomization(ctx context.Context, input *BuildAndApplyKustomizationInput) error { Expect(input.validate()).To(BeNil()) var err error kustomization := input.Kustomization @@ -465,6 +467,7 @@ func BuildAndApplyKustomize(ctx context.Context, input *BuildAndApplyKustomizeIn if err != nil { return err } + err = clusterProxy.Apply(ctx, manifest) if err != nil { return err @@ -518,3 +521,30 @@ func DeploymentRolledOut(ctx context.Context, clusterProxy framework.ClusterProx } return false } + +// KubectlDelete shells out to kubectl delete. +func KubectlDelete(ctx context.Context, kubeconfigPath string, resources []byte, args ...string) error { + aargs := append([]string{"delete", "--kubeconfig", kubeconfigPath, "-f", "-"}, args...) + rbytes := bytes.NewReader(resources) + deleteCmd := testexec.NewCommand( + testexec.WithCommand("kubectl"), + testexec.WithArgs(aargs...), + testexec.WithStdin(rbytes), + ) + + fmt.Printf("Running kubectl %s\n", strings.Join(aargs, " ")) + stdout, stderr, err := deleteCmd.Run(ctx) + fmt.Printf("stderr:\n%s\n", string(stderr)) + fmt.Printf("stdout:\n%s\n", string(stdout)) + return err +} + +// BuildAndRemoveKustomization builds the provided kustomization to resources and removes them from the cluster +// provided by clusterProxy. +func BuildAndRemoveKustomization(ctx context.Context, kustomization string, clusterProxy framework.ClusterProxy) error { + manifest, err := buildKustomizeManifest(kustomization) + if err != nil { + return err + } + return KubectlDelete(ctx, clusterProxy.GetKubeconfigPath(), manifest) +} diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index dfeb7777f2..fd8d163e10 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -129,7 +129,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { if e2eConfig.GetVariable("DEPLOY_IRONIC") != "false" { // Install Ironic By("Installing Ironic") - err := BuildAndApplyKustomize(ctx, &BuildAndApplyKustomizeInput{ + err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ Kustomization: e2eConfig.GetVariable("IRONIC_KUSTOMIZATION"), ClusterProxy: clusterProxy, WaitForDeployment: true, @@ -146,7 +146,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { if e2eConfig.GetVariable("DEPLOY_BMO") != "false" { // Install BMO By("Installing BMO") - err := BuildAndApplyKustomize(ctx, &BuildAndApplyKustomizeInput{ + err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ Kustomization: e2eConfig.GetVariable("BMO_KUSTOMIZATION"), ClusterProxy: clusterProxy, WaitForDeployment: true, diff --git a/test/e2e/upgrade_test.go b/test/e2e/upgrade_test.go index ef64fb76cf..204305a570 100644 --- a/test/e2e/upgrade_test.go +++ b/test/e2e/upgrade_test.go @@ -175,13 +175,12 @@ var _ = Describe("BMO Upgrade", func() { specName = "upgrade" secretName = "bmc-credentials" namespace *corev1.Namespace - bmoIronicNamespace string + bmoIronicNamespace = "baremetal-operator-system" upgradeClusterProvider bootstrap.ClusterProvider upgradeClusterProxy framework.ClusterProxy bmh metal3api.BareMetalHost ) BeforeEach(func() { - bmoIronicNamespace = "baremetal-operator-system" var kubeconfigPath string if useExistingCluster { @@ -203,6 +202,7 @@ var _ = Describe("BMO Upgrade", func() { framework.TryAddDefaultSchemes(scheme) metal3api.AddToScheme(scheme) upgradeClusterProxy = framework.NewClusterProxy("bmo-e2e-upgrade", kubeconfigPath, scheme) + if e2eConfig.GetVariable("UPGRADE_DEPLOY_CERT_MANAGER") != "false" { By("Installing cert-manager on the upgrade cluster") cmVersion := e2eConfig.GetVariable("CERT_MANAGER_VERSION") @@ -215,12 +215,15 @@ var _ = Describe("BMO Upgrade", func() { err = checkCertManagerAPI(upgradeClusterProxy) Expect(err).NotTo(HaveOccurred()) } + }) + It("Should upgrade BMO to latest version", func() { if e2eConfig.GetVariable("UPGRADE_DEPLOY_IRONIC") != "false" { // Install Ironic + ironicKustomization := e2eConfig.GetVariable("IRONIC_KUSTOMIZATION") By("Installing Ironic on the upgrade cluster") - err := BuildAndApplyKustomize(ctx, &BuildAndApplyKustomizeInput{ - Kustomization: e2eConfig.GetVariable("IRONIC_KUSTOMIZATION"), + err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ + Kustomization: ironicKustomization, ClusterProxy: upgradeClusterProxy, WaitForDeployment: true, WatchDeploymentLogs: true, @@ -230,23 +233,31 @@ var _ = Describe("BMO Upgrade", func() { WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), }) Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + By("Removing Ironic on the upgrade cluster") + BuildAndRemoveKustomization(ctx, ironicKustomization, upgradeClusterProxy) + }) } if e2eConfig.GetVariable("UPGRADE_DEPLOY_BMO") != "false" { - bmoKustomization := e2eConfig.GetVariable("UPGRADE_BMO_KUSTOMIZATION_FROM") - bmoKustomizationName := filepath.Base(bmoKustomization) - By(fmt.Sprintf("Installing BMO from %s on the upgrade cluster", bmoKustomization)) - err := BuildAndApplyKustomize(ctx, &BuildAndApplyKustomizeInput{ - Kustomization: bmoKustomization, + bmoFromKustomization := e2eConfig.GetVariable("UPGRADE_BMO_KUSTOMIZATION_FROM") + bmoFromKustomizationName := filepath.Base(bmoFromKustomization) + By(fmt.Sprintf("Installing BMO from %s on the upgrade cluster", bmoFromKustomization)) + err := BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ + Kustomization: bmoFromKustomization, ClusterProxy: upgradeClusterProxy, WaitForDeployment: true, WatchDeploymentLogs: true, DeploymentName: "baremetal-operator-controller-manager", DeploymentNamespace: bmoIronicNamespace, - LogPath: filepath.Join(artifactFolder, "logs", fmt.Sprintf("%s-%s", bmoIronicNamespace, specName), fmt.Sprintf("bmo-%s", bmoKustomizationName)), + LogPath: filepath.Join(artifactFolder, "logs", fmt.Sprintf("%s-%s", bmoIronicNamespace, specName), fmt.Sprintf("bmo-%s", bmoFromKustomizationName)), WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), }) Expect(err).NotTo(HaveOccurred()) + DeferCleanup(func() { + By(fmt.Sprintf("Removing BMO from %s on the upgrade cluster", bmoFromKustomization)) + BuildAndRemoveKustomization(ctx, bmoFromKustomization, upgradeClusterProxy) + }) } namespace, cancelWatches = framework.CreateNamespaceAndWatchEvents(ctx, framework.CreateNamespaceAndWatchEventsInput{ @@ -255,9 +266,6 @@ var _ = Describe("BMO Upgrade", func() { Name: fmt.Sprintf("%s-%s", specName, util.RandomString(6)), LogFolder: artifactFolder, }) - }) - - It("Should upgrade BMO to latest version", func() { By("Creating a secret with BMH credentials") bmcCredentialsData := map[string]string{ "username": bmc.User, @@ -304,7 +312,7 @@ var _ = Describe("BMO Upgrade", func() { Expect(err).NotTo(HaveOccurred()) bmoKustomization := e2eConfig.GetVariable("BMO_KUSTOMIZATION") bmoKustomizationName := filepath.Base(bmoKustomization) - err = BuildAndApplyKustomize(ctx, &BuildAndApplyKustomizeInput{ + err = BuildAndApplyKustomization(ctx, &BuildAndApplyKustomizationInput{ Kustomization: bmoKustomization, ClusterProxy: upgradeClusterProxy, WaitForDeployment: false, @@ -314,6 +322,10 @@ var _ = Describe("BMO Upgrade", func() { LogPath: filepath.Join(artifactFolder, "logs", fmt.Sprintf("%s-%s", bmoIronicNamespace, specName), fmt.Sprintf("bmo-%s", bmoKustomizationName)), WaitIntervals: e2eConfig.GetIntervals("default", "wait-deployment"), }) + DeferCleanup(func() { + By("Removing BMO main e2e deployment") + BuildAndRemoveKustomization(ctx, bmoKustomization, upgradeClusterProxy) + }) Expect(err).NotTo(HaveOccurred()) By("Waiting for BMO update to rollout") Eventually(func() bool {