Add retries for resources cleanup in Botkube E2E and integration tests

test/cloud-slack-dev-e2e/cloud_slack_dev_e2e_test.go

@@ -6,6 +6,7 @@ import (
 	"context"
 	"errors"
 	"fmt"
+	"github.com/avast/retry-go/v4"
 	"net/http"
 	"net/url"
 	"os"
@@ -40,8 +41,9 @@ import (
 
 const (
 	// Chromium is not supported by Slack web app for some reason
-	chromeUserAgent = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
-	authHeaderName  = "Authorization"
+	chromeUserAgent      = "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/114.0.0.0 Safari/537.36"
+	authHeaderName       = "Authorization"
+	cleanupRetryAttempts = 5
 )
 
 type E2ESlackConfig struct {
@@ -303,7 +305,12 @@ func TestCloudSlackE2E(t *testing.T) {
 				return
 			}
 			t.Log("Disconnecting Slack workspace...")
-			gqlCli.MustDeleteSlackWorkspace(t, cfg.BotkubeCloud.TeamOrganizationID, slackWorkspace.ID)
+			err = retryOperation(func() error {
+				return gqlCli.DeleteSlackWorkspace(t, cfg.BotkubeCloud.TeamOrganizationID, slackWorkspace.ID)
+			})
+			if err != nil {
+				t.Logf("Failed to disconnect Slack workspace: %s", err.Error())
+			}
 		})
 
 		t.Log("Initializing Slack...")
@@ -327,14 +334,24 @@ func TestCloudSlackE2E(t *testing.T) {
 			assert.NoError(t, err)
 
 			t.Log("Deleting first deployment...")
-			gqlCli.MustDeleteDeployment(t, graphql.ID(deployment.ID))
+			err = retryOperation(func() error {
+				return gqlCli.DeleteDeployment(t, graphql.ID(deployment.ID))
+			})
+			if err != nil {
+				t.Logf("Failed to delete first deployment: %s", err.Error())
+			}
 		})
 
 		t.Log("Creating a second deployment...")
 		deployment2 := gqlCli.MustCreateBasicDeploymentWithCloudSlack(t, fmt.Sprintf("%s-2", channel.Name()), slackWorkspace.TeamID, channel.Name())
 		t.Cleanup(func() {
 			t.Log("Deleting second deployment...")
-			gqlCli.MustDeleteDeployment(t, graphql.ID(deployment2.ID))
+			err = retryOperation(func() error {
+				return gqlCli.DeleteDeployment(t, graphql.ID(deployment2.ID))
+			})
+			if err != nil {
+				t.Logf("Failed to delete second deployment: %s", err.Error())
+			}
 		})
 
 		botkubeDeploymentUninstalled.Store(false) // about to be installed
@@ -733,3 +750,11 @@ func closePage(t *testing.T, name string, page *rod.Page) {
 		t.Logf("Failed to close page %q: %v", name, err)
 	}
 }
+
+func retryOperation(fn func() error) error {
+	return retry.Do(fn,
+		retry.Attempts(cleanupRetryAttempts),
+		retry.Delay(500*time.Millisecond),
+		retry.LastErrorOnly(false),
+	)
+}

test/cloud_graphql/graphql_client.go

@@ -466,9 +466,16 @@ func (c *Client) MustReportDeploymentStartup(t *testing.T, deploymentId string)
 	return mutation.ReportDeploymentStartup
 }
 
-// MustDeleteSlackWorkspace deletes a slack workspace.
+// MustDeleteSlackWorkspace deletes a slack workspace with panics on error.
 func (c *Client) MustDeleteSlackWorkspace(t *testing.T, orgID, slackWorkspaceID string) {
 	t.Helper()
+	err := c.DeleteSlackWorkspace(t, orgID, slackWorkspaceID)
+	require.NoError(t, err)
+}
+
+// DeleteSlackWorkspace deletes a slack workspace.
+func (c *Client) DeleteSlackWorkspace(t *testing.T, orgID, slackWorkspaceID string) error {
+	t.Helper()
 
 	type Identifiable struct {
 		ID string `graphql:"id"`
@@ -478,15 +485,14 @@ func (c *Client) MustDeleteSlackWorkspace(t *testing.T, orgID, slackWorkspaceID
 		RemovePlatformFromOrganization Identifiable `graphql:"removePlatformFromOrganization(input: $input)"`
 	}
 
-	err := c.Client.Mutate(context.Background(), &mutation, map[string]interface{}{
+	return c.Client.Mutate(context.Background(), &mutation, map[string]interface{}{
 		"input": gqlModel.RemovePlatformFromOrganizationInput{
 			OrganizationID: orgID,
 			Slack: &gqlModel.RemoveSlackFromOrganizationInput{
 				ID: slackWorkspaceID,
 			},
 		},
 	})
-	require.NoError(t, err)
 }
 
 // MustListSlackWorkspacesForOrg returns all slack workspaces scoped to a given organization.

test/e2e/bots_test.go

@@ -6,6 +6,7 @@ import (
 	"bytes"
 	"context"
 	"fmt"
+	"github.com/avast/retry-go/v4"
 	"net/http"
 	"os"
 	"regexp"
@@ -271,7 +272,16 @@ func runBotTest(t *testing.T,
 			assert.NoError(t, err)
 
 			t.Log("Deleting Botkube Cloud instance...")
-			gqlCli.MustDeleteDeployment(t, graphql.ID(deployment.ID))
+			err = retry.Do(func() error {
+				return gqlCli.DeleteDeployment(t, graphql.ID(deployment.ID))
+			},
+				retry.Attempts(5),
+				retry.Delay(500*time.Millisecond),
+				retry.LastErrorOnly(false),
+			)
+			if err != nil {
+				t.Logf("Failed to delete deployment: %s", err.Error())
+			}
 		})
 
 		botkubeDeploymentUninstalled.Store(false) // about to be installed

test/go.mod

@@ -34,7 +34,10 @@ require (
 	k8s.io/utils v0.0.0-20240102154912-e7106e64919e
 )
 
-require gotest.tools/v3 v3.5.1
+require (
+	github.com/avast/retry-go/v4 v4.3.3
+	gotest.tools/v3 v3.5.1
+)
 
 require (
 	cloud.google.com/go v0.112.0 // indirect
@@ -56,7 +59,6 @@ require (
 	github.com/alexflint/go-scalar v1.1.0 // indirect
 	github.com/andybalholm/cascadia v1.3.1 // indirect
 	github.com/auth0/go-jwt-middleware/v2 v2.1.0 // indirect
-	github.com/avast/retry-go/v4 v4.3.3 // indirect
 	github.com/aws/aws-sdk-go v1.44.122 // indirect
 	github.com/bgentry/go-netrc v0.0.0-20140422174119-9fd32a8b3d3d // indirect
 	github.com/blang/semver v3.5.1+incompatible // indirect