Migrate Cloud Slack Dev E2E tests and fix migration test

kubeshop · Sep 26, 2023 · d299736 · d299736
1 parent fdb389c
commit d299736
Show file tree

Hide file tree

Showing 18 changed files with 2,862 additions and 73 deletions.
diff --git a/.github/workflows/branch-build.yml b/.github/workflows/branch-build.yml
@@ -4,6 +4,7 @@ on:
   push:
     branches:
       - main
+      - 'cloud-slack-dev-e2e' # TODO: Remove before merge
   repository_dispatch:
     types: [ trigger-e2e-tests ]
 
@@ -49,66 +50,10 @@ jobs:
         env:
           ANALYTICS_API_KEY: ${{ secrets.ANALYTICS_API_KEY }}
       - name: Install Helm
-        uses: azure/setup-helm@v1
+        uses: azure/setup-helm@v3
         with:
           version: ${{ env.HELM_VERSION }}
 
-  migration-e2e-test:
-    name: Migration e2e test
-    runs-on: ubuntu-latest
-    needs: [ build ]
-    concurrency:
-      group: e2e-migration
-      cancel-in-progress: false
-    permissions:
-      contents: read
-      packages: read
-    strategy:
-      fail-fast: false
-      matrix:
-        e2e:
-          - discord
-    steps:
-      - name: Checkout code
-        uses: actions/checkout@v4
-        with:
-          persist-credentials: false
-      - name: Install GoReleaser
-        uses: goreleaser/goreleaser-action@v5
-        with:
-          install-only: true
-          version: latest
-      - name: Setup Go
-        uses: actions/setup-go@v4
-        with:
-          go-version-file: 'go.mod'
-          cache: true
-      - name: Run GoReleaser
-        run: make release-snapshot-cli
-      - name: Add botkube alias
-        run: |
-          echo BOTKUBE_BINARY_PATH="$PWD/dist/botkube-cli_linux_amd64_v1/botkube" >> $GITHUB_ENV
-      - name: Install Helm
-        uses: azure/setup-helm@v1
-        with:
-          version: ${{ env.HELM_VERSION }}
-      - name: Download k3d
-        run: "wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=${K3D_VERSION} bash"
-      - name: Create k3d cluster
-        run: "k3d cluster create migration-test-cluster --wait --timeout=5m"
-      - name: Run e2e tests for botkube client
-        env:
-          DISCORD_BOT_ID: ${{ secrets.DISCORD_BOT_ID }}
-          DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
-          DISCORD_GUILD_ID: ${{ secrets.DISCORD_GUILD_ID }}
-          DISCORD_TESTER_APP_TOKEN: ${{ secrets.DISCORD_TESTER_APP_TOKEN }}
-          BOTKUBE_CLOUD_DEV_GQL_ENDPOINT: ${{ secrets.BOTKUBE_CLOUD_DEV_GQL_ENDPOINT }}
-          BOTKUBE_CLOUD_DEV_REFRESH_TOKEN: ${{ secrets.BOTKUBE_CLOUD_DEV_REFRESH_TOKEN }}
-          BOTKUBE_CLOUD_DEV_AUTH0_CLIENT_ID: ${{ secrets.BOTKUBE_CLOUD_DEV_AUTH0_CLIENT_ID }}
-        run: |
-          KUBECONFIG=$(k3d kubeconfig write migration-test-cluster) \
-            make test-migration-tool
-
   integration-tests:
     name: Integration tests
     runs-on: ubuntu-latest
@@ -140,7 +85,7 @@ jobs:
           cache: true
 
       - name: Install Helm
-        uses: azure/setup-helm@v1
+        uses: azure/setup-helm@v3
         with:
           version: ${{ env.HELM_VERSION }}
 
@@ -192,14 +137,76 @@ jobs:
           KUBECONFIG=$(k3d kubeconfig write ${{ matrix.integration }}-test-cluster) \
             make test-integration-${{ matrix.integration }}
 
-  slackNotification:
-    name: Slack Notification
+  cli-migration-e2e:
+    name: CLI Migration E2E tests
     runs-on: ubuntu-latest
-    needs: [ migration-e2e-test ]
-    if: failure()
+    needs: [ build ]
+    permissions:
+      contents: read
+      packages: read
+    concurrency:
+      group: cli-migration-e2e
+      cancel-in-progress: false
+    strategy:
+      fail-fast: false
+      matrix:
+        e2e:
+          - discord
     steps:
+      - name: Checkout code
+        uses: actions/checkout@v4
+        with:
+          persist-credentials: false
+      - name: Install GoReleaser
+        uses: goreleaser/goreleaser-action@v5
+        with:
+          install-only: true
+          version: latest
+      - name: Setup Go
+        uses: actions/setup-go@v4
+        with:
+          go-version-file: 'go.mod'
+          cache: true
+      - name: Run GoReleaser
+        run: make release-snapshot-cli
+      - name: Add botkube alias
+        run: |
+          echo BOTKUBE_BINARY_PATH="$PWD/dist/botkube-cli_linux_amd64_v1/botkube" >> $GITHUB_ENV
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: ${{ env.HELM_VERSION }}
+      - name: Download k3d
+        run: "wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=${K3D_VERSION} bash"
+      - name: Create k3d cluster
+        run: "k3d cluster create cli-migration-e2e-cluster --wait --timeout=5m"
+      - name: Run e2e tests for botkube client
+        env:
+          DISCORD_BOT_ID: ${{ secrets.DISCORD_BOT_ID }}
+          DISCORD_BOT_TOKEN: ${{ secrets.DISCORD_BOT_TOKEN }}
+          DISCORD_GUILD_ID: ${{ secrets.DISCORD_GUILD_ID }}
+          DISCORD_TESTER_APP_TOKEN: ${{ secrets.DISCORD_TESTER_APP_TOKEN }}
+          BOTKUBE_CLOUD_DEV_GQL_ENDPOINT: ${{ secrets.BOTKUBE_CLOUD_DEV_GQL_ENDPOINT }}
+          BOTKUBE_CLOUD_DEV_REFRESH_TOKEN: ${{ secrets.BOTKUBE_CLOUD_DEV_REFRESH_TOKEN }}
+          BOTKUBE_CLOUD_DEV_AUTH0_CLIENT_ID: ${{ secrets.BOTKUBE_CLOUD_DEV_AUTH0_CLIENT_ID }}
+        run: |
+          KUBECONFIG=$(k3d kubeconfig write cli-migration-e2e-cluster) make test-cli-migration-e2e
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: screenshots_dump_${{github.sha}}
+          path: ${{ runner.temp }}/screenshots
+          retention-days: 5
+
+      - name: Dump cluster state
+        if: ${{ failure() }}
+        uses: ./.github/actions/dump-cluster
+
       - name: Slack Notification
         uses: rtCamp/action-slack-notify@v2
+        if: ${{ failure() }}
         env:
           SLACK_USERNAME: Botkube Cloud CI
           SLACK_COLOR: 'red'
@@ -209,3 +216,69 @@ jobs:
           SLACK_ICON_EMOJI: ':this-is-fine-fire:'
           SLACK_FOOTER: "Fingers crossed it's just an outdated/flaky test..."
           SLACK_WEBHOOK: ${{ secrets.SLACK_CI_ALERTS_WEBHOOK }}
+
+  cloud-slack-dev-e2e:
+    name: Botkube Cloud Slack Dev E2E
+    runs-on: ubuntu-latest
+    needs: [ build ]
+    permissions:
+      contents: read
+      packages: read
+    concurrency:
+      group: cloud-slack-dev-e2e
+      cancel-in-progress: false
+    steps:
+      - name: Checkout
+        uses: actions/checkout@v3
+
+      - name: Install Helm
+        uses: azure/setup-helm@v3
+        with:
+          version: ${{ env.HELM_VERSION }}
+
+      - name: Download k3d
+        run: "wget -q -O - https://raw.githubusercontent.com/k3d-io/k3d/main/install.sh | TAG=${K3D_VERSION} bash"
+      - name: Create k3d cluster
+        run: "k3d cluster create cloud-slack-dev-e2e-cluster --wait --timeout=5m"
+
+      - name: Run e2e tests
+        env:
+          SLACK_WORKSPACE_NAME: ${{ secrets.E2E_DEV_SLACK_WORKSPACE_NAME }}
+          SLACK_EMAIL: ${{ secrets.E2E_DEV_SLACK_EMAIL }}
+          SLACK_PASSWORD: ${{ secrets.E2E_DEV_SLACK_USER_PASSWORD }}
+          SLACK_TESTER_TESTER_BOT_TOKEN: ${{ secrets.E2E_DEV_SLACK_TESTER_BOT_TOKEN }}
+          SLACK_TESTER_BOT_NAME: botkubedev
+          SLACK_TESTER_MESSAGE_WAIT_TIMEOUT: 1m
+          BOTKUBE_CLOUD_EMAIL: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_EMAIL }}
+          BOTKUBE_CLOUD_PASSWORD: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_PASSWORD }}
+          BOTKUBE_CLOUD_TEAM_ORGANIZATION_ID: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_TEAM_ORGANIZATION_ID }}
+          BOTKUBE_CLOUD_FREE_ORGANIZATION_ID: ${{ secrets.E2E_DEV_BOTKUBE_CLOUD_FREE_ORGANIZATION_ID }}
+          SCREENSHOTS_DIR: ${{ runner.temp }}/screenshots
+          DEBUG_MODE: true
+        run: |
+          KUBECONFIG=$(k3d kubeconfig write cli-migration-e2e-cluster) make test-cloud-slack-dev-e2e
+
+      - name: Upload artifacts
+        uses: actions/upload-artifact@v3
+        if: ${{ always() }}
+        with:
+          name: screenshots_dump_${{github.sha}}
+          path: ${{ runner.temp }}/screenshots
+          retention-days: 5
+
+      - name: Dump cluster state
+        if: ${{ failure() }}
+        uses: ./.github/actions/dump-cluster
+
+      - name: Slack Notification
+        uses: rtCamp/action-slack-notify@v2
+        if: ${{ failure() }}
+        env:
+          SLACK_USERNAME: Botkube Cloud CI
+          SLACK_COLOR: 'red'
+          SLACK_TITLE: 'Message'
+          SLACK_CHANNEL: 'botkube-cloud-ci-alerts'
+          SLACK_MESSAGE: 'Cloud Slack Dev E2E tests failed :scream:'
+          SLACK_ICON_EMOJI: ':this-is-fine-fire:'
+          SLACK_FOOTER: "Fingers crossed it's just an outdated/flaky test..."
+          SLACK_WEBHOOK: ${{ secrets.SLACK_CI_ALERTS_WEBHOOK }}
diff --git a/.github/workflows/pr-build.yaml b/.github/workflows/pr-build.yaml
@@ -210,7 +210,7 @@ jobs:
           cache: true
 
       - name: Install Helm
-        uses: azure/setup-helm@v1
+        uses: azure/setup-helm@v3
         with:
           version: ${{ env.HELM_VERSION }}
 

diff --git a/Makefile b/Makefile
@@ -23,9 +23,15 @@ test-integration-slack: system-check
 test-integration-discord: system-check
 	@go test -v -tags=integration -race -count=1 ./test/e2e/... -run "TestDiscord"
 
-test-migration-tool: system-check
+test-cli-migration-e2e: system-check
 	@go test -v -tags=migration -race -count=1 ./test/e2e/...
 
+test-cloud-slack-dev-e2e: system-check
+	@go test -tags=cloud_slack_dev_e2e -race -p 1 -v -timeout 30m ./test/cloud-slack-dev-e2e/...
+
+test-cloud-slack-dev-e2e-show-browser: system-check
+	@go test -tags=cloud_slack_dev_e2e -race -p 1 -v -timeout 30m -rod=show ./test/cloud-slack-dev-e2e/...
+
 # Build the binary
 build: pre-build
 	@cd cmd/botkube-agent;GOOS_VAL=$(shell go env GOOS) CGO_ENABLED=0 GOARCH_VAL=$(shell go env GOARCH) go build -o $(shell go env GOPATH)/bin/botkube

diff --git a/go.mod b/go.mod
@@ -25,6 +25,7 @@ require (
 	github.com/go-playground/locales v0.14.0
 	github.com/go-playground/universal-translator v0.18.0
 	github.com/go-playground/validator/v10 v10.11.0
+	github.com/go-rod/rod v0.113.3
 	github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572
 	github.com/google/go-github/v53 v53.2.0
 	github.com/google/go-querystring v1.1.0
@@ -237,6 +238,11 @@ require (
 	github.com/xlab/treeprint v1.2.0 // indirect
 	github.com/xo/terminfo v0.0.0-20220910002029-abceb7e1c41e // indirect
 	github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c // indirect
+	github.com/ysmood/fetchup v0.2.3 // indirect
+	github.com/ysmood/goob v0.4.0 // indirect
+	github.com/ysmood/got v0.34.1 // indirect
+	github.com/ysmood/gson v0.7.3 // indirect
+	github.com/ysmood/leakless v0.8.0 // indirect
 	go.opencensus.io v0.24.0 // indirect
 	go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.36.4 // indirect
 	go.opentelemetry.io/otel v1.16.0 // indirect

diff --git a/go.sum b/go.sum
@@ -515,6 +515,8 @@ github.com/go-playground/validator/v10 v10.2.0/go.mod h1:uOYAAleCW8F/7oMFd6aG0GO
 github.com/go-playground/validator/v10 v10.4.1/go.mod h1:nlOn6nFhuKACm19sB/8EGNn9GlaMV7XkbRSipzJ0Ii4=
 github.com/go-playground/validator/v10 v10.11.0 h1:0W+xRM511GY47Yy3bZUbJVitCNg2BOGlCyvTqsp/xIw=
 github.com/go-playground/validator/v10 v10.11.0/go.mod h1:i+3WkQ1FvaUjjxh1kSvIA4dMGDBiPU55YFDl0WbKdWU=
+github.com/go-rod/rod v0.113.3 h1:oLiKZW721CCMwA5g7977cWfcAKQ+FuosP47Zf1QiDrA=
+github.com/go-rod/rod v0.113.3/go.mod h1:aiedSEFg5DwG/fnNbUOTPMTTWX3MRj6vIs/a684Mthw=
 github.com/go-sql-driver/mysql v1.6.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg=
 github.com/go-sql-driver/mysql v1.7.1 h1:lUIinVbN1DY0xBg0eMOzmmtGoHwWBbvnWubQUrtU8EI=
 github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY=
@@ -1269,6 +1271,20 @@ github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c h1:3lbZUMbMiGUW/LMkfsEAB
 github.com/xtgo/uuid v0.0.0-20140804021211-a0b114877d4c/go.mod h1:UrdRz5enIKZ63MEE3IF9l2/ebyx59GyGgPi+tICQdmM=
 github.com/xyproto/randomstring v1.0.5 h1:YtlWPoRdgMu3NZtP45drfy1GKoojuR7hmRcnhZqKjWU=
 github.com/xyproto/randomstring v1.0.5/go.mod h1:rgmS5DeNXLivK7YprL0pY+lTuhNQW3iGxZ18UQApw/E=
+github.com/ysmood/fetchup v0.2.3 h1:ulX+SonA0Vma5zUFXtv52Kzip/xe7aj4vqT5AJwQ+ZQ=
+github.com/ysmood/fetchup v0.2.3/go.mod h1:xhibcRKziSvol0H1/pj33dnKrYyI2ebIvz5cOOkYGns=
+github.com/ysmood/goob v0.4.0 h1:HsxXhyLBeGzWXnqVKtmT9qM7EuVs/XOgkX7T6r1o1AQ=
+github.com/ysmood/goob v0.4.0/go.mod h1:u6yx7ZhS4Exf2MwciFr6nIM8knHQIE22lFpWHnfql18=
+github.com/ysmood/gop v0.0.2 h1:VuWweTmXK+zedLqYufJdh3PlxDNBOfFHjIZlPT2T5nw=
+github.com/ysmood/gop v0.0.2/go.mod h1:rr5z2z27oGEbyB787hpEcx4ab8cCiPnKxn0SUHt6xzk=
+github.com/ysmood/got v0.34.1 h1:IrV2uWLs45VXNvZqhJ6g2nIhY+pgIG1CUoOcqfXFl1s=
+github.com/ysmood/got v0.34.1/go.mod h1:yddyjq/PmAf08RMLSwDjPyCvHvYed+WjHnQxpH851LM=
+github.com/ysmood/gotrace v0.6.0 h1:SyI1d4jclswLhg7SWTL6os3L1WOKeNn/ZtzVQF8QmdY=
+github.com/ysmood/gotrace v0.6.0/go.mod h1:TzhIG7nHDry5//eYZDYcTzuJLYQIkykJzCRIo4/dzQM=
+github.com/ysmood/gson v0.7.3 h1:QFkWbTH8MxyUTKPkVWAENJhxqdBa4lYTQWqZCiLG6kE=
+github.com/ysmood/gson v0.7.3/go.mod h1:3Kzs5zDl21g5F/BlLTNcuAGAYLKt2lV5G8D1zF3RNmg=
+github.com/ysmood/leakless v0.8.0 h1:BzLrVoiwxikpgEQR0Lk8NyBN5Cit2b1z+u0mgL4ZJak=
+github.com/ysmood/leakless v0.8.0/go.mod h1:R8iAXPRaG97QJwqxs74RdwzcRHT1SWCGTNqY8q0JvMQ=
 github.com/yuin/goldmark v1.1.25/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.27/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=
 github.com/yuin/goldmark v1.1.32/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9decYSb74=

diff --git a/test/README.md b/test/README.md
@@ -198,3 +198,56 @@ kubectl delete cm botkube-system -n botkube # or the namespace where Botkube is
 ```
 
 If you don't remove the ConfigMap, any e2e tests looking to verify that a help message is displayed will error. This also stops the rest of the e2e tests from running.
+
+## Testing Cloud Slack end-to-end with Botkube Cloud
+
+You can test Botkube Cloud Slack with Botkube Cloud. Follow the instructions below to set up the test environment and run the tests. 
+
+### Setting up test environment
+
+1. Create Slack user for testing purposes with access to a given Slack workspace.
+1. Create Botkube Cloud user with two organizations: FREE and TEAM one (with Cloud Slack quota).
+1. Follow the [Testing Slack](#testing-slack) instructions to set up "Tester" bot.
+
+### Running tests
+
+To run the tests, get all the noted data from previous steps and export them as environment variables.
+
+```bash
+# Required
+export SLACK_WORKSPACE_NAME="" # e.g. my-workspace
+# The test log ins to the Slack workspace using the credentials below
+export SLACK_EMAIL="" # e.g. [email protected]
+export SLACK_PASSWORD=""
+export SLACK_TESTER_BOT_NAME="" # e.g. botkubedev
+export SLACK_TESTER_TESTER_BOT_TOKEN="" # e.g. xoxb-...
+export BOTKUBE_CLOUD_EMAIL="" # e.g. [email protected]
+export BOTKUBE_CLOUD_PASSWORD=""
+export BOTKUBE_CLOUD_TEAM_ORGANIZATION_ID="" # e.g. 204a2d86-265c-4ae2-89a8-928f823ec4da
+export BOTKUBE_CLOUD_FREE_ORGANIZATION_ID="" # e.g. c03bd605-7b8d-490f-b4d5-57c8a0560e83
+
+# Optional - useful for running tests locally 
+export KUBECONFIG="" # path to your kubeconfig
+export BOTKUBE_CLOUD_API_BASE_URL="" e.g. http://localhost:8080
+export BOTKUBE_CLOUD_API_SLACK_APP_INSTALLATION_BASE_URL_OVERRIDE="" # provide if necessary e.g. using ngrok: https://d5ac-194-33-77-250.ngrok-free.app
+export SLACK_BOT_DISPLAY_NAME="" # e.g. BotkubeDev
+export SLACK_WORKSPACE_ALREADY_CONNECTED="true"
+export SLACK_DISCONNECT_WORKSPACE_AFTER_TESTS="false"
+export PAGE_TIMEOUT="1m"
+export SCREENSHOTS_ENABLED="false" # disable screenshots
+```
+
+To run the test in headless mode (without browser window), run:
+
+```shell
+make test-cloud-slack-dev-e2e
+```
+
+To run the tests with Chromium browser window visible, run:
+
+```shell
+test-cloud-slack-dev-e2e-show-browser
+```
+
+Refer to the `E2ESlackConfig` (`./cloud-slack-dev-e2e/e2e_test.go`) for all possible environment variables.
+