Add release process for Botkube plugins (kubeshop#882)

.github/workflows/branch-build.yml

@@ -113,17 +113,20 @@ jobs:
           version: latest
 
       - name: Build all plugins into dist directory
+        env:
+          # we hardcode plugins version, so it's predictable in e2e tests
+          GORELEASER_CURRENT_TAG: "v1.0.0-devel"
         run: |
           make build-plugins
 
       - name: Run ${{ matrix.integration }} tests
         env:
           SLACK_TESTER_APP_TOKEN: ${{ secrets.SLACK_TESTER_APP_TOKEN }}
-          SLACK_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${GITHUB_SHA} - https://github.com/kubeshop/botkube/commit/${GITHUB_SHA}"
+          SLACK_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}"
           DISCORD_TESTER_APP_TOKEN: ${{ secrets.DISCORD_TESTER_APP_TOKEN }}
           DISCORD_GUILD_ID: ${{ secrets.DISCORD_GUILD_ID }}
-          DISCORD_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${GITHUB_SHA} - https://github.com/kubeshop/botkube/commit/${GITHUB_SHA}"
-          PLUGINS_BINARIES_DIRECTORY: ${{ github.workspace }}/dist
+          DISCORD_ADDITIONAL_CONTEXT_MESSAGE: "Branch test - commit SHA: ${{github.sha}} - https://github.com/kubeshop/botkube/commit/${{github.sha}}"
+          PLUGINS_BINARIES_DIRECTORY: ${{ github.workspace }}/plugin-dist
         run: |
           KUBECONFIG=$(k3d kubeconfig write ${{ matrix.integration }}-test-cluster) \
             make test-integration-${{ matrix.integration }}

.github/workflows/pr-build.yaml

@@ -205,17 +205,20 @@ jobs:
           version: latest
 
       - name: Build all plugins into dist directory
+        env:
+          # we hardcode plugins version, so it's predictable in e2e tests
+          GORELEASER_CURRENT_TAG: "v1.0.0-devel"
         run: |
           make build-plugins
 
       - name: Run ${{ matrix.integration }} tests
         env:
           SLACK_TESTER_APP_TOKEN: ${{ secrets.SLACK_TESTER_APP_TOKEN }}
-          SLACK_ADDITIONAL_CONTEXT_MESSAGE: "Pull request: ${PR_NUMBER} - https://github.com/kubeshop/botkube/pull/${PR_NUMBER}"
+          SLACK_ADDITIONAL_CONTEXT_MESSAGE: "Pull request: ${{ github.event.pull_request.number }} - https://github.com/kubeshop/botkube/pull/${{ github.event.pull_request.number }}"
           DISCORD_TESTER_APP_TOKEN: ${{ secrets.DISCORD_TESTER_APP_TOKEN }}
           DISCORD_GUILD_ID: ${{ secrets.DISCORD_GUILD_ID }}
-          DISCORD_ADDITIONAL_CONTEXT_MESSAGE: "Pull request: ${PR_NUMBER} - https://github.com/kubeshop/botkube/pull/${PR_NUMBER}"
-          PLUGINS_BINARIES_DIRECTORY: ${{ github.workspace }}/dist
+          DISCORD_ADDITIONAL_CONTEXT_MESSAGE: "Pull request: ${{ github.event.pull_request.number }} - https://github.com/kubeshop/botkube/pull/${{ github.event.pull_request.number }}"
+          PLUGINS_BINARIES_DIRECTORY: ${{ github.workspace }}/plugin-dist
         run: |
           KUBECONFIG=$(k3d kubeconfig write ${{ matrix.integration }}-test-cluster) \
             make test-integration-${{ matrix.integration }}

.github/workflows/release.yml

@@ -47,57 +47,34 @@ jobs:
           BASE_TAG=$(echo "${{ github.ref_name }}" | awk -F- '{print $1}')
           git fetch origin "refs/notes/*:refs/notes/*"
           BASE_VERSION=$(echo "${BASE_TAG}" | cut -c2- |  awk 'BEGIN{FS=OFS="."}NF--')
-          PREV_VERSION=$(echo $(git log --pretty=format:"%N" --show-notes="release-${BASE_VERSION}") | awk -F',' '{ print $1 }' | awk NF | awk '{ print $2 }') 
+          PREV_VERSION=$(echo $(git log --pretty=format:"%N" --show-notes="release-${BASE_VERSION}") | awk -F',' '{ print $1 }' | awk NF | awk '{ print $2 }')
           echo ::set-output name=previous-version::$PREV_VERSION
 
-      - name: Run GoReleaser
+      - name: Install GoReleaser
         uses: goreleaser/goreleaser-action@v3
         with:
-          args: release --rm-dist
+          install-only: true
+          version: latest
+
+      - name: Build plugins and generate plugins index.yaml
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
-          IMAGE_TAG: ${{ github.ref_name }}
-          ANALYTICS_API_KEY: ${{ secrets.ANALYTICS_API_KEY }}
+          PLUGIN_DOWNLOAD_URL_BASE_PATH: "https://github.com/${{ github.repository }}/releases/download/${{ github.ref_name }}"
+        run: |
+          make gen-plugins-index
 
-      - name: Extract Tag name
-        id: draft-release-tag-name
+      - name: Generate changelog
         env:
-          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+          GH_TOKEN: ${{ github.token }}
         run: |
-          export URL=$(gh api \
-          -H "Accept: application/vnd.github+json" \
-          /repos/$GITHUB_REPOSITORY/releases --jq '.[] | select(.tag_name == "${{ github.ref_name }}") | .html_url')
-          export TAG="${URL##*/}"
-          echo "Found tag: $TAG"
-          echo ::set-output name=tag::$TAG
+          gh api repos/$GITHUB_REPOSITORY/releases/generate-notes \
+            -f tag_name="${GITHUB_REF#refs/tags/}" \
+            -f target_commitish="${{ steps.prev-version.outputs.previous-version }}" \
+            -q .body > CHANGELOG.md
 
-      - name: Create release
-        uses: actions/github-script@v6
-        with:
-          github-token: ${{ secrets.GH_TOKEN }}
-          script: |
-            try {
-              const releaseNotes = await github.rest.repos.generateReleaseNotes({
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-                tag_name: "${{ steps.draft-release-tag-name.outputs.tag }}",
-                previous_tag_name: "${{ steps.prev-version.outputs.previous-version }}"
-              });
-              const release = await github.rest.repos.getReleaseByTag({
-                tag: "${{ steps.draft-release-tag-name.outputs.tag }}",
-                owner: context.repo.owner,
-                repo: context.repo.repo,
-              });
-              await github.rest.repos.updateRelease({
-                release_id: release.data.id,
-                draft: false,
-                name: "${{ github.ref_name }}",
-                owner: context.repo.owner,
-                prerelease: true,
-                repo: context.repo.repo,
-                tag_name: "${{ github.ref_name }}",
-                body: releaseNotes.data.body
-              });
-            } catch (error) {
-              core.setFailed(error.message);
-            }
+      - name: Publish draft release
+        env:
+          GITHUB_TOKEN: ${{ secrets.GH_TOKEN }}
+          IMAGE_TAG: ${{ github.ref_name }}
+          ANALYTICS_API_KEY: ${{ secrets.ANALYTICS_API_KEY }}
+        run: |
+          goreleaser release --rm-dist --release-notes=CHANGELOG.md

.gitignore

@@ -39,4 +39,8 @@ tags
 ./botkube
 ./botkube-test
 
+# Release
 dist/
+plugin-dist/
+plugins-index.yaml
+CHANGELOG.md

.goreleaser.plugin.yaml

@@ -1,33 +1,37 @@
+# GoReleaser already creates Botkube artifacts in the ./dist folder.
+# To not override them during release, we use a different folder
+dist: plugin-dist
+
 before:
   hooks:
     - go mod download
 
 builds:
   - id: echo
+    main: cmd/executor/echo/main.go
     binary: executor_echo_{{ .Os }}_{{ .Arch }}
+
     no_unique_dist_dir: true
-    main: cmd/executor/echo/main.go
-    env:
+    env: &env
       - CGO_ENABLED=0
-    goos:
+    goos: &goos
       - linux
       - darwin
-    goarch:
+    goarch: &goarch
       - amd64
       - arm64
-    goarm:
+    goarm: &goarm
       - 7
+
   - id: cm-watcher
+    main: cmd/source/cm-watcher/main.go
     binary: source_cm-watcher_{{ .Os }}_{{ .Arch }}
+
     no_unique_dist_dir: true
-    main: cmd/source/cm-watcher/main.go
-    env:
-      - CGO_ENABLED=0
-    goos:
-      - linux
-      - darwin
-    goarch:
-      - amd64
-      - arm64
-    goarm:
-      - 7
+    env: *env
+    goos: *goos
+    goarch: *goarch
+    goarm: *goarm
+
+snapshot:
+  name_template: 'v{{ .Version }}'

.goreleaser.yml

@@ -27,20 +27,26 @@ builds:
       - arm64
     goarm:
       - 7
-archives:
-  - id: my-archive
-    builds:
-      - botkube
+
 snapshot:
   name_template: "{{ .Env.IMAGE_TAG }}"
+
 release:
   # If set to true, will not auto-publish the release.
   draft: true
   prerelease: auto
-  ids:
-    - botkube
+
+  # Add extra pre-existing files to the release.
+  # Prerequisites:
+  # - build plugin binaries
+  # - build plugin index.yaml
+  extra_files:
+    - glob: ./plugin-dist/executor_*
+    - glob: ./plugin-dist/source_*
+    - glob: ./plugins-index.yaml
+
 changelog:
-  skip: true
+  skip: false
 
 dockers:
   - image_templates:

Makefile

@@ -1,5 +1,5 @@
 .DEFAULT_GOAL := build
-.PHONY: container-image test test-integration-slack test-integration-discord build pre-build publish lint lint-fix go-import-fmt system-check save-images load-and-push-images gen-grpc-resources build-plugins build-plugins-single
+.PHONY: container-image test test-integration-slack test-integration-discord build pre-build publish lint lint-fix go-import-fmt system-check save-images load-and-push-images gen-grpc-resources gen-plugins-index build-plugins build-plugins-single
 
 # Show this help.
 help:
@@ -80,6 +80,10 @@ system-check:
 gen-grpc-resources:
 	@./hack/gen-grpc-resources.sh
 
+# Generate plugins YAML index file.
+gen-plugins-index: build-plugins
+	go run ./hack/gen-plugin-index.go
+
 # Pre-build checks
 pre-build: system-check
 

hack/gen-plugin-index.go

@@ -0,0 +1,51 @@
+package main
+
+import (
+	"flag"
+	"log"
+	"os"
+	"path/filepath"
+
+	"github.com/sirupsen/logrus"
+	"gopkg.in/yaml.v3"
+
+	"github.com/kubeshop/botkube/internal/plugin"
+)
+
+const filePerm = 0o644
+
+func main() {
+	var (
+		urlBasePath = flag.String("url-base-path", os.Getenv("PLUGIN_DOWNLOAD_URL_BASE_PATH"), "Defines the URL base path for downloading the plugin binaries")
+		binsDir     = flag.String("binaries-path", "./plugin-dist", "Defines the local path to plugins binaries folder")
+		output      = flag.String("output-path", "./plugins-index.yaml", "Defines the local path where index YAML should be saved")
+	)
+
+	flag.Parse()
+	logger := logrus.New()
+
+	idxBuilder := plugin.NewIndexBuilder(logger)
+
+	absBinsDir, err := filepath.Abs(*binsDir)
+	exitOnError("while resolving an absolute path of binaries folder", err)
+
+	logger.WithFields(logrus.Fields{
+		"binDir":      absBinsDir,
+		"urlBasePath": *urlBasePath,
+	}).Info("Building index..")
+	idx, err := idxBuilder.Build(absBinsDir, *urlBasePath)
+	exitOnError("while building plugin index", err)
+
+	raw, err := yaml.Marshal(idx)
+	exitOnError("while marshaling index into YAML format", err)
+
+	logger.WithField("output", *output).Info("Saving index file...")
+	err = os.WriteFile(*output, raw, filePerm)
+	exitOnError("while saving index file", err)
+}
+
+func exitOnError(context string, err error) {
+	if err != nil {
+		log.Fatalf("%s: %s", context, err)
+	}
+}

helm/botkube/e2e-test-values.yaml

@@ -180,7 +180,7 @@ executors:
         resources: [ "deployments" ]
 
   'plugin-based':
-    botkube/echo@v0.1.0:
+    botkube/echo@v1.0.1-devel:
       enabled: true
       config:
         changeResponseToUpperCase: true

test/README.md

@@ -156,12 +156,12 @@ We test also the Botkube plugins system. To compile Botkube plugins, run:
 make build-plugins
 ```
 
-By default, built plugins' binaries are available under `dist` directory. The e2e framework builds the plugins index file dynamically and starts the HTTP server that is later accessible from the k3d cluster.
+By default, built plugins' binaries are available under `plugin-dist` directory. The e2e framework builds the plugins index file dynamically and starts the HTTP server that is later accessible from the k3d cluster.
 
 To override default settings, export following environment variables:
 
 ```bash
-export PLUGINS_BINARIES_DIRECTORY="./dist"
+export PLUGINS_BINARIES_DIRECTORY="./plugin-dist"
 export PLUGINS_SERVER_HOST="http://host.k3d.internal" # on K3d enabling you to access your host system by referring to it as host.k3d.internal
 export PLUGINS_SERVER_PORT="3000"
 ```

test/helpers/plugin_server.go

@@ -17,7 +17,7 @@ func main() {
 		host = "http://localhost"
 	}
 
-	binDir := filepath.Join(dir, "dist")
+	binDir := filepath.Join(dir, "plugin-dist")
 	indexEndpoint, startServerFn := fake.NewPluginServer(fake.PluginConfig{
 		BinariesDirectory: binDir,
 		Server: fake.PluginServer{