Support go plugins for forges and agent backends (#2751)

As of #2520 

Support to load new forges and agent backends at runtime using go's
plugin system. (https://pkg.go.dev/plugin)

I also added a simple example addon (a new forge which just prints log
statements), it should be removed later of course, but you can see an
example.

---------

Co-authored-by: Michalis Zampetakis <[email protected]>
Co-authored-by: Anbraten <[email protected]>

cmd/agent/agent.go

@@ -43,6 +43,8 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc"
+	"go.woodpecker-ci.org/woodpecker/v2/shared/addon"
+	addonTypes "go.woodpecker-ci.org/woodpecker/v2/shared/addon/types"
 	"go.woodpecker-ci.org/woodpecker/v2/shared/utils"
 	"go.woodpecker-ci.org/woodpecker/v2/version"
 )
@@ -149,21 +151,23 @@ func run(c *cli.Context) error {
 		return err
 	}
 
-	backendCtx := context.WithValue(ctx, types.CliContext, c)
-	backend.Init(backendCtx)
-
 	var wg sync.WaitGroup
 	parallel := c.Int("max-workflows")
 	wg.Add(parallel)
 
-	// new backend
-	backendEngine, err := backend.FindBackend(backendCtx, c.String("backend-engine"))
+	// new engine
+	backendCtx := context.WithValue(ctx, types.CliContext, c)
+	backendEngine, err := getBackendEngine(backendCtx, c.String("backend-engine"), c.StringSlice("addons"))
 	if err != nil {
-		log.Error().Err(err).Msgf("cannot find backend engine '%s'", c.String("backend-engine"))
 		return err
 	}
 
-	// load backend (e.g. init api client)
+	if !backendEngine.IsAvailable(backendCtx) {
+		log.Error().Str("engine", backendEngine.Name()).Msg("selected backend engine is unavailable")
+		return fmt.Errorf("selected backend engine %s is unavailable", backendEngine.Name())
+	}
+
+	// load engine (e.g. init api client)
 	engInfo, err := backendEngine.Load(backendCtx)
 	if err != nil {
 		log.Error().Err(err).Msg("cannot load backend engine")
@@ -247,6 +251,25 @@ func run(c *cli.Context) error {
 	return nil
 }
 
+func getBackendEngine(backendCtx context.Context, backendName string, addons []string) (types.Backend, error) {
+	addonBackend, err := addon.Load[types.Backend](addons, addonTypes.TypeBackend)
+	if err != nil {
+		log.Error().Err(err).Msg("cannot load backend addon")
+		return nil, err
+	}
+	if addonBackend != nil {
+		return addonBackend.Value, nil
+	}
+
+	backend.Init(backendCtx)
+	engine, err := backend.FindBackend(backendCtx, backendName)
+	if err != nil {
+		log.Error().Err(err).Msgf("cannot find backend engine '%s'", backendName)
+		return nil, err
+	}
+	return engine, nil
+}
+
 func runWithRetry(context *cli.Context) error {
 	retryCount := context.Int("connect-retry-count")
 	retryDelay := context.Duration("connect-retry-delay")

cmd/agent/flags.go

@@ -97,4 +97,9 @@ var flags = []cli.Flag{
 		Usage:   "backend to run pipelines on",
 		Value:   "auto-detect",
 	},
+	&cli.StringSliceFlag{
+		EnvVars: []string{"WOODPECKER_ADDONS"},
+		Name:    "addons",
+		Usage:   "list of addon files",
+	},
 }

cmd/server/flags.go

@@ -251,6 +251,11 @@ var flags = append([]cli.Flag{
 		Name:    "enable-swagger",
 		Value:   true,
 	},
+	&cli.StringSliceFlag{
+		EnvVars: []string{"WOODPECKER_ADDONS"},
+		Name:    "addons",
+		Usage:   "list of addon files",
+	},
 	//
 	// backend options for pipeline compiler
 	//

cmd/server/setup.go

@@ -49,6 +49,8 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v2/server/store"
 	"go.woodpecker-ci.org/woodpecker/v2/server/store/datastore"
 	"go.woodpecker-ci.org/woodpecker/v2/server/store/types"
+	"go.woodpecker-ci.org/woodpecker/v2/shared/addon"
+	addonTypes "go.woodpecker-ci.org/woodpecker/v2/shared/addon/types"
 )
 
 func setupStore(c *cli.Context) (store.Store, error) {
@@ -130,8 +132,16 @@ func setupMembershipService(_ *cli.Context, r forge.Forge) cache.MembershipServi
 	return cache.NewMembershipService(r)
 }
 
-// setupForge helper function to setup the forge from the CLI arguments.
+// setupForge helper function to set up the forge from the CLI arguments.
 func setupForge(c *cli.Context) (forge.Forge, error) {
+	addonForge, err := addon.Load[forge.Forge](c.StringSlice("addons"), addonTypes.TypeForge)
+	if err != nil {
+		return nil, err
+	}
+	if addonForge != nil {
+		return addonForge.Value, nil
+	}
+
 	switch {
 	case c.Bool("github"):
 		return setupGitHub(c)

docs/docs/30-administration/10-server-config.md

@@ -521,6 +521,12 @@ Supported variables:
 - `owner`: the repo's owner
 - `repo`: the repo's name
 
+### WOODPECKER_ADDONS
+
+> Default: empty
+
+List of addon files. See [addons](./75-addons/00-overview.md).
+
 ---
 
 ### `WOODPECKER_LIMIT_MEM_SWAP`

docs/docs/30-administration/15-agent-config.md

@@ -178,6 +178,12 @@ Configures if the gRPC server certificate should be verified, only valid when `W
 
 Configures the backend engine to run pipelines on. Possible values are `auto-detect`, `docker`, `local` or `kubernetes`.
 
+### WOODPECKER_ADDONS
+
+> Default: empty
+
+List of addon files. See [addons](./75-addons/00-overview.md).
+
 ### `WOODPECKER_BACKEND_DOCKER_*`
 
 See [Docker backend configuration](./22-backends/10-docker.md#configuration)

docs/docs/30-administration/75-addons/00-overview.md

@@ -0,0 +1,48 @@
+# Addons
+
+:::warning
+Addons are still experimental. Their implementation can change and break at any time.
+:::
+
+:::danger
+You need to trust the author of the addons you use. Depending on their type, addons can access forge authentication codes, your secrets or other sensitive information.
+:::
+
+To adapt Woodpecker to your needs beyond the [configuration](../10-server-config.md), Woodpecker has its own **addon** system, built ontop of [Go's internal plugin system](https://go.dev/pkg/plugin).
+
+Addons can be used for:
+
+- Forges
+- Agent backends
+
+## Restrictions
+
+Addons are restricted by how Go plugins work. This includes the following restrictions:
+
+- only supported on Linux, FreeBSD and macOS
+- addons must have been built for the correct Woodpecker version. If an addon is not provided specifically for this version, you likely won't be able to use it.
+
+## Usage
+
+To use an addon, download the addon version built for your Woodpecker version. Then, you can add the following to your configuration:
+
+```diff
+# docker-compose.yml
+version: '3'
+
+services:
+  woodpecker-server:
+    [...]
+    environment:
++     - WOODPECKER_ADDONS=/path/to/your/addon/file.so
+```
+
+In case you run Woodpecker as container, you probably want to mount the addon binaries to `/opt/addons/`.
+
+You can list multiple addons, Woodpecker will automatically determine their type. If you specify multiple addons with the same type, only the first one will be used.
+
+Using an addon always overwrites Woodpecker's internal setup. This means, that a forge addon will be used if specified, no matter what's configured for the forges natively supported by Woodpecker.
+
+### Bug reports
+
+If you experience bugs, please check which component has the issue. If it's the addon, **do not raise an issue in the main repository**, but rather use the separate addon repositories. To check which component is responsible for the bug, look at the logs. Logs from addons are marked with a special field `addon` containing their addon file name.

docs/docs/30-administration/75-addons/20-creating-addons.md

@@ -0,0 +1,88 @@
+# Creating addons
+
+Addons are written in Go.
+
+## Writing your code
+
+An addon consists of two variables/functions in Go.
+
+1. The `Type` variable. Specifies the type of the addon and must be directly accessed from `shared/addons/types/types.go`.
+2. The `Addon` function which is the main point of your addon.
+   This function takes two arguments:
+
+   1. The zerolog logger you should use to log errors, warnings etc.
+   2. A slice of strings with the environment variables used as configuration.
+
+   It returns two values:
+
+   1. The actual addon. For type reference see [table below](#return-types).
+   2. An error. If this error is not `nil`, Woodpecker exits.
+
+Directly import Woodpecker's Go package (`go.woodpecker-ci.org/woodpecker/woodpecker/v2`) and use the interfaces and types defined there.
+
+### Return types
+
+| Addon type | Return type                                                                      |
+| ---------- | -------------------------------------------------------------------------------- |
+| `Forge`    | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge`             |
+| `Backend`  | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/pipeline/backend/types".Backend` |
+
+## Compiling
+
+After you write your addon code, compile your addon:
+
+```sh
+go build -buildmode plugin
+```
+
+The output file is your addon which is now ready to be used.
+
+## Restrictions
+
+Addons must directly depend on Woodpecker's core (`go.woodpecker-ci.org/woodpecker/woodpecker/v2`).
+The addon must have been built with **excatly the same code** as the Woodpecker instance you'd like to use it on. This means: If you build your addon with a specific commit from Woodpecker `next`, you can likely only use it with the Woodpecker version compiled from this commit.
+Also, if you change something inside of Woodpecker without committing, it might fail because you need to recompile your addon with this code first.
+
+In addition to this, addons are only supported on Linux, FreeBSD and macOS.
+
+:::info
+It is recommended to at least support the latest released version of Woodpecker.
+:::
+
+### Compile for different versions
+
+As long as there are no changes to Woodpecker's interfaces or they are backwards-compatible, you can easily compile the addon for multiple version by changing the version of `go.woodpecker-ci.org/woodpecker/woodpecker/v2` using `go get` before compiling.
+
+## Logging
+
+The entrypoint receives a `zerolog.Logger` as input. **Do not use any other logging solution.** This logger follows the configuration of the Woodpecker instance and adds a special field `addon` to the log entries which allows users to find out which component is writing the log messages.
+
+## Example structure
+
+```go
+package main
+
+import (
+  "context"
+  "net/http"
+
+  "github.com/rs/zerolog"
+  "go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge"
+  forge_types "go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge/types"
+  "go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/model"
+  addon_types "go.woodpecker-ci.org/woodpecker/woodpecker/v2/shared/addon/types"
+)
+
+var Type = addon_types.TypeForge
+
+func Addon(logger zerolog.Logger, env []string) (forge.Forge, error) {
+  logger.Info().Msg("hello world from addon")
+  return &config{l: logger}, nil
+}
+
+type config struct {
+  l zerolog.Logger
+}
+
+// ... in this case, `config` must implement `forge.Forge`. You must directly use Woodpecker's packages - see imports above.
+```

docs/docs/30-administration/75-addons/_category_.yml

@@ -0,0 +1,6 @@
+label: 'Addons'
+collapsible: true
+collapsed: true
+link:
+  type: 'doc'
+  id: 'overview'

shared/addon/addon.go

@@ -0,0 +1,63 @@
+package addon
+
+import (
+	"errors"
+	"os"
+	"plugin"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+
+	"go.woodpecker-ci.org/woodpecker/v2/shared/addon/types"
+)
+
+var pluginCache = map[string]*plugin.Plugin{}
+
+type Addon[T any] struct {
+	Type  types.Type
+	Value T
+}
+
+func Load[T any](files []string, t types.Type) (*Addon[T], error) {
+	for _, file := range files {
+		if _, has := pluginCache[file]; !has {
+			p, err := plugin.Open(file)
+			if err != nil {
+				return nil, err
+			}
+			pluginCache[file] = p
+		}
+
+		typeLookup, err := pluginCache[file].Lookup("Type")
+		if err != nil {
+			return nil, err
+		}
+		if addonType, is := typeLookup.(*types.Type); !is {
+			return nil, errors.New("addon type is incorrect")
+		} else if *addonType != t {
+			continue
+		}
+
+		mainLookup, err := pluginCache[file].Lookup("Addon")
+		if err != nil {
+			return nil, err
+		}
+		main, is := mainLookup.(func(zerolog.Logger, []string) (T, error))
+		if !is {
+			return nil, errors.New("addon main function has incorrect type")
+		}
+
+		logger := log.Logger.With().Str("addon", file).Logger()
+
+		mainOut, err := main(logger, os.Environ())
+		if err != nil {
+			return nil, err
+		}
+		return &Addon[T]{
+			Type:  t,
+			Value: mainOut,
+		}, nil
+	}
+
+	return nil, nil
+}

shared/addon/types/types.go

@@ -0,0 +1,8 @@
+package types
+
+type Type string
+
+const (
+	TypeForge   Type = "forge"
+	TypeBackend Type = "backend"
+)