Support go plugins for forges and agent backends

cmd/agent/agent.go

@@ -43,6 +43,8 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/backend/types"
 	"go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc"
+	"go.woodpecker-ci.org/woodpecker/v2/shared/addon"
+	addonTypes "go.woodpecker-ci.org/woodpecker/v2/shared/addon/types"
 	"go.woodpecker-ci.org/woodpecker/v2/shared/utils"
 	"go.woodpecker-ci.org/woodpecker/v2/version"
 )
@@ -149,20 +151,22 @@ func run(c *cli.Context) error {
 		return err
 	}
 
-	backendCtx := context.WithValue(ctx, types.CliContext, c)
-	backend.Init(backendCtx)
-
 	var wg sync.WaitGroup
 	parallel := c.Int("max-workflows")
 	wg.Add(parallel)
 
 	// new engine
-	engine, err := backend.FindEngine(backendCtx, c.String("backend-engine"))
+	backendCtx := context.WithValue(ctx, types.CliContext, c)
+	engine, err := getEngine(backendCtx, c.String("backend-engine"), c.StringSlice("addons"))
 	if err != nil {
-		log.Error().Err(err).Msgf("cannot find backend engine '%s'", c.String("backend-engine"))
 		return err
 	}
 
+	if !engine.IsAvailable(backendCtx) {
+		log.Error().Str("engine", engine.Name()).Msg("selected backend engine unavailable")
+		return fmt.Errorf("selected backend engine %s unavailable", engine.Name())
+	}
+
 	// load engine (e.g. init api client)
 	engInfo, err := engine.Load(backendCtx)
 	if err != nil {
@@ -247,6 +251,25 @@ func run(c *cli.Context) error {
 	return nil
 }
 
+func getEngine(backendCtx context.Context, engineName string, addons []string) (types.Engine, error) {
+	addonEngine, err := addon.Load[types.Engine](addons, addonTypes.TypeEngine)
+	if err != nil {
+		log.Error().Err(err).Msg("cannot load addon")
+		return nil, err
+	}
+	if addonEngine != nil {
+		return addonEngine.Value, nil
+	}
+
+	backend.Init(backendCtx)
+	engine, err := backend.FindEngine(backendCtx, engineName)
+	if err != nil {
+		log.Error().Err(err).Msgf("cannot find backend engine '%s'", engineName)
+		return nil, err
+	}
+	return engine, nil
+}
+
 func runWithRetry(context *cli.Context) error {
 	retryCount := context.Int("connect-retry-count")
 	retryDelay := context.Duration("connect-retry-delay")

cmd/agent/flags.go

@@ -97,4 +97,8 @@ var flags = []cli.Flag{
 		Usage:   "backend engine to run pipelines on",
 		Value:   "auto-detect",
 	},
+	&cli.StringSliceFlag{
+		EnvVars: []string{"WOODPECKER_ADDONS"},
+		Name:    "addons",
+	},
 }

cmd/server/flags.go

@@ -251,6 +251,11 @@ var flags = append([]cli.Flag{
 		Name:    "enable-swagger",
 		Value:   true,
 	},
+	&cli.StringSliceFlag{
+		EnvVars: []string{"WOODPECKER_ADDONS"},
+		Name:    "addons",
+		Usage:   "list of addon files",
+	},
 	//
 	// backend options for pipeline compiler
 	//

cmd/server/setup.go

@@ -49,6 +49,8 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v2/server/store"
 	"go.woodpecker-ci.org/woodpecker/v2/server/store/datastore"
 	"go.woodpecker-ci.org/woodpecker/v2/server/store/types"
+	"go.woodpecker-ci.org/woodpecker/v2/shared/addon"
+	addonTypes "go.woodpecker-ci.org/woodpecker/v2/shared/addon/types"
 )
 
 func setupStore(c *cli.Context) (store.Store, error) {
@@ -130,8 +132,16 @@ func setupMembershipService(_ *cli.Context, r forge.Forge) cache.MembershipServi
 	return cache.NewMembershipService(r)
 }
 
-// setupForge helper function to setup the forge from the CLI arguments.
+// setupForge helper function to set up the forge from the CLI arguments.
 func setupForge(c *cli.Context) (forge.Forge, error) {
+	add, err := addon.Load[forge.Forge](c.StringSlice("addons"), addonTypes.TypeForge)
+	if err != nil {
+		return nil, err
+	}
+	if add != nil {
+		return add.Value, nil
+	}
+
 	switch {
 	case c.Bool("github"):
 		return setupGitHub(c)

docs/docs/30-administration/10-server-config.md

@@ -521,6 +521,12 @@ Supported variables:
 - `owner`: the repo's owner
 - `repo`: the repo's name
 
+### WOODPECKER_ADDONS
+
+> Default: empty
+
+List of addon files. See [addons](./75-addons/00-overview.md).
+
 ---
 
 ### `WOODPECKER_LIMIT_MEM_SWAP`

docs/docs/30-administration/15-agent-config.md

@@ -178,6 +178,12 @@ Configures if the gRPC server certificate should be verified, only valid when `W
 
 Configures the backend engine to run pipelines on. Possible values are `auto-detect`, `docker`, `local` or `kubernetes`.
 
+### WOODPECKER_ADDONS
+
+> Default: empty
+
+List of addon files. See [addons](./75-addons/00-overview.md).
+
 ### `WOODPECKER_BACKEND_DOCKER_*`
 
 See [Docker backend configuration](./22-backends/10-docker.md#configuration)

docs/docs/30-administration/75-addons/00-overview.md

@@ -0,0 +1,44 @@
+# Addons
+
+:::warning
+Addons are still experimental. Their implementation can change and break at any time.
+:::
+
+To adapt Woodpecker to your needs beyond the [configuration](../10-server-config.md), Woodpecker has its own **addon** system, built ontop of [Go's internal plugin system](https://go.dev/pkg/plugin).
+
+Addons can be used for:
+
+- Forges
+- Agent backends
+
+## Restrictions
+
+Addons are restricted by how Go plugins work. This includes the following restrictions:
+
+- only supported on Linux, FreeBSD and macOS
+- addons must have been built for the correct Woodpecker version. If an addon is not provided specifically for this version, you likely won't be able to use it.
+
+## Usage
+
+To use an addon, download the addon version built for your Woodpecker version. Then, you can add the following to your configuration:
+
+```diff
+# docker-compose.yml
+version: '3'
+
+services:
+  woodpecker-server:
+    [...]
+    environment:
++     - WOODPECKER_ADDONS=/path/to/your/addon/file.so
+```
+
+You may need to [mount the addon file as volume](https://docs.docker.com/storage/volumes/#create-and-manage-volumes) to access it from inside the Docker container.
+
+You can list multiple addons, Woodpecker will automatically determine their type. If you specify multiple addons with the same type, only the first one will be used.
+
+Using an addon always overwrites Woodpecker's internal setup. This means, that a forge addon will be used if specified, no matter what's configured for the forges natively supported by Woodpecker.
+
+### Bug reports
+
+If you experience bugs, please check which component has the issue. If it's the addon, **do not raise an issue in the main repository**, but rather use the separate addon repositories. To check which component is responsible for the bug, look at the logs. Logs from addons are marked with a special field `addon` containing their addon file name.

docs/docs/30-administration/75-addons/20-creating-addons.md

@@ -0,0 +1,88 @@
+# Creating addons
+
+Addons are written in Go.
+
+## Writing your code
+
+An addon consists of two variables/functions in Go.
+
+1. The `Type` variable. Specifies the type of the addon and must be directly accessed from `shared/addons/types/types.go`.
+2. The `Addon` function which is the main point of your addon.
+   This function takes two arguments:
+
+   1. The zerolog logger you should use to log errors, warnings etc.
+   2. A slice of strings with the environment variables used as configuration.
+
+   It returns two values:
+
+   1. The actual addon. For type reference see [table below](#return-types).
+   2. An error. If this error is not `nil`, Woodpecker exits.
+
+Directly import Woodpecker's Go package (`go.woodpecker-ci.org/woodpecker/woodpecker`) and use the interfaces and types defined there.
+
+### Return types
+
+| Addon type | Return type                                                                  |
+| ---------- | ---------------------------------------------------------------------------- |
+| `Forge`    | `"go.woodpecker-ci.org/woodpecker/woodpecker/server/forge".Forge`            |
+| `Engine`   | `"go.woodpecker-ci.org/woodpecker/woodpecker/pipeline/backend/types".Engine` |
+
+## Compiling
+
+After you wrote your addon code, compile your addon:
+
+```sh
+go build -buildmode plugin
+```
+
+The output file is your addon which is now ready to use.
+
+## Restrictions
+
+Addons must directly depend on Woodpecker's core (`go.woodpecker-ci.org/woodpecker/woodpecker`).
+The addon must have been built with **excatly the same code** as the Woodpecker instance you'd like to use it on. This means: If you build your addon with a specific commit from Woodpecker `next`, you can likely only use it with the Woodpecker version compiled from this commit.
+Also, if you change something inside of Woodpecker without commiting, it might fail because you need to recompile your addon with this code.
+
+In addition to this, addons are only supported on Linux, FreeBSD and macOS.
+
+:::info
+It is recommended to at least support the latest released version of Woodpecker.
+:::
+
+### Compile for different versions
+
+As long as there were no changes to Woodpecker's interfaces or they are backwards-compatible, you can easily compile the addon for multiple version by changing the version of `go.woodpecker-ci.org/woodpecker/woodpecker` using `go get` before compiling.
+
+## Logging
+
+The entrypoint receives a `zerolog.Logger` as input. **Do not use any other logging solution.** This logger follows the configuration of the Woodpecker instance and adds a special field `addon` to the log entries which allows users to find out which component is writing the log messages.
+
+## Example structure
+
+```go
+package main
+
+import (
+  "context"
+  "net/http"
+
+  "github.com/rs/zerolog"
+  "go.woodpecker-ci.org/woodpecker/woodpecker/server/forge"
+  forge_types "go.woodpecker-ci.org/woodpecker/woodpecker/server/forge/types"
+  "go.woodpecker-ci.org/woodpecker/woodpecker/server/model"
+  addon_types "go.woodpecker-ci.org/woodpecker/woodpecker/shared/addon/types"
+)
+
+var Type = addon_types.TypeForge
+
+func Addon(logger zerolog.Logger, env []string) (forge.Forge, error) {
+  logger.Info().Msg("hello world from addon")
+  return &config{l: logger}, nil
+}
+
+type config struct {
+  l zerolog.Logger
+}
+
+// ... in this case, `config` must implement `forge.Forge`. You must directly use Woodpecker's packages - see imports above.
+```

docs/docs/30-administration/75-addons/_category_.yml

@@ -0,0 +1,6 @@
+label: 'Addons'
+collapsible: true
+collapsed: true
+link:
+  type: 'doc'
+  id: 'overview'

shared/addon/addon.go

@@ -0,0 +1,66 @@
+package addon
+
+import (
+	"errors"
+	"fmt"
+	"os"
+	"plugin"
+	"reflect"
+
+	"github.com/rs/zerolog"
+	"github.com/rs/zerolog/log"
+
+	"go.woodpecker-ci.org/woodpecker/v2/shared/addon/types"
+)
+
+var pluginCache = map[string]*plugin.Plugin{}
+
+type Addon[T any] struct {
+	Type  types.Type
+	Value T
+}
+
+func Load[T any](files []string, t types.Type) (*Addon[T], error) {
+	for _, file := range files {
+		if _, has := pluginCache[file]; !has {
+			p, err := plugin.Open(file)
+			if err != nil {
+				return nil, err
+			}
+			pluginCache[file] = p
+		}
+
+		typeLookup, err := pluginCache[file].Lookup("Type")
+		if err != nil {
+			return nil, err
+		}
+		if addonType, is := typeLookup.(*types.Type); !is {
+			return nil, errors.New("addon type has incorrect type")
+		} else if *addonType != t {
+			continue
+		}
+
+		mainLookup, err := pluginCache[file].Lookup("Addon")
+		if err != nil {
+			return nil, err
+		}
+		fmt.Println(reflect.TypeOf(mainLookup))
+		main, is := mainLookup.(func(zerolog.Logger, []string) (T, error))
+		if !is {
+			return nil, errors.New("addon main has incorrect type")
+		}
+
+		logger := log.Logger.With().Str("addon", file).Logger()
+
+		mainOut, err := main(logger, os.Environ())
+		if err != nil {
+			return nil, err
+		}
+		return &Addon[T]{
+			Type:  t,
+			Value: mainOut,
+		}, nil
+	}
+
+	return nil, nil
+}

shared/addon/types/types.go

@@ -0,0 +1,8 @@
+package types
+
+type Type string
+
+const (
+	TypeForge  Type = "forge"
+	TypeEngine Type = "engine"
+)