Refactor internal services (#915)

cmd/server/server.go

@@ -42,12 +42,11 @@ import (
 	woodpeckerGrpcServer "go.woodpecker-ci.org/woodpecker/v2/server/grpc"
 	"go.woodpecker-ci.org/woodpecker/v2/server/logging"
 	"go.woodpecker-ci.org/woodpecker/v2/server/model"
-	// "go.woodpecker-ci.org/woodpecker/v2/server/plugins/encryption"
-	// encryptedStore "go.woodpecker-ci.org/woodpecker/v2/server/plugins/encryption/wrapper/store"
-	"go.woodpecker-ci.org/woodpecker/v2/server/plugins/permissions"
 	"go.woodpecker-ci.org/woodpecker/v2/server/pubsub"
 	"go.woodpecker-ci.org/woodpecker/v2/server/router"
 	"go.woodpecker-ci.org/woodpecker/v2/server/router/middleware"
+	"go.woodpecker-ci.org/woodpecker/v2/server/services"
+	"go.woodpecker-ci.org/woodpecker/v2/server/services/permissions"
 	"go.woodpecker-ci.org/woodpecker/v2/server/store"
 	"go.woodpecker-ci.org/woodpecker/v2/server/web"
 	"go.woodpecker-ci.org/woodpecker/v2/shared/constant"
@@ -271,48 +270,21 @@ func run(c *cli.Context) error {
 	return g.Wait()
 }
 
-func setupEvilGlobals(c *cli.Context, v store.Store, f forge.Forge) error {
+func setupEvilGlobals(c *cli.Context, s store.Store, f forge.Forge) error {
 	// forge
 	server.Config.Services.Forge = f
-	server.Config.Services.Timeout = c.Duration("forge-timeout")
 
 	// services
-	server.Config.Services.Queue = setupQueue(c, v)
+	server.Config.Services.Queue = setupQueue(c, s)
 	server.Config.Services.Logs = logging.New()
 	server.Config.Services.Pubsub = pubsub.New()
-	var err error
-	server.Config.Services.Registries, err = setupRegistryService(c, v)
-	if err != nil {
-		return err
-	}
-
-	// TODO(1544): fix encrypted store
-	// // encryption
-	// encryptedSecretStore := encryptedStore.NewSecretStore(v)
-	// err := encryption.Encryption(c, v).WithClient(encryptedSecretStore).Build()
-	// if err != nil {
-	// 	log.Fatal().Err(err).Msg("could not create encryption service")
-	// }
-	// server.Config.Services.Secrets = setupSecretService(c, encryptedSecretStore)
-	server.Config.Services.Secrets, err = setupSecretService(c, v)
-	if err != nil {
-		return err
-	}
-	server.Config.Services.Environ, err = setupEnvironService(c, v)
-	if err != nil {
-		return err
-	}
 	server.Config.Services.Membership = setupMembershipService(c, f)
 
-	server.Config.Services.SignaturePrivateKey, server.Config.Services.SignaturePublicKey, err = setupSignatureKeys(v)
+	serviceMangager, err := services.NewManager(c, s)
 	if err != nil {
-		return err
-	}
-
-	server.Config.Services.ConfigService, err = setupConfigService(c)
-	if err != nil {
-		return err
+		return fmt.Errorf("could not setup service manager: %w", err)
 	}
+	server.Config.Services.Manager = serviceMangager
 
 	// authentication
 	server.Config.Pipeline.AuthenticatePublicRepos = c.Bool("authenticate-public-repos")

cmd/server/setup.go

@@ -17,11 +17,6 @@ package main
 
 import (
 	"context"
-	"crypto"
-	"crypto/ed25519"
-	"crypto/rand"
-	"encoding/hex"
-	"errors"
 	"fmt"
 	"net/url"
 	"os"
@@ -41,15 +36,9 @@ import (
 	"go.woodpecker-ci.org/woodpecker/v2/server/forge/gitea"
 	"go.woodpecker-ci.org/woodpecker/v2/server/forge/github"
 	"go.woodpecker-ci.org/woodpecker/v2/server/forge/gitlab"
-	"go.woodpecker-ci.org/woodpecker/v2/server/model"
-	"go.woodpecker-ci.org/woodpecker/v2/server/plugins/config"
-	"go.woodpecker-ci.org/woodpecker/v2/server/plugins/environments"
-	"go.woodpecker-ci.org/woodpecker/v2/server/plugins/registry"
-	"go.woodpecker-ci.org/woodpecker/v2/server/plugins/secrets"
 	"go.woodpecker-ci.org/woodpecker/v2/server/queue"
 	"go.woodpecker-ci.org/woodpecker/v2/server/store"
 	"go.woodpecker-ci.org/woodpecker/v2/server/store/datastore"
-	"go.woodpecker-ci.org/woodpecker/v2/server/store/types"
 	"go.woodpecker-ci.org/woodpecker/v2/shared/addon"
 	addonTypes "go.woodpecker-ci.org/woodpecker/v2/shared/addon/types"
 )
@@ -111,48 +100,6 @@ func setupQueue(c *cli.Context, s store.Store) queue.Queue {
 	return queue.WithTaskStore(queue.New(c.Context), s)
 }
 
-func setupSecretService(c *cli.Context, s model.SecretStore) (model.SecretService, error) {
-	addonService, err := addon.Load[model.SecretService](c.StringSlice("addons"), addonTypes.TypeSecretService)
-	if err != nil {
-		return nil, err
-	}
-	if addonService != nil {
-		return addonService.Value, nil
-	}
-
-	return secrets.New(c.Context, s), nil
-}
-
-func setupRegistryService(c *cli.Context, s store.Store) (model.RegistryService, error) {
-	addonService, err := addon.Load[model.RegistryService](c.StringSlice("addons"), addonTypes.TypeRegistryService)
-	if err != nil {
-		return nil, err
-	}
-	if addonService != nil {
-		return addonService.Value, nil
-	}
-
-	if c.String("docker-config") != "" {
-		return registry.Combined(
-			registry.New(s),
-			registry.Filesystem(c.String("docker-config")),
-		), nil
-	}
-	return registry.New(s), nil
-}
-
-func setupEnvironService(c *cli.Context, _ store.Store) (model.EnvironService, error) {
-	addonService, err := addon.Load[model.EnvironService](c.StringSlice("addons"), addonTypes.TypeEnvironmentService)
-	if err != nil {
-		return nil, err
-	}
-	if addonService != nil {
-		return addonService.Value, nil
-	}
-
-	return environments.Parse(c.StringSlice("environment")), nil
-}
-
 func setupMembershipService(_ *cli.Context, r forge.Forge) cache.MembershipService {
 	return cache.NewMembershipService(r)
 }
@@ -292,46 +239,3 @@ func setupMetrics(g *errgroup.Group, _store store.Store) {
 		}
 	})
 }
-
-// setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for extensions)
-func setupSignatureKeys(_store store.Store) (crypto.PrivateKey, crypto.PublicKey, error) {
-	privKeyID := "signature-private-key"
-
-	privKey, err := _store.ServerConfigGet(privKeyID)
-	if errors.Is(err, types.RecordNotExist) {
-		_, privKey, err := ed25519.GenerateKey(rand.Reader)
-		if err != nil {
-			return nil, nil, fmt.Errorf("failed to generate private key: %w", err)
-		}
-		err = _store.ServerConfigSet(privKeyID, hex.EncodeToString(privKey))
-		if err != nil {
-			return nil, nil, fmt.Errorf("failed to store private key: %w", err)
-		}
-		log.Debug().Msg("created private key")
-		return privKey, privKey.Public(), nil
-	} else if err != nil {
-		return nil, nil, fmt.Errorf("failed to load private key: %w", err)
-	}
-	privKeyStr, err := hex.DecodeString(privKey)
-	if err != nil {
-		return nil, nil, fmt.Errorf("failed to decode private key: %w", err)
-	}
-	privateKey := ed25519.PrivateKey(privKeyStr)
-	return privateKey, privateKey.Public(), nil
-}
-
-func setupConfigService(c *cli.Context) (config.Extension, error) {
-	addonExt, err := addon.Load[config.Extension](c.StringSlice("addons"), addonTypes.TypeConfigService)
-	if err != nil {
-		return nil, err
-	}
-	if addonExt != nil {
-		return addonExt.Value, nil
-	}
-
-	if endpoint := c.String("config-service-endpoint"); endpoint != "" {
-		return config.NewHTTP(endpoint, server.Config.Services.SignaturePrivateKey), nil
-	}
-
-	return nil, nil
-}

docs/docs/20-usage/15-terminiology/index.md

@@ -31,6 +31,7 @@
 - **YAML File**: A file format used to define and configure [workflows][Workflow].
 - **Dependency**: [Workflows][Workflow] can depend on each other, and if possible, they are executed in parallel.
 - **Status**: Status refers to the outcome of a step or [workflow][Workflow] after it has been executed, determined by the internal command exit code. At the end of a [workflow][Workflow], its status is sent to the [forge][Forge].
+- **Service extension**: Some parts of woodpecker internal services like secrets storage or config fetcher can be replaced through service extensions.
 
 ## Pipeline events
 

docs/docs/30-administration/75-addons/00-overview.md

@@ -13,10 +13,6 @@ To adapt Woodpecker to your needs beyond the [configuration](../10-server-config
 Addons can be used for:
 
 - Forges
-- Config services
-- Secret services
-- Environment services
-- Registry services
 
 ## Restrictions
 

docs/docs/30-administration/75-addons/20-creating-addons.md

@@ -19,13 +19,9 @@ Directly import Woodpecker's Go package (`go.woodpecker-ci.org/woodpecker/woodpe
 
 ### Return types
 
-| Addon type           | Return type                                                            |
-| -------------------- | ---------------------------------------------------------------------- |
-| `Forge`              | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge`   |
-| `ConfigService`      | `"go.woodpecker-ci.org/woodpecker/v2/server/plugins/config".Extension` |
-| `SecretService`      | `"go.woodpecker-ci.org/woodpecker/v2/server/model".SecretService`      |
-| `EnvironmentService` | `"go.woodpecker-ci.org/woodpecker/v2/server/model".EnvironmentService` |
-| `RegistryService`    | `"go.woodpecker-ci.org/woodpecker/v2/server/model".RegistryService`    |
+| Addon type | Return type                                                          |
+| ---------- | -------------------------------------------------------------------- |
+| `Forge`    | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge` |
 
 ### Using configurations
 

server/api/global_secret.go

@@ -35,7 +35,8 @@ import (
 //	@Param		page			query	int		false	"for response pagination, page offset number"	default(1)
 //	@Param		perPage			query	int		false	"for response pagination, max items per page"	default(50)
 func GetGlobalSecretList(c *gin.Context) {
-	list, err := server.Config.Services.Secrets.GlobalSecretList(session.Pagination(c))
+	secretService := server.Config.Services.Manager.SecretService()
+	list, err := secretService.GlobalSecretList(session.Pagination(c))
 	if err != nil {
 		c.String(http.StatusInternalServerError, "Error getting global secret list. %s", err)
 		return
@@ -59,7 +60,8 @@ func GetGlobalSecretList(c *gin.Context) {
 //	@Param		secret			path	string	true	"the secret's name"
 func GetGlobalSecret(c *gin.Context) {
 	name := c.Param("secret")
-	secret, err := server.Config.Services.Secrets.GlobalSecretFind(name)
+	secretService := server.Config.Services.Manager.SecretService()
+	secret, err := secretService.GlobalSecretFind(name)
 	if err != nil {
 		handleDBError(c, err)
 		return
@@ -92,7 +94,9 @@ func PostGlobalSecret(c *gin.Context) {
 		c.String(http.StatusBadRequest, "Error inserting global secret. %s", err)
 		return
 	}
-	if err := server.Config.Services.Secrets.GlobalSecretCreate(secret); err != nil {
+
+	secretService := server.Config.Services.Manager.SecretService()
+	if err := secretService.GlobalSecretCreate(secret); err != nil {
 		c.String(http.StatusInternalServerError, "Error inserting global secret %q. %s", in.Name, err)
 		return
 	}
@@ -119,7 +123,8 @@ func PatchGlobalSecret(c *gin.Context) {
 		return
 	}
 
-	secret, err := server.Config.Services.Secrets.GlobalSecretFind(name)
+	secretService := server.Config.Services.Manager.SecretService()
+	secret, err := secretService.GlobalSecretFind(name)
 	if err != nil {
 		handleDBError(c, err)
 		return
@@ -138,7 +143,8 @@ func PatchGlobalSecret(c *gin.Context) {
 		c.String(http.StatusBadRequest, "Error updating global secret. %s", err)
 		return
 	}
-	if err := server.Config.Services.Secrets.GlobalSecretUpdate(secret); err != nil {
+
+	if err := secretService.GlobalSecretUpdate(secret); err != nil {
 		c.String(http.StatusInternalServerError, "Error updating global secret %q. %s", in.Name, err)
 		return
 	}
@@ -156,7 +162,8 @@ func PatchGlobalSecret(c *gin.Context) {
 //	@Param		secret			path	string	true	"the secret's name"
 func DeleteGlobalSecret(c *gin.Context) {
 	name := c.Param("secret")
-	if err := server.Config.Services.Secrets.GlobalSecretDelete(name); err != nil {
+	secretService := server.Config.Services.Manager.SecretService()
+	if err := secretService.GlobalSecretDelete(name); err != nil {
 		handleDBError(c, err)
 		return
 	}

server/api/org_secret.go

@@ -44,7 +44,8 @@ func GetOrgSecret(c *gin.Context) {
 		return
 	}
 
-	secret, err := server.Config.Services.Secrets.OrgSecretFind(orgID, name)
+	secretService := server.Config.Services.Manager.SecretService()
+	secret, err := secretService.OrgSecretFind(orgID, name)
 	if err != nil {
 		handleDBError(c, err)
 		return
@@ -70,7 +71,8 @@ func GetOrgSecretList(c *gin.Context) {
 		return
 	}
 
-	list, err := server.Config.Services.Secrets.OrgSecretList(orgID, session.Pagination(c))
+	secretService := server.Config.Services.Manager.SecretService()
+	list, err := secretService.OrgSecretList(orgID, session.Pagination(c))
 	if err != nil {
 		c.String(http.StatusInternalServerError, "Error getting secret list for %q. %s", orgID, err)
 		return
@@ -116,7 +118,9 @@ func PostOrgSecret(c *gin.Context) {
 		c.String(http.StatusUnprocessableEntity, "Error inserting org %q secret. %s", orgID, err)
 		return
 	}
-	if err := server.Config.Services.Secrets.OrgSecretCreate(orgID, secret); err != nil {
+
+	secretService := server.Config.Services.Manager.SecretService()
+	if err := secretService.OrgSecretCreate(orgID, secret); err != nil {
 		c.String(http.StatusInternalServerError, "Error inserting org %q secret %q. %s", orgID, in.Name, err)
 		return
 	}
@@ -149,7 +153,8 @@ func PatchOrgSecret(c *gin.Context) {
 		return
 	}
 
-	secret, err := server.Config.Services.Secrets.OrgSecretFind(orgID, name)
+	secretService := server.Config.Services.Manager.SecretService()
+	secret, err := secretService.OrgSecretFind(orgID, name)
 	if err != nil {
 		handleDBError(c, err)
 		return
@@ -168,7 +173,8 @@ func PatchOrgSecret(c *gin.Context) {
 		c.String(http.StatusUnprocessableEntity, "Error updating org %q secret. %s", orgID, err)
 		return
 	}
-	if err := server.Config.Services.Secrets.OrgSecretUpdate(orgID, secret); err != nil {
+
+	if err := secretService.OrgSecretUpdate(orgID, secret); err != nil {
 		c.String(http.StatusInternalServerError, "Error updating org %q secret %q. %s", orgID, in.Name, err)
 		return
 	}
@@ -193,7 +199,8 @@ func DeleteOrgSecret(c *gin.Context) {
 		return
 	}
 
-	if err := server.Config.Services.Secrets.OrgSecretDelete(orgID, name); err != nil {
+	secretService := server.Config.Services.Manager.SecretService()
+	if err := secretService.OrgSecretDelete(orgID, name); err != nil {
 		handleDBError(c, err)
 		return
 	}

server/api/pipeline.go

@@ -434,13 +434,7 @@ func PostPipeline(c *gin.Context) {
 		}
 	}
 
-	netrc, err := server.Config.Services.Forge.Netrc(user, repo)
-	if err != nil {
-		handlePipelineErr(c, err)
-		return
-	}
-
-	newpipeline, err := pipeline.Restart(c, _store, pl, user, repo, envs, netrc)
+	newpipeline, err := pipeline.Restart(c, _store, pl, user, repo, envs)
 	if err != nil {
 		handlePipelineErr(c, err)
 	} else {