From df09748d6d148bc13ba4b46721a20fed66d92c72 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Wed, 7 Feb 2024 21:26:30 +0100 Subject: [PATCH 01/28] addons --- cmd/server/setup.go | 33 --------- server/model/registry.go | 15 ---- server/model/repo.go | 6 ++ server/model/secret.go | 23 ------ server/plugins/plugins.go | 119 ++++++++++++++++++++++++++++++ server/plugins/registry/plugin.go | 18 +++++ server/plugins/secrets/builtin.go | 2 +- server/plugins/secrets/plugin.go | 26 +++++++ 8 files changed, 170 insertions(+), 72 deletions(-) create mode 100644 server/plugins/plugins.go create mode 100644 server/plugins/registry/plugin.go create mode 100644 server/plugins/secrets/plugin.go diff --git a/cmd/server/setup.go b/cmd/server/setup.go index b7ebb8aeb4..9cfefe1b01 100644 --- a/cmd/server/setup.go +++ b/cmd/server/setup.go @@ -17,11 +17,6 @@ package main import ( "context" - "crypto" - "crypto/ed25519" - "crypto/rand" - "encoding/hex" - "errors" "fmt" "net/url" "os" @@ -49,7 +44,6 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/queue" "go.woodpecker-ci.org/woodpecker/v2/server/store" "go.woodpecker-ci.org/woodpecker/v2/server/store/datastore" - "go.woodpecker-ci.org/woodpecker/v2/server/store/types" "go.woodpecker-ci.org/woodpecker/v2/shared/addon" addonTypes "go.woodpecker-ci.org/woodpecker/v2/shared/addon/types" ) @@ -293,33 +287,6 @@ func setupMetrics(g *errgroup.Group, _store store.Store) { }) } -// setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for extensions) -func setupSignatureKeys(_store store.Store) (crypto.PrivateKey, crypto.PublicKey, error) { - privKeyID := "signature-private-key" - - privKey, err := _store.ServerConfigGet(privKeyID) - if errors.Is(err, types.RecordNotExist) { - _, privKey, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - return nil, nil, fmt.Errorf("failed to generate private key: %w", err) - } - err = _store.ServerConfigSet(privKeyID, hex.EncodeToString(privKey)) - if err != nil { - return nil, nil, fmt.Errorf("failed to store private key: %w", err) - } - log.Debug().Msg("created private key") - return privKey, privKey.Public(), nil - } else if err != nil { - return nil, nil, fmt.Errorf("failed to load private key: %w", err) - } - privKeyStr, err := hex.DecodeString(privKey) - if err != nil { - return nil, nil, fmt.Errorf("failed to decode private key: %w", err) - } - privateKey := ed25519.PrivateKey(privKeyStr) - return privateKey, privateKey.Public(), nil -} - func setupConfigService(c *cli.Context) (config.Extension, error) { addonExt, err := addon.Load[config.Extension](c.StringSlice("addons"), addonTypes.TypeConfigService) if err != nil { diff --git a/server/model/registry.go b/server/model/registry.go index 0c270756e1..e2bfe9ee9c 100644 --- a/server/model/registry.go +++ b/server/model/registry.go @@ -26,21 +26,6 @@ var ( errRegistryPasswordInvalid = errors.New("invalid registry password") ) -// RegistryService defines a service for managing registries. -type RegistryService interface { - RegistryFind(*Repo, string) (*Registry, error) - RegistryList(*Repo, *ListOptions) ([]*Registry, error) - RegistryCreate(*Repo, *Registry) error - RegistryUpdate(*Repo, *Registry) error - RegistryDelete(*Repo, string) error -} - -// ReadOnlyRegistryService defines a service for managing registries. -type ReadOnlyRegistryService interface { - RegistryFind(*Repo, string) (*Registry, error) - RegistryList(*Repo, *ListOptions) ([]*Registry, error) -} - // RegistryStore persists registry information to storage. type RegistryStore interface { RegistryFind(*Repo, string) (*Registry, error) diff --git a/server/model/repo.go b/server/model/repo.go index 34d50bb508..e2748a8c1c 100644 --- a/server/model/repo.go +++ b/server/model/repo.go @@ -49,6 +49,9 @@ type Repo struct { Perm *Perm `json:"-" xorm:"-"` CancelPreviousPipelineEvents []WebhookEvent `json:"cancel_previous_pipeline_events" xorm:"json 'cancel_previous_pipeline_events'"` NetrcOnlyTrusted bool `json:"netrc_only_trusted" xorm:"NOT NULL DEFAULT true 'netrc_only_trusted'"` + SecretEndpoint string `json:"secret_endpoint" xorm:"varchar(500) 'secret_endpoint'"` + RegistryEndpoint string `json:"registry_endpoint" xorm:"varchar(500) 'registry_endpoint'"` + ConfigEndpoint string `json:"config_endpoint" xorm:"varchar(500) 'config_endpoint'"` } // @name Repo // TableName return database table name for xorm @@ -114,6 +117,9 @@ type RepoPatch struct { AllowPull *bool `json:"allow_pr,omitempty"` CancelPreviousPipelineEvents *[]WebhookEvent `json:"cancel_previous_pipeline_events"` NetrcOnlyTrusted *bool `json:"netrc_only_trusted"` + ConfigEndpoint *string `json:"config_endpoint,omitempty"` + SecretEndpoint *string `json:"secret_endpoint,omitempty"` + RegistryEndpoint *string `json:"registry_endpoint,omitempty"` } // @name RepoPatch type ForgeRemoteID string diff --git a/server/model/secret.go b/server/model/secret.go index d5d97ca8a5..cde0f49b4a 100644 --- a/server/model/secret.go +++ b/server/model/secret.go @@ -29,29 +29,6 @@ var ( ErrSecretEventInvalid = errors.New("invalid secret event") ) -// SecretService defines a service for managing secrets. -type SecretService interface { - SecretListPipeline(*Repo, *Pipeline, *ListOptions) ([]*Secret, error) - // Repository secrets - SecretFind(*Repo, string) (*Secret, error) - SecretList(*Repo, *ListOptions) ([]*Secret, error) - SecretCreate(*Repo, *Secret) error - SecretUpdate(*Repo, *Secret) error - SecretDelete(*Repo, string) error - // Organization secrets - OrgSecretFind(int64, string) (*Secret, error) - OrgSecretList(int64, *ListOptions) ([]*Secret, error) - OrgSecretCreate(int64, *Secret) error - OrgSecretUpdate(int64, *Secret) error - OrgSecretDelete(int64, string) error - // Global secrets - GlobalSecretFind(string) (*Secret, error) - GlobalSecretList(*ListOptions) ([]*Secret, error) - GlobalSecretCreate(*Secret) error - GlobalSecretUpdate(*Secret) error - GlobalSecretDelete(string) error -} - // SecretStore persists secret information to storage. type SecretStore interface { SecretFind(*Repo, string) (*Secret, error) diff --git a/server/plugins/plugins.go b/server/plugins/plugins.go new file mode 100644 index 0000000000..f9976e8de1 --- /dev/null +++ b/server/plugins/plugins.go @@ -0,0 +1,119 @@ +package extensions + +import ( + "crypto" + "crypto/ed25519" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" + "go.woodpecker-ci.org/woodpecker/v2/server/forge" + "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/environments" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/registry" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secret" + "go.woodpecker-ci.org/woodpecker/v2/server/store" + "go.woodpecker-ci.org/woodpecker/v2/server/store/types" +) + +type Manager struct { + secrets secret.SecretExtension + registries registry.RegistryService + config config.Extension + environ environments.EnvironPlugin + signaturePrivateKey crypto.PrivateKey + signaturePublicKey crypto.PublicKey +} + +func NewManager(store store.Store, forge forge.Forge, c *cli.Context) (*Manager, error) { + signaturePrivateKey, signaturePublicKey, err := setupSignatureKeys(store) + if err != nil { + return nil, err + } + + return &Manager{ + signaturePrivateKey: signaturePrivateKey, + signaturePublicKey: signaturePublicKey, + secrets: secret.NewBuiltin(store), + registries: setupRegistryExtension(store, c.String("docker-config")), + config: config.NewCombined(forge, c.String("config-service-endpoint"), signaturePrivateKey), + environ: environments.Parse(c.StringSlice("environment")), + }, nil +} + +func setupRegistryExtension(store store.Store, dockerConfig string) registry.RegistryService { + if dockerConfig != "" { + return registry.NewCombined( + registry.NewBuiltin(store), + registry.NewFilesystem(dockerConfig), + ) + } + return registry.NewBuiltin(store) +} + +// setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for extensions) +func setupSignatureKeys(_store store.Store) (crypto.PrivateKey, crypto.PublicKey, error) { + privKeyID := "signature-private-key" + + privKey, err := _store.ServerConfigGet(privKeyID) + if errors.Is(err, types.RecordNotExist) { + _, privKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate private key: %w", err) + } + err = _store.ServerConfigSet(privKeyID, hex.EncodeToString(privKey)) + if err != nil { + return nil, nil, fmt.Errorf("failed to store private key: %w", err) + } + log.Debug().Msg("created private key") + return privKey, privKey.Public(), nil + } else if err != nil { + return nil, nil, fmt.Errorf("failed to load private key: %w", err) + } + privKeyStr, err := hex.DecodeString(privKey) + if err != nil { + return nil, nil, fmt.Errorf("failed to decode private key: %w", err) + } + privateKey := ed25519.PrivateKey(privKeyStr) + return privateKey, privateKey.Public(), nil +} + +func (e *Manager) SignaturePublicKey() crypto.PublicKey { + return e.signaturePublicKey +} + +func (e *Manager) SecretsFromRepo(repo *model.Repo) secret.SecretExtension { + if repo.SecretEndpoint != "" { + return secret.NewHTTP(repo.SecretEndpoint, e.signaturePrivateKey) + } + + return e.secrets +} + +func (e *Manager) RegistriesFromRepo(repo *model.Repo) registry.RegistryService { + if repo.SecretEndpoint != "" { + return registry.NewHTTP(repo.SecretEndpoint, e.signaturePrivateKey) + } + + return e.registries +} + +func (e *Manager) Config() config.Extension { + return e.config +} + +func (e *Manager) ConfigExtensionsFromRepo(repo *model.Repo) *config.HttpFetcher { + if repo.ConfigEndpoint != "" { + return config.NewHTTP(repo.ConfigEndpoint, e.signaturePrivateKey) + } + + return nil +} + +func (e *Manager) Environ() environments.EnvironExtension { + return e.environ +} diff --git a/server/plugins/registry/plugin.go b/server/plugins/registry/plugin.go new file mode 100644 index 0000000000..bd1c6077ee --- /dev/null +++ b/server/plugins/registry/plugin.go @@ -0,0 +1,18 @@ +package registry + +import "go.woodpecker-ci.org/woodpecker/v2/server/model" + +// RegistryService defines a service for managing registries. +type RegistryService interface { + RegistryFind(*model.Repo, string) (*model.Registry, error) + RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error) + RegistryCreate(*model.Repo, *model.Registry) error + RegistryUpdate(*model.Repo, *model.Registry) error + RegistryDelete(*model.Repo, string) error +} + +// ReadOnlyRegistryService defines a service for managing registries. +type ReadOnlyRegistryService interface { + RegistryFind(*model.Repo, string) (*model.Registry, error) + RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error) +} diff --git a/server/plugins/secrets/builtin.go b/server/plugins/secrets/builtin.go index bb2e40ffae..206835e086 100644 --- a/server/plugins/secrets/builtin.go +++ b/server/plugins/secrets/builtin.go @@ -26,7 +26,7 @@ type builtin struct { } // New returns a new local secret service. -func New(ctx context.Context, store model.SecretStore) model.SecretService { +func New(ctx context.Context, store model.SecretStore) SecretService { return &builtin{store: store, Context: ctx} } diff --git a/server/plugins/secrets/plugin.go b/server/plugins/secrets/plugin.go new file mode 100644 index 0000000000..6cd1443e49 --- /dev/null +++ b/server/plugins/secrets/plugin.go @@ -0,0 +1,26 @@ +package secrets + +import "go.woodpecker-ci.org/woodpecker/v2/server/model" + +// SecretService defines a service for managing secrets. +type SecretService interface { + SecretListPipeline(*model.Repo, *model.Pipeline, *model.ListOptions) ([]*model.Secret, error) + // Repository secrets + SecretFind(*model.Repo, string) (*model.Secret, error) + SecretList(*model.Repo, *model.ListOptions) ([]*model.Secret, error) + SecretCreate(*model.Repo, *model.Secret) error + SecretUpdate(*model.Repo, *model.Secret) error + SecretDelete(*model.Repo, string) error + // Organization secrets + OrgSecretFind(int64, string) (*model.Secret, error) + OrgSecretList(int64, *model.ListOptions) ([]*model.Secret, error) + OrgSecretCreate(int64, *model.Secret) error + OrgSecretUpdate(int64, *model.Secret) error + OrgSecretDelete(int64, string) error + // Global secrets + GlobalSecretFind(string) (*model.Secret, error) + GlobalSecretList(*model.ListOptions) ([]*model.Secret, error) + GlobalSecretCreate(*model.Secret) error + GlobalSecretUpdate(*model.Secret) error + GlobalSecretDelete(string) error +} From 69824f251d64358ae9dc3873ac5d89cf8f034cf8 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Wed, 7 Feb 2024 21:52:36 +0100 Subject: [PATCH 02/28] further improvements --- cmd/server/server.go | 34 +---- cmd/server/setup.go | 63 -------- server/config.go | 20 +-- server/model/environ.go | 5 - server/plugins/config/http.go | 2 +- .../config/{extension.go => service.go} | 2 +- server/plugins/environments/parse.go | 2 +- server/plugins/environments/service.go | 8 ++ server/plugins/manager.go | 76 ++++++++++ server/plugins/plugins.go | 119 --------------- server/plugins/registry/combine.go | 16 +-- server/plugins/registry/db.go | 22 +-- server/plugins/registry/filesystem.go | 8 +- .../registry/{plugin.go => service.go} | 8 +- server/plugins/secret/db.go | 133 +++++++++++++++++ .../builtin_test.go => secret/db_test.go} | 12 +- .../{secrets/plugin.go => secret/service.go} | 6 +- server/plugins/secrets/builtin.go | 136 ------------------ server/plugins/setup.go | 129 +++++++++++++++++ 19 files changed, 392 insertions(+), 409 deletions(-) rename server/plugins/config/{extension.go => service.go} (97%) create mode 100644 server/plugins/environments/service.go create mode 100644 server/plugins/manager.go delete mode 100644 server/plugins/plugins.go rename server/plugins/registry/{plugin.go => service.go} (70%) create mode 100644 server/plugins/secret/db.go rename server/plugins/{secrets/builtin_test.go => secret/db_test.go} (82%) rename server/plugins/{secrets/plugin.go => secret/service.go} (90%) delete mode 100644 server/plugins/secrets/builtin.go create mode 100644 server/plugins/setup.go diff --git a/cmd/server/server.go b/cmd/server/server.go index eb5b9f0350..4baaf094f2 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -42,6 +42,7 @@ import ( woodpeckerGrpcServer "go.woodpecker-ci.org/woodpecker/v2/server/grpc" "go.woodpecker-ci.org/woodpecker/v2/server/logging" "go.woodpecker-ci.org/woodpecker/v2/server/model" + // "go.woodpecker-ci.org/woodpecker/v2/server/plugins/encryption" // encryptedStore "go.woodpecker-ci.org/woodpecker/v2/server/plugins/encryption/wrapper/store" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/permissions" @@ -280,39 +281,8 @@ func setupEvilGlobals(c *cli.Context, v store.Store, f forge.Forge) error { server.Config.Services.Queue = setupQueue(c, v) server.Config.Services.Logs = logging.New() server.Config.Services.Pubsub = pubsub.New() - var err error - server.Config.Services.Registries, err = setupRegistryService(c, v) - if err != nil { - return err - } - - // TODO(1544): fix encrypted store - // // encryption - // encryptedSecretStore := encryptedStore.NewSecretStore(v) - // err := encryption.Encryption(c, v).WithClient(encryptedSecretStore).Build() - // if err != nil { - // log.Fatal().Err(err).Msg("could not create encryption service") - // } - // server.Config.Services.Secrets = setupSecretService(c, encryptedSecretStore) - server.Config.Services.Secrets, err = setupSecretService(c, v) - if err != nil { - return err - } - server.Config.Services.Environ, err = setupEnvironService(c, v) - if err != nil { - return err - } server.Config.Services.Membership = setupMembershipService(c, f) - - server.Config.Services.SignaturePrivateKey, server.Config.Services.SignaturePublicKey, err = setupSignatureKeys(v) - if err != nil { - return err - } - - server.Config.Services.ConfigService, err = setupConfigService(c) - if err != nil { - return err - } + // TODO: setup addon manager // authentication server.Config.Pipeline.AuthenticatePublicRepos = c.Bool("authenticate-public-repos") diff --git a/cmd/server/setup.go b/cmd/server/setup.go index 9cfefe1b01..63479c901f 100644 --- a/cmd/server/setup.go +++ b/cmd/server/setup.go @@ -36,11 +36,6 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/forge/gitea" "go.woodpecker-ci.org/woodpecker/v2/server/forge/github" "go.woodpecker-ci.org/woodpecker/v2/server/forge/gitlab" - "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/environments" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/registry" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secrets" "go.woodpecker-ci.org/woodpecker/v2/server/queue" "go.woodpecker-ci.org/woodpecker/v2/server/store" "go.woodpecker-ci.org/woodpecker/v2/server/store/datastore" @@ -105,48 +100,6 @@ func setupQueue(c *cli.Context, s store.Store) queue.Queue { return queue.WithTaskStore(queue.New(c.Context), s) } -func setupSecretService(c *cli.Context, s model.SecretStore) (model.SecretService, error) { - addonService, err := addon.Load[model.SecretService](c.StringSlice("addons"), addonTypes.TypeSecretService) - if err != nil { - return nil, err - } - if addonService != nil { - return addonService.Value, nil - } - - return secrets.New(c.Context, s), nil -} - -func setupRegistryService(c *cli.Context, s store.Store) (model.RegistryService, error) { - addonService, err := addon.Load[model.RegistryService](c.StringSlice("addons"), addonTypes.TypeRegistryService) - if err != nil { - return nil, err - } - if addonService != nil { - return addonService.Value, nil - } - - if c.String("docker-config") != "" { - return registry.Combined( - registry.New(s), - registry.Filesystem(c.String("docker-config")), - ), nil - } - return registry.New(s), nil -} - -func setupEnvironService(c *cli.Context, _ store.Store) (model.EnvironService, error) { - addonService, err := addon.Load[model.EnvironService](c.StringSlice("addons"), addonTypes.TypeEnvironmentService) - if err != nil { - return nil, err - } - if addonService != nil { - return addonService.Value, nil - } - - return environments.Parse(c.StringSlice("environment")), nil -} - func setupMembershipService(_ *cli.Context, r forge.Forge) cache.MembershipService { return cache.NewMembershipService(r) } @@ -286,19 +239,3 @@ func setupMetrics(g *errgroup.Group, _store store.Store) { } }) } - -func setupConfigService(c *cli.Context) (config.Extension, error) { - addonExt, err := addon.Load[config.Extension](c.StringSlice("addons"), addonTypes.TypeConfigService) - if err != nil { - return nil, err - } - if addonExt != nil { - return addonExt.Value, nil - } - - if endpoint := c.String("config-service-endpoint"); endpoint != "" { - return config.NewHTTP(endpoint, server.Config.Services.SignaturePrivateKey), nil - } - - return nil, nil -} diff --git a/server/config.go b/server/config.go index a06f42b164..31e98a1533 100644 --- a/server/config.go +++ b/server/config.go @@ -18,14 +18,12 @@ package server import ( - "crypto" "time" "go.woodpecker-ci.org/woodpecker/v2/server/cache" "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/logging" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/pubsub" "go.woodpecker-ci.org/woodpecker/v2/server/queue" @@ -33,18 +31,12 @@ import ( var Config = struct { Services struct { - Pubsub *pubsub.Publisher - Queue queue.Queue - Logs logging.Log - Secrets model.SecretService - Registries model.RegistryService - Environ model.EnvironService - Forge forge.Forge - Timeout time.Duration - Membership cache.MembershipService - ConfigService config.Extension - SignaturePrivateKey crypto.PrivateKey - SignaturePublicKey crypto.PublicKey + Pubsub *pubsub.Publisher + Queue queue.Queue + Logs logging.Log + Forge forge.Forge + Timeout time.Duration + Membership cache.MembershipService } Storage struct { // Users model.UserStore diff --git a/server/model/environ.go b/server/model/environ.go index 4c70db4c61..87581ecb41 100644 --- a/server/model/environ.go +++ b/server/model/environ.go @@ -24,11 +24,6 @@ var ( errEnvironValueInvalid = errors.New("invalid Environment Variable Value") ) -// EnvironService defines a service for managing environment variables. -type EnvironService interface { - EnvironList(*Repo) ([]*Environ, error) -} - // EnvironStore persists environment information to storage. type EnvironStore interface { EnvironList(*Repo) ([]*Environ, error) diff --git a/server/plugins/config/http.go b/server/plugins/config/http.go index 5dbc2909bc..d548cdb2c5 100644 --- a/server/plugins/config/http.go +++ b/server/plugins/config/http.go @@ -46,7 +46,7 @@ type responseStructure struct { Configs []config `json:"configs"` } -func NewHTTP(endpoint string, privateKey crypto.PrivateKey) Extension { +func NewHTTP(endpoint string, privateKey crypto.PrivateKey) Service { return &http{endpoint, privateKey} } diff --git a/server/plugins/config/extension.go b/server/plugins/config/service.go similarity index 97% rename from server/plugins/config/extension.go rename to server/plugins/config/service.go index 9b9778af60..326cb50a61 100644 --- a/server/plugins/config/extension.go +++ b/server/plugins/config/service.go @@ -21,6 +21,6 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/model" ) -type Extension interface { +type Service interface { FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge_types.FileMeta, netrc *model.Netrc) (configData []*forge_types.FileMeta, useOld bool, err error) } diff --git a/server/plugins/environments/parse.go b/server/plugins/environments/parse.go index b06294c2e6..a4990c3450 100644 --- a/server/plugins/environments/parse.go +++ b/server/plugins/environments/parse.go @@ -27,7 +27,7 @@ type builtin struct { } // Parse returns a model.EnvironService based on a string slice where key and value are separated by a ":" delimiter. -func Parse(params []string) model.EnvironService { +func Parse(params []string) Service { var globals []*model.Environ for _, item := range params { diff --git a/server/plugins/environments/service.go b/server/plugins/environments/service.go new file mode 100644 index 0000000000..1d9ef80b92 --- /dev/null +++ b/server/plugins/environments/service.go @@ -0,0 +1,8 @@ +package environments + +import "go.woodpecker-ci.org/woodpecker/v2/server/model" + +// Service defines a service for managing environment variables. +type Service interface { + EnvironList(*model.Repo) ([]*model.Environ, error) +} diff --git a/server/plugins/manager.go b/server/plugins/manager.go new file mode 100644 index 0000000000..410e241b1e --- /dev/null +++ b/server/plugins/manager.go @@ -0,0 +1,76 @@ +package extensions + +import ( + "crypto" + + "github.com/urfave/cli/v2" + "go.woodpecker-ci.org/woodpecker/v2/server/forge" + "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/environments" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/registry" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secret" + "go.woodpecker-ci.org/woodpecker/v2/server/store" +) + +type Manager struct { + secret secret.Service + registry registry.Service + config config.Service + environ environments.Service + signaturePrivateKey crypto.PrivateKey + signaturePublicKey crypto.PublicKey +} + +func NewManager(store store.Store, forge forge.Forge, c *cli.Context) (*Manager, error) { + signaturePrivateKey, signaturePublicKey, err := setupSignatureKeys(store) + if err != nil { + return nil, err + } + + config, err := setupConfigService(c, store, signaturePrivateKey) + if err != nil { + return nil, err + } + + return &Manager{ + signaturePrivateKey: signaturePrivateKey, + signaturePublicKey: signaturePublicKey, + secret: setupSecretExtension(store), + registry: setupRegistryExtension(store, c.String("docker-config")), + config: config, + environ: environments.Parse(c.StringSlice("environment")), + }, nil +} + +func (e *Manager) SignaturePublicKey() crypto.PublicKey { + return e.signaturePublicKey +} + +func (e *Manager) SecretAddonFromRepo(repo *model.Repo) secret.Service { + // if repo.SecretEndpoint != "" { + // return secret.NewHTTP(repo.SecretEndpoint, e.signaturePrivateKey) + // } + + return e.secret +} + +func (e *Manager) RegistryAddonFromRepo(repo *model.Repo) registry.Service { + // if repo.SecretEndpoint != "" { + // return registry.NewHTTP(repo.SecretEndpoint, e.signaturePrivateKey) + // } + + return e.registry +} + +func (e *Manager) ConfigAddonFromRepo(repo *model.Repo) config.Service { + // if repo.ConfigEndpoint != "" { + // return config.NewHTTP(repo.ConfigEndpoint, e.signaturePrivateKey) + // } + + return e.config +} + +func (e *Manager) EnvironAddonFromRepo(repo *model.Repo) environments.Service { + return e.environ +} diff --git a/server/plugins/plugins.go b/server/plugins/plugins.go deleted file mode 100644 index f9976e8de1..0000000000 --- a/server/plugins/plugins.go +++ /dev/null @@ -1,119 +0,0 @@ -package extensions - -import ( - "crypto" - "crypto/ed25519" - "crypto/rand" - "encoding/hex" - "errors" - "fmt" - - "github.com/rs/zerolog/log" - "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/forge" - "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/environments" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/registry" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secret" - "go.woodpecker-ci.org/woodpecker/v2/server/store" - "go.woodpecker-ci.org/woodpecker/v2/server/store/types" -) - -type Manager struct { - secrets secret.SecretExtension - registries registry.RegistryService - config config.Extension - environ environments.EnvironPlugin - signaturePrivateKey crypto.PrivateKey - signaturePublicKey crypto.PublicKey -} - -func NewManager(store store.Store, forge forge.Forge, c *cli.Context) (*Manager, error) { - signaturePrivateKey, signaturePublicKey, err := setupSignatureKeys(store) - if err != nil { - return nil, err - } - - return &Manager{ - signaturePrivateKey: signaturePrivateKey, - signaturePublicKey: signaturePublicKey, - secrets: secret.NewBuiltin(store), - registries: setupRegistryExtension(store, c.String("docker-config")), - config: config.NewCombined(forge, c.String("config-service-endpoint"), signaturePrivateKey), - environ: environments.Parse(c.StringSlice("environment")), - }, nil -} - -func setupRegistryExtension(store store.Store, dockerConfig string) registry.RegistryService { - if dockerConfig != "" { - return registry.NewCombined( - registry.NewBuiltin(store), - registry.NewFilesystem(dockerConfig), - ) - } - return registry.NewBuiltin(store) -} - -// setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for extensions) -func setupSignatureKeys(_store store.Store) (crypto.PrivateKey, crypto.PublicKey, error) { - privKeyID := "signature-private-key" - - privKey, err := _store.ServerConfigGet(privKeyID) - if errors.Is(err, types.RecordNotExist) { - _, privKey, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - return nil, nil, fmt.Errorf("failed to generate private key: %w", err) - } - err = _store.ServerConfigSet(privKeyID, hex.EncodeToString(privKey)) - if err != nil { - return nil, nil, fmt.Errorf("failed to store private key: %w", err) - } - log.Debug().Msg("created private key") - return privKey, privKey.Public(), nil - } else if err != nil { - return nil, nil, fmt.Errorf("failed to load private key: %w", err) - } - privKeyStr, err := hex.DecodeString(privKey) - if err != nil { - return nil, nil, fmt.Errorf("failed to decode private key: %w", err) - } - privateKey := ed25519.PrivateKey(privKeyStr) - return privateKey, privateKey.Public(), nil -} - -func (e *Manager) SignaturePublicKey() crypto.PublicKey { - return e.signaturePublicKey -} - -func (e *Manager) SecretsFromRepo(repo *model.Repo) secret.SecretExtension { - if repo.SecretEndpoint != "" { - return secret.NewHTTP(repo.SecretEndpoint, e.signaturePrivateKey) - } - - return e.secrets -} - -func (e *Manager) RegistriesFromRepo(repo *model.Repo) registry.RegistryService { - if repo.SecretEndpoint != "" { - return registry.NewHTTP(repo.SecretEndpoint, e.signaturePrivateKey) - } - - return e.registries -} - -func (e *Manager) Config() config.Extension { - return e.config -} - -func (e *Manager) ConfigExtensionsFromRepo(repo *model.Repo) *config.HttpFetcher { - if repo.ConfigEndpoint != "" { - return config.NewHTTP(repo.ConfigEndpoint, e.signaturePrivateKey) - } - - return nil -} - -func (e *Manager) Environ() environments.EnvironExtension { - return e.environ -} diff --git a/server/plugins/registry/combine.go b/server/plugins/registry/combine.go index d4f6c7e45c..4f52bca1a5 100644 --- a/server/plugins/registry/combine.go +++ b/server/plugins/registry/combine.go @@ -19,11 +19,11 @@ import ( ) type combined struct { - registries []model.ReadOnlyRegistryService - dbRegistry model.RegistryService + registries []ReadOnlyService + dbRegistry Service } -func Combined(dbRegistry model.RegistryService, registries ...model.ReadOnlyRegistryService) model.RegistryService { +func NewCombined(dbRegistry Service, registries ...ReadOnlyService) Service { registries = append(registries, dbRegistry) return &combined{ registries: registries, @@ -31,7 +31,7 @@ func Combined(dbRegistry model.RegistryService, registries ...model.ReadOnlyRegi } } -func (c combined) RegistryFind(repo *model.Repo, name string) (*model.Registry, error) { +func (c *combined) RegistryFind(repo *model.Repo, name string) (*model.Registry, error) { for _, registry := range c.registries { res, err := registry.RegistryFind(repo, name) if err != nil { @@ -44,7 +44,7 @@ func (c combined) RegistryFind(repo *model.Repo, name string) (*model.Registry, return nil, nil } -func (c combined) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { +func (c *combined) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { var registries []*model.Registry for _, registry := range c.registries { list, err := registry.RegistryList(repo, &model.ListOptions{All: true}) @@ -56,14 +56,14 @@ func (c combined) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model return model.ApplyPagination(p, registries), nil } -func (c combined) RegistryCreate(repo *model.Repo, registry *model.Registry) error { +func (c *combined) RegistryCreate(repo *model.Repo, registry *model.Registry) error { return c.dbRegistry.RegistryCreate(repo, registry) } -func (c combined) RegistryUpdate(repo *model.Repo, registry *model.Registry) error { +func (c *combined) RegistryUpdate(repo *model.Repo, registry *model.Registry) error { return c.dbRegistry.RegistryUpdate(repo, registry) } -func (c combined) RegistryDelete(repo *model.Repo, name string) error { +func (c *combined) RegistryDelete(repo *model.Repo, name string) error { return c.dbRegistry.RegistryDelete(repo, name) } diff --git a/server/plugins/registry/db.go b/server/plugins/registry/db.go index d36b3aa3e5..5e7339557f 100644 --- a/server/plugins/registry/db.go +++ b/server/plugins/registry/db.go @@ -23,26 +23,26 @@ type db struct { } // New returns a new local registry service. -func New(store model.RegistryStore) model.RegistryService { +func NewDB(store model.RegistryStore) Service { return &db{store} } -func (b *db) RegistryFind(repo *model.Repo, name string) (*model.Registry, error) { - return b.store.RegistryFind(repo, name) +func (d *db) RegistryFind(repo *model.Repo, name string) (*model.Registry, error) { + return d.store.RegistryFind(repo, name) } -func (b *db) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { - return b.store.RegistryList(repo, p) +func (d *db) RegistryList(repo *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { + return d.store.RegistryList(repo, p) } -func (b *db) RegistryCreate(_ *model.Repo, in *model.Registry) error { - return b.store.RegistryCreate(in) +func (d *db) RegistryCreate(_ *model.Repo, in *model.Registry) error { + return d.store.RegistryCreate(in) } -func (b *db) RegistryUpdate(_ *model.Repo, in *model.Registry) error { - return b.store.RegistryUpdate(in) +func (d *db) RegistryUpdate(_ *model.Repo, in *model.Registry) error { + return d.store.RegistryUpdate(in) } -func (b *db) RegistryDelete(repo *model.Repo, addr string) error { - return b.store.RegistryDelete(repo, addr) +func (d *db) RegistryDelete(repo *model.Repo, addr string) error { + return d.store.RegistryDelete(repo, addr) } diff --git a/server/plugins/registry/filesystem.go b/server/plugins/registry/filesystem.go index db9e24a0d3..4d98818cc0 100644 --- a/server/plugins/registry/filesystem.go +++ b/server/plugins/registry/filesystem.go @@ -31,7 +31,7 @@ type filesystem struct { path string } -func Filesystem(path string) model.ReadOnlyRegistryService { +func NewFilesystem(path string) ReadOnlyService { return &filesystem{path} } @@ -85,12 +85,12 @@ func parseDockerConfig(path string) ([]*model.Registry, error) { return auths, nil } -func (b *filesystem) RegistryFind(*model.Repo, string) (*model.Registry, error) { +func (f *filesystem) RegistryFind(*model.Repo, string) (*model.Registry, error) { return nil, nil } -func (b *filesystem) RegistryList(_ *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { - regs, err := parseDockerConfig(b.path) +func (f *filesystem) RegistryList(_ *model.Repo, p *model.ListOptions) ([]*model.Registry, error) { + regs, err := parseDockerConfig(f.path) if err != nil { return nil, err } diff --git a/server/plugins/registry/plugin.go b/server/plugins/registry/service.go similarity index 70% rename from server/plugins/registry/plugin.go rename to server/plugins/registry/service.go index bd1c6077ee..03f269a82c 100644 --- a/server/plugins/registry/plugin.go +++ b/server/plugins/registry/service.go @@ -2,8 +2,8 @@ package registry import "go.woodpecker-ci.org/woodpecker/v2/server/model" -// RegistryService defines a service for managing registries. -type RegistryService interface { +// Service defines a service for managing registries. +type Service interface { RegistryFind(*model.Repo, string) (*model.Registry, error) RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error) RegistryCreate(*model.Repo, *model.Registry) error @@ -11,8 +11,8 @@ type RegistryService interface { RegistryDelete(*model.Repo, string) error } -// ReadOnlyRegistryService defines a service for managing registries. -type ReadOnlyRegistryService interface { +// ReadOnlyService defines a service for managing registries. +type ReadOnlyService interface { RegistryFind(*model.Repo, string) (*model.Registry, error) RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error) } diff --git a/server/plugins/secret/db.go b/server/plugins/secret/db.go new file mode 100644 index 0000000000..b3d3202618 --- /dev/null +++ b/server/plugins/secret/db.go @@ -0,0 +1,133 @@ +// Copyright 2022 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package secret + +import ( + "go.woodpecker-ci.org/woodpecker/v2/server/model" +) + +type db struct { + store model.SecretStore +} + +// NewDB returns a new local secret service. +func NewDB(store model.SecretStore) Service { + return &db{store: store} +} + +func (d *db) SecretFind(repo *model.Repo, name string) (*model.Secret, error) { + return d.store.SecretFind(repo, name) +} + +func (d *db) SecretList(repo *model.Repo, p *model.ListOptions) ([]*model.Secret, error) { + return d.store.SecretList(repo, false, p) +} + +func (d *db) SecretListPipeline(repo *model.Repo, _ *model.Pipeline, p *model.ListOptions) ([]*model.Secret, error) { + s, err := d.store.SecretList(repo, true, p) + if err != nil { + return nil, err + } + + // Return only secrets with unique name + // Priority order in case of duplicate names are repository, user/organization, global + secrets := make([]*model.Secret, 0, len(s)) + uniq := make(map[string]struct{}) + for _, condition := range []struct { + IsRepository bool + IsOrganization bool + IsGlobal bool + }{ + {IsRepository: true}, + {IsOrganization: true}, + {IsGlobal: true}, + } { + for _, secret := range s { + if secret.IsRepository() != condition.IsRepository || secret.IsOrganization() != condition.IsOrganization || secret.IsGlobal() != condition.IsGlobal { + continue + } + if _, ok := uniq[secret.Name]; ok { + continue + } + uniq[secret.Name] = struct{}{} + secrets = append(secrets, secret) + } + } + return secrets, nil +} + +func (d *db) SecretCreate(_ *model.Repo, in *model.Secret) error { + return d.store.SecretCreate(in) +} + +func (d *db) SecretUpdate(_ *model.Repo, in *model.Secret) error { + return d.store.SecretUpdate(in) +} + +func (d *db) SecretDelete(repo *model.Repo, name string) error { + secret, err := d.store.SecretFind(repo, name) + if err != nil { + return err + } + return d.store.SecretDelete(secret) +} + +func (d *db) OrgSecretFind(owner int64, name string) (*model.Secret, error) { + return d.store.OrgSecretFind(owner, name) +} + +func (d *db) OrgSecretList(owner int64, p *model.ListOptions) ([]*model.Secret, error) { + return d.store.OrgSecretList(owner, p) +} + +func (d *db) OrgSecretCreate(_ int64, in *model.Secret) error { + return d.store.SecretCreate(in) +} + +func (d *db) OrgSecretUpdate(_ int64, in *model.Secret) error { + return d.store.SecretUpdate(in) +} + +func (d *db) OrgSecretDelete(owner int64, name string) error { + secret, err := d.store.OrgSecretFind(owner, name) + if err != nil { + return err + } + return d.store.SecretDelete(secret) +} + +func (d *db) GlobalSecretFind(owner string) (*model.Secret, error) { + return d.store.GlobalSecretFind(owner) +} + +func (d *db) GlobalSecretList(p *model.ListOptions) ([]*model.Secret, error) { + return d.store.GlobalSecretList(p) +} + +func (d *db) GlobalSecretCreate(in *model.Secret) error { + return d.store.SecretCreate(in) +} + +func (d *db) GlobalSecretUpdate(in *model.Secret) error { + return d.store.SecretUpdate(in) +} + +func (d *db) GlobalSecretDelete(name string) error { + secret, err := d.store.GlobalSecretFind(name) + if err != nil { + return err + } + return d.store.SecretDelete(secret) +} diff --git a/server/plugins/secrets/builtin_test.go b/server/plugins/secret/db_test.go similarity index 82% rename from server/plugins/secrets/builtin_test.go rename to server/plugins/secret/db_test.go index 6c35aebfd7..979d4124ed 100644 --- a/server/plugins/secrets/builtin_test.go +++ b/server/plugins/secret/db_test.go @@ -12,23 +12,21 @@ // See the License for the specific language governing permissions and // limitations under the License. -package secrets_test +package secret_test import ( - "context" "testing" "github.com/franela/goblin" "github.com/stretchr/testify/mock" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secrets" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secret" mocks_store "go.woodpecker-ci.org/woodpecker/v2/server/store/mocks" ) func TestSecretListPipeline(t *testing.T) { g := goblin.Goblin(t) - ctx := context.Background() mockStore := mocks_store.NewStore(t) // global secret @@ -66,7 +64,7 @@ func TestSecretListPipeline(t *testing.T) { repoSecret, }, nil) - s, err := secrets.New(ctx, mockStore).SecretListPipeline(&model.Repo{}, &model.Pipeline{}, &model.ListOptions{}) + s, err := secret.NewDB(mockStore).SecretListPipeline(&model.Repo{}, &model.Pipeline{}, &model.ListOptions{}) g.Assert(err).IsNil() g.Assert(len(s)).Equal(1) @@ -79,7 +77,7 @@ func TestSecretListPipeline(t *testing.T) { orgSecret, }, nil) - s, err := secrets.New(ctx, mockStore).SecretListPipeline(&model.Repo{}, &model.Pipeline{}, &model.ListOptions{}) + s, err := secret.NewDB(mockStore).SecretListPipeline(&model.Repo{}, &model.Pipeline{}, &model.ListOptions{}) g.Assert(err).IsNil() g.Assert(len(s)).Equal(1) @@ -91,7 +89,7 @@ func TestSecretListPipeline(t *testing.T) { globalSecret, }, nil) - s, err := secrets.New(ctx, mockStore).SecretListPipeline(&model.Repo{}, &model.Pipeline{}, &model.ListOptions{}) + s, err := secret.NewDB(mockStore).SecretListPipeline(&model.Repo{}, &model.Pipeline{}, &model.ListOptions{}) g.Assert(err).IsNil() g.Assert(len(s)).Equal(1) diff --git a/server/plugins/secrets/plugin.go b/server/plugins/secret/service.go similarity index 90% rename from server/plugins/secrets/plugin.go rename to server/plugins/secret/service.go index 6cd1443e49..d8a9c3bc72 100644 --- a/server/plugins/secrets/plugin.go +++ b/server/plugins/secret/service.go @@ -1,9 +1,9 @@ -package secrets +package secret import "go.woodpecker-ci.org/woodpecker/v2/server/model" -// SecretService defines a service for managing secrets. -type SecretService interface { +// Service defines a service for managing secrets. +type Service interface { SecretListPipeline(*model.Repo, *model.Pipeline, *model.ListOptions) ([]*model.Secret, error) // Repository secrets SecretFind(*model.Repo, string) (*model.Secret, error) diff --git a/server/plugins/secrets/builtin.go b/server/plugins/secrets/builtin.go deleted file mode 100644 index 206835e086..0000000000 --- a/server/plugins/secrets/builtin.go +++ /dev/null @@ -1,136 +0,0 @@ -// Copyright 2022 Woodpecker Authors -// -// Licensed under the Apache License, Version 2.0 (the "License"); -// you may not use this file except in compliance with the License. -// You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, software -// distributed under the License is distributed on an "AS IS" BASIS, -// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -// See the License for the specific language governing permissions and -// limitations under the License. - -package secrets - -import ( - "context" - - "go.woodpecker-ci.org/woodpecker/v2/server/model" -) - -type builtin struct { - context.Context - store model.SecretStore -} - -// New returns a new local secret service. -func New(ctx context.Context, store model.SecretStore) SecretService { - return &builtin{store: store, Context: ctx} -} - -func (b *builtin) SecretFind(repo *model.Repo, name string) (*model.Secret, error) { - return b.store.SecretFind(repo, name) -} - -func (b *builtin) SecretList(repo *model.Repo, p *model.ListOptions) ([]*model.Secret, error) { - return b.store.SecretList(repo, false, p) -} - -func (b *builtin) SecretListPipeline(repo *model.Repo, _ *model.Pipeline, p *model.ListOptions) ([]*model.Secret, error) { - s, err := b.store.SecretList(repo, true, p) - if err != nil { - return nil, err - } - - // Return only secrets with unique name - // Priority order in case of duplicate names are repository, user/organization, global - secrets := make([]*model.Secret, 0, len(s)) - uniq := make(map[string]struct{}) - for _, condition := range []struct { - IsRepository bool - IsOrganization bool - IsGlobal bool - }{ - {IsRepository: true}, - {IsOrganization: true}, - {IsGlobal: true}, - } { - for _, secret := range s { - if secret.IsRepository() != condition.IsRepository || secret.IsOrganization() != condition.IsOrganization || secret.IsGlobal() != condition.IsGlobal { - continue - } - if _, ok := uniq[secret.Name]; ok { - continue - } - uniq[secret.Name] = struct{}{} - secrets = append(secrets, secret) - } - } - return secrets, nil -} - -func (b *builtin) SecretCreate(_ *model.Repo, in *model.Secret) error { - return b.store.SecretCreate(in) -} - -func (b *builtin) SecretUpdate(_ *model.Repo, in *model.Secret) error { - return b.store.SecretUpdate(in) -} - -func (b *builtin) SecretDelete(repo *model.Repo, name string) error { - secret, err := b.store.SecretFind(repo, name) - if err != nil { - return err - } - return b.store.SecretDelete(secret) -} - -func (b *builtin) OrgSecretFind(owner int64, name string) (*model.Secret, error) { - return b.store.OrgSecretFind(owner, name) -} - -func (b *builtin) OrgSecretList(owner int64, p *model.ListOptions) ([]*model.Secret, error) { - return b.store.OrgSecretList(owner, p) -} - -func (b *builtin) OrgSecretCreate(_ int64, in *model.Secret) error { - return b.store.SecretCreate(in) -} - -func (b *builtin) OrgSecretUpdate(_ int64, in *model.Secret) error { - return b.store.SecretUpdate(in) -} - -func (b *builtin) OrgSecretDelete(owner int64, name string) error { - secret, err := b.store.OrgSecretFind(owner, name) - if err != nil { - return err - } - return b.store.SecretDelete(secret) -} - -func (b *builtin) GlobalSecretFind(owner string) (*model.Secret, error) { - return b.store.GlobalSecretFind(owner) -} - -func (b *builtin) GlobalSecretList(p *model.ListOptions) ([]*model.Secret, error) { - return b.store.GlobalSecretList(p) -} - -func (b *builtin) GlobalSecretCreate(in *model.Secret) error { - return b.store.SecretCreate(in) -} - -func (b *builtin) GlobalSecretUpdate(in *model.Secret) error { - return b.store.SecretUpdate(in) -} - -func (b *builtin) GlobalSecretDelete(name string) error { - secret, err := b.store.GlobalSecretFind(name) - if err != nil { - return err - } - return b.store.SecretDelete(secret) -} diff --git a/server/plugins/setup.go b/server/plugins/setup.go new file mode 100644 index 0000000000..045e8163ab --- /dev/null +++ b/server/plugins/setup.go @@ -0,0 +1,129 @@ +package extensions + +import ( + "crypto" + "crypto/ed25519" + "crypto/rand" + "encoding/hex" + "errors" + "fmt" + + "github.com/rs/zerolog/log" + "github.com/urfave/cli/v2" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/registry" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secret" + "go.woodpecker-ci.org/woodpecker/v2/server/store" + "go.woodpecker-ci.org/woodpecker/v2/server/store/types" + "go.woodpecker-ci.org/woodpecker/v2/shared/addon" + addonTypes "go.woodpecker-ci.org/woodpecker/v2/shared/addon/types" +) + +func setupRegistryExtension(store store.Store, dockerConfig string) registry.Service { + if dockerConfig != "" { + return registry.NewCombined( + registry.NewDB(store), + registry.NewFilesystem(dockerConfig), + ) + } + + return registry.NewDB(store) +} + +func setupSecretExtension(store store.Store) secret.Service { + // TODO(1544): fix encrypted store + // // encryption + // encryptedSecretStore := encryptedStore.NewSecretStore(v) + // err := encryption.Encryption(c, v).WithClient(encryptedSecretStore).Build() + // if err != nil { + // log.Fatal().Err(err).Msg("could not create encryption service") + // } + // server.Config.Services.Secrets = setupSecretService(c, encryptedSecretStore) + + return secret.NewDB(store) +} + +func setupConfigService(c *cli.Context, store store.Store, privateSignatureKey crypto.PrivateKey) (config.Service, error) { + addonExt, err := addon.Load[config.Service](c.StringSlice("addons"), addonTypes.TypeConfigService) + if err != nil { + return nil, err + } + if addonExt != nil { + return addonExt.Value, nil + } + + if endpoint := c.String("config-service-endpoint"); endpoint != "" { + return config.NewHTTP(endpoint, privateSignatureKey), nil + } + + return nil, nil +} + +// setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for extensions) +func setupSignatureKeys(_store store.Store) (crypto.PrivateKey, crypto.PublicKey, error) { + privKeyID := "signature-private-key" + + privKey, err := _store.ServerConfigGet(privKeyID) + if errors.Is(err, types.RecordNotExist) { + _, privKey, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + return nil, nil, fmt.Errorf("failed to generate private key: %w", err) + } + err = _store.ServerConfigSet(privKeyID, hex.EncodeToString(privKey)) + if err != nil { + return nil, nil, fmt.Errorf("failed to store private key: %w", err) + } + log.Debug().Msg("created private key") + return privKey, privKey.Public(), nil + } else if err != nil { + return nil, nil, fmt.Errorf("failed to load private key: %w", err) + } + privKeyStr, err := hex.DecodeString(privKey) + if err != nil { + return nil, nil, fmt.Errorf("failed to decode private key: %w", err) + } + privateKey := ed25519.PrivateKey(privKeyStr) + return privateKey, privateKey.Public(), nil +} + +// func setupSecretService(c *cli.Context, s model.SecretStore) (model.SecretService, error) { +// addonService, err := addon.Load[model.SecretService](c.StringSlice("addons"), addonTypes.TypeSecretService) +// if err != nil { +// return nil, err +// } +// if addonService != nil { +// return addonService.Value, nil +// } + +// return secrets.New(c.Context, s), nil +// } + +// func setupRegistryService(c *cli.Context, s store.Store) (model.RegistryService, error) { +// addonService, err := addon.Load[model.RegistryService](c.StringSlice("addons"), addonTypes.TypeRegistryService) +// if err != nil { +// return nil, err +// } +// if addonService != nil { +// return addonService.Value, nil +// } + +// if c.String("docker-config") != "" { +// return registry.Combined( +// registry.New(s), +// registry.Filesystem(c.String("docker-config")), +// ), nil +// } +// return registry.New(s), nil +// } + +// func setupEnvironService(c *cli.Context, _ store.Store) (model.EnvironService, error) { +// addonService, err := addon.Load[model.EnvironService](c.StringSlice("addons"), addonTypes.TypeEnvironmentService) +// if err != nil { +// return nil, err +// } +// if addonService != nil { +// return addonService.Value, nil +// } + +// return environments.Parse(c.StringSlice("environment")), nil +// } From ba2b536ce1740d777fb133c223107d6bc3b0caf6 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Wed, 7 Feb 2024 23:05:01 +0100 Subject: [PATCH 03/28] improve logic --- cmd/server/server.go | 12 ++- server/api/global_secret.go | 19 ++-- server/api/org_secret.go | 19 ++-- server/api/registry.go | 32 ++++--- server/api/repo_secret.go | 32 ++++--- server/api/signature_public_key.go | 2 +- server/config.go | 2 + server/pipeline/create.go | 6 +- server/pipeline/items.go | 12 ++- server/pipeline/restart.go | 20 ++-- .../config/forge.go} | 95 ++++++++++--------- .../config/forge_test.go} | 7 +- server/plugins/config/http.go | 44 ++++----- server/plugins/config/service.go | 11 ++- server/plugins/manager.go | 34 +++---- server/plugins/setup.go | 2 +- 16 files changed, 188 insertions(+), 161 deletions(-) rename server/{forge/configFetcher.go => plugins/config/forge.go} (69%) rename server/{forge/configFetcher_test.go => plugins/config/forge_test.go} (98%) diff --git a/cmd/server/server.go b/cmd/server/server.go index 4baaf094f2..ce093f6a4c 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -42,6 +42,7 @@ import ( woodpeckerGrpcServer "go.woodpecker-ci.org/woodpecker/v2/server/grpc" "go.woodpecker-ci.org/woodpecker/v2/server/logging" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins" // "go.woodpecker-ci.org/woodpecker/v2/server/plugins/encryption" // encryptedStore "go.woodpecker-ci.org/woodpecker/v2/server/plugins/encryption/wrapper/store" @@ -272,17 +273,22 @@ func run(c *cli.Context) error { return g.Wait() } -func setupEvilGlobals(c *cli.Context, v store.Store, f forge.Forge) error { +func setupEvilGlobals(c *cli.Context, s store.Store, f forge.Forge) error { // forge server.Config.Services.Forge = f server.Config.Services.Timeout = c.Duration("forge-timeout") // services - server.Config.Services.Queue = setupQueue(c, v) + server.Config.Services.Queue = setupQueue(c, s) server.Config.Services.Logs = logging.New() server.Config.Services.Pubsub = pubsub.New() server.Config.Services.Membership = setupMembershipService(c, f) - // TODO: setup addon manager + + pluginManager, err := plugins.NewManager(c, s, f) + if err != nil { + return fmt.Errorf("could not setup plugin manager: %w", err) + } + server.Config.Services.Manager = pluginManager // authentication server.Config.Pipeline.AuthenticatePublicRepos = c.Bool("authenticate-public-repos") diff --git a/server/api/global_secret.go b/server/api/global_secret.go index e2322035b7..b01de72e5b 100644 --- a/server/api/global_secret.go +++ b/server/api/global_secret.go @@ -35,7 +35,8 @@ import ( // @Param page query int false "for response pagination, page offset number" default(1) // @Param perPage query int false "for response pagination, max items per page" default(50) func GetGlobalSecretList(c *gin.Context) { - list, err := server.Config.Services.Secrets.GlobalSecretList(session.Pagination(c)) + secretService := server.Config.Services.Manager.SecretService() + list, err := secretService.GlobalSecretList(session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting global secret list. %s", err) return @@ -59,7 +60,8 @@ func GetGlobalSecretList(c *gin.Context) { // @Param secret path string true "the secret's name" func GetGlobalSecret(c *gin.Context) { name := c.Param("secret") - secret, err := server.Config.Services.Secrets.GlobalSecretFind(name) + secretService := server.Config.Services.Manager.SecretService() + secret, err := secretService.GlobalSecretFind(name) if err != nil { handleDBError(c, err) return @@ -92,7 +94,9 @@ func PostGlobalSecret(c *gin.Context) { c.String(http.StatusBadRequest, "Error inserting global secret. %s", err) return } - if err := server.Config.Services.Secrets.GlobalSecretCreate(secret); err != nil { + + secretService := server.Config.Services.Manager.SecretService() + if err := secretService.GlobalSecretCreate(secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting global secret %q. %s", in.Name, err) return } @@ -119,7 +123,8 @@ func PatchGlobalSecret(c *gin.Context) { return } - secret, err := server.Config.Services.Secrets.GlobalSecretFind(name) + secretService := server.Config.Services.Manager.SecretService() + secret, err := secretService.GlobalSecretFind(name) if err != nil { handleDBError(c, err) return @@ -138,7 +143,8 @@ func PatchGlobalSecret(c *gin.Context) { c.String(http.StatusBadRequest, "Error updating global secret. %s", err) return } - if err := server.Config.Services.Secrets.GlobalSecretUpdate(secret); err != nil { + + if err := secretService.GlobalSecretUpdate(secret); err != nil { c.String(http.StatusInternalServerError, "Error updating global secret %q. %s", in.Name, err) return } @@ -156,7 +162,8 @@ func PatchGlobalSecret(c *gin.Context) { // @Param secret path string true "the secret's name" func DeleteGlobalSecret(c *gin.Context) { name := c.Param("secret") - if err := server.Config.Services.Secrets.GlobalSecretDelete(name); err != nil { + secretService := server.Config.Services.Manager.SecretService() + if err := secretService.GlobalSecretDelete(name); err != nil { handleDBError(c, err) return } diff --git a/server/api/org_secret.go b/server/api/org_secret.go index b598ef378b..579820bc98 100644 --- a/server/api/org_secret.go +++ b/server/api/org_secret.go @@ -44,7 +44,8 @@ func GetOrgSecret(c *gin.Context) { return } - secret, err := server.Config.Services.Secrets.OrgSecretFind(orgID, name) + secretService := server.Config.Services.Manager.SecretService() + secret, err := secretService.OrgSecretFind(orgID, name) if err != nil { handleDBError(c, err) return @@ -70,7 +71,8 @@ func GetOrgSecretList(c *gin.Context) { return } - list, err := server.Config.Services.Secrets.OrgSecretList(orgID, session.Pagination(c)) + secretService := server.Config.Services.Manager.SecretService() + list, err := secretService.OrgSecretList(orgID, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting secret list for %q. %s", orgID, err) return @@ -116,7 +118,9 @@ func PostOrgSecret(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error inserting org %q secret. %s", orgID, err) return } - if err := server.Config.Services.Secrets.OrgSecretCreate(orgID, secret); err != nil { + + secretService := server.Config.Services.Manager.SecretService() + if err := secretService.OrgSecretCreate(orgID, secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting org %q secret %q. %s", orgID, in.Name, err) return } @@ -149,7 +153,8 @@ func PatchOrgSecret(c *gin.Context) { return } - secret, err := server.Config.Services.Secrets.OrgSecretFind(orgID, name) + secretService := server.Config.Services.Manager.SecretService() + secret, err := secretService.OrgSecretFind(orgID, name) if err != nil { handleDBError(c, err) return @@ -168,7 +173,8 @@ func PatchOrgSecret(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error updating org %q secret. %s", orgID, err) return } - if err := server.Config.Services.Secrets.OrgSecretUpdate(orgID, secret); err != nil { + + if err := secretService.OrgSecretUpdate(orgID, secret); err != nil { c.String(http.StatusInternalServerError, "Error updating org %q secret %q. %s", orgID, in.Name, err) return } @@ -193,7 +199,8 @@ func DeleteOrgSecret(c *gin.Context) { return } - if err := server.Config.Services.Secrets.OrgSecretDelete(orgID, name); err != nil { + secretService := server.Config.Services.Manager.SecretService() + if err := secretService.OrgSecretDelete(orgID, name); err != nil { handleDBError(c, err) return } diff --git a/server/api/registry.go b/server/api/registry.go index 27b1c9e047..60e6648ff8 100644 --- a/server/api/registry.go +++ b/server/api/registry.go @@ -35,11 +35,11 @@ import ( // @Param repo_id path int true "the repository id" // @Param registry path string true "the registry name" func GetRegistry(c *gin.Context) { - var ( - repo = session.Repo(c) - name = c.Param("registry") - ) - registry, err := server.Config.Services.Registries.RegistryFind(repo, name) + repo := session.Repo(c) + name := c.Param("registry") + + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + registry, err := registryService.RegistryFind(repo, name) if err != nil { handleDBError(c, err) return @@ -75,7 +75,9 @@ func PostRegistry(c *gin.Context) { c.String(http.StatusBadRequest, "Error inserting registry. %s", err) return } - if err := server.Config.Services.Registries.RegistryCreate(repo, registry); err != nil { + + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + if err := registryService.RegistryCreate(repo, registry); err != nil { c.String(http.StatusInternalServerError, "Error inserting registry %q. %s", in.Address, err) return } @@ -106,7 +108,8 @@ func PatchRegistry(c *gin.Context) { return } - registry, err := server.Config.Services.Registries.RegistryFind(repo, name) + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + registry, err := registryService.RegistryFind(repo, name) if err != nil { handleDBError(c, err) return @@ -122,7 +125,7 @@ func PatchRegistry(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error updating registry. %s", err) return } - if err := server.Config.Services.Registries.RegistryUpdate(repo, registry); err != nil { + if err := registryService.RegistryUpdate(repo, registry); err != nil { c.String(http.StatusInternalServerError, "Error updating registry %q. %s", in.Address, err) return } @@ -142,7 +145,8 @@ func PatchRegistry(c *gin.Context) { // @Param perPage query int false "for response pagination, max items per page" default(50) func GetRegistryList(c *gin.Context) { repo := session.Repo(c) - list, err := server.Config.Services.Registries.RegistryList(repo, session.Pagination(c)) + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + list, err := registryService.RegistryList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting registry list. %s", err) return @@ -166,11 +170,11 @@ func GetRegistryList(c *gin.Context) { // @Param repo_id path int true "the repository id" // @Param registry path string true "the registry name" func DeleteRegistry(c *gin.Context) { - var ( - repo = session.Repo(c) - name = c.Param("registry") - ) - err := server.Config.Services.Registries.RegistryDelete(repo, name) + repo := session.Repo(c) + name := c.Param("registry") + + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + err := registryService.RegistryDelete(repo, name) if err != nil { handleDBError(c, err) return diff --git a/server/api/repo_secret.go b/server/api/repo_secret.go index c7ef9eb72d..2a1b963483 100644 --- a/server/api/repo_secret.go +++ b/server/api/repo_secret.go @@ -36,11 +36,11 @@ import ( // @Param repo_id path int true "the repository id" // @Param secretName path string true "the secret name" func GetSecret(c *gin.Context) { - var ( - repo = session.Repo(c) - name = c.Param("secret") - ) - secret, err := server.Config.Services.Secrets.SecretFind(repo, name) + repo := session.Repo(c) + name := c.Param("secret") + + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + secret, err := secretService.SecretFind(repo, name) if err != nil { handleDBError(c, err) return @@ -77,7 +77,9 @@ func PostSecret(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error inserting secret. %s", err) return } - if err := server.Config.Services.Secrets.SecretCreate(repo, secret); err != nil { + + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + if err := secretService.SecretCreate(repo, secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting secret %q. %s", in.Name, err) return } @@ -108,7 +110,8 @@ func PatchSecret(c *gin.Context) { return } - secret, err := server.Config.Services.Secrets.SecretFind(repo, name) + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + secret, err := secretService.SecretFind(repo, name) if err != nil { handleDBError(c, err) return @@ -127,7 +130,7 @@ func PatchSecret(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error updating secret. %s", err) return } - if err := server.Config.Services.Secrets.SecretUpdate(repo, secret); err != nil { + if err := secretService.SecretUpdate(repo, secret); err != nil { c.String(http.StatusInternalServerError, "Error updating secret %q. %s", in.Name, err) return } @@ -147,7 +150,8 @@ func PatchSecret(c *gin.Context) { // @Param perPage query int false "for response pagination, max items per page" default(50) func GetSecretList(c *gin.Context) { repo := session.Repo(c) - list, err := server.Config.Services.Secrets.SecretList(repo, session.Pagination(c)) + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + list, err := secretService.SecretList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting secret list. %s", err) return @@ -171,11 +175,11 @@ func GetSecretList(c *gin.Context) { // @Param repo_id path int true "the repository id" // @Param secretName path string true "the secret name" func DeleteSecret(c *gin.Context) { - var ( - repo = session.Repo(c) - name = c.Param("secret") - ) - if err := server.Config.Services.Secrets.SecretDelete(repo, name); err != nil { + repo := session.Repo(c) + name := c.Param("secret") + + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + if err := secretService.SecretDelete(repo, name); err != nil { handleDBError(c, err) return } diff --git a/server/api/signature_public_key.go b/server/api/signature_public_key.go index 937f900247..b9364166b4 100644 --- a/server/api/signature_public_key.go +++ b/server/api/signature_public_key.go @@ -34,7 +34,7 @@ import ( // @Tags System // @Param Authorization header string true "Insert your personal access token" default(Bearer ) func GetSignaturePublicKey(c *gin.Context) { - b, err := x509.MarshalPKIXPublicKey(server.Config.Services.SignaturePublicKey) + b, err := x509.MarshalPKIXPublicKey(server.Config.Services.Manager.SignaturePublicKey()) if err != nil { log.Error().Err(err).Msg("can't marshal public key") c.AbortWithStatus(http.StatusInternalServerError) diff --git a/server/config.go b/server/config.go index 31e98a1533..fb600f6e39 100644 --- a/server/config.go +++ b/server/config.go @@ -24,6 +24,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/logging" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/pubsub" "go.woodpecker-ci.org/woodpecker/v2/server/queue" @@ -37,6 +38,7 @@ var Config = struct { Forge forge.Forge Timeout time.Duration Membership cache.MembershipService + Manager *plugins.Manager } Storage struct { // Users model.UserStore diff --git a/server/pipeline/create.go b/server/pipeline/create.go index da467425c5..abede02367 100644 --- a/server/pipeline/create.go +++ b/server/pipeline/create.go @@ -68,8 +68,10 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline } // fetch the pipeline file from the forge - configFetcher := forge.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.Timeout, server.Config.Services.ConfigService, repoUser, repo, pipeline) - forgeYamlConfigs, configFetchErr := configFetcher.Fetch(ctx) + _forge := server.Config.Services.Forge + configService := server.Config.Services.Manager.ConfigServiceFromRepo(repo) + // configFetcher := configService.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.Timeout, server.Config.Services.ConfigService, repoUser, repo, pipeline) + forgeYamlConfigs, configFetchErr := configService.Fetch(ctx, _forge, repoUser, repo, pipeline) if errors.Is(configFetchErr, &forge_types.ErrConfigNotFound{}) { log.Debug().Str("repo", repo.FullName).Err(configFetchErr).Msgf("cannot find config '%s' in '%s' with user: '%s'", repo.Config, pipeline.Ref, repoUser.Login) if err := _store.DeletePipeline(pipeline); err != nil { diff --git a/server/pipeline/items.go b/server/pipeline/items.go index 3e24a28add..35206cd117 100644 --- a/server/pipeline/items.go +++ b/server/pipeline/items.go @@ -42,12 +42,14 @@ func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *mod log.Error().Err(err).Str("repo", repo.FullName).Msgf("error getting last pipeline before pipeline number '%d'", currentPipeline.Number) } - secs, err := server.Config.Services.Secrets.SecretListPipeline(repo, currentPipeline, &model.ListOptions{All: true}) + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + secs, err := secretService.SecretListPipeline(repo, currentPipeline, &model.ListOptions{All: true}) if err != nil { log.Error().Err(err).Msgf("error getting secrets for %s#%d", repo.FullName, currentPipeline.Number) } - regs, err := server.Config.Services.Registries.RegistryList(repo, &model.ListOptions{All: true}) + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + regs, err := registryService.RegistryList(repo, &model.ListOptions{All: true}) if err != nil { log.Error().Err(err).Msgf("error getting registry credentials for %s#%d", repo.FullName, currentPipeline.Number) } @@ -55,8 +57,10 @@ func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *mod if envs == nil { envs = map[string]string{} } - if server.Config.Services.Environ != nil { - globals, _ := server.Config.Services.Environ.EnvironList(repo) + + environmentService := server.Config.Services.Manager.EnvironmentService() + if environmentService != nil { + globals, _ := environmentService.EnvironList(repo) for _, global := range globals { envs[global.Name] = global.Value } diff --git a/server/pipeline/restart.go b/server/pipeline/restart.go index 73933ceeb3..29045bf072 100644 --- a/server/pipeline/restart.go +++ b/server/pipeline/restart.go @@ -29,6 +29,7 @@ import ( // Restart a pipeline by creating a new one out of the old and start it func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipeline, user *model.User, repo *model.Repo, envs map[string]string, netrc *model.Netrc) (*model.Pipeline, error) { + forge := server.Config.Services.Forge switch lastPipeline.Status { case model.StatusDeclined, model.StatusBlocked: @@ -49,20 +50,11 @@ func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipelin } // If the config extension is active we should refetch the config in case something changed - if server.Config.Services.ConfigService != nil { - currentFileMeta := make([]*forge_types.FileMeta, len(configs)) - for i, cfg := range configs { - currentFileMeta[i] = &forge_types.FileMeta{Name: cfg.Name, Data: cfg.Data} - } - - newConfig, useOld, err := server.Config.Services.ConfigService.FetchConfig(ctx, repo, lastPipeline, currentFileMeta, netrc) - if err != nil { - return nil, &ErrBadRequest{ - Msg: fmt.Sprintf("On fetching external pipeline config: %s", err), - } - } - if !useOld { - pipelineFiles = newConfig + configService := server.Config.Services.Manager.ConfigServiceFromRepo(repo) + pipelineFiles, err = configService.Fetch(ctx, forge, user, repo, lastPipeline) + if err != nil { + return nil, &ErrBadRequest{ + Msg: fmt.Sprintf("On fetching external pipeline config: %s", err), } } diff --git a/server/forge/configFetcher.go b/server/plugins/config/forge.go similarity index 69% rename from server/forge/configFetcher.go rename to server/plugins/config/forge.go index ef989b34c5..8675ce27ee 100644 --- a/server/forge/configFetcher.go +++ b/server/plugins/config/forge.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package forge +package config import ( "context" @@ -23,29 +23,39 @@ import ( "github.com/rs/zerolog/log" + "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" "go.woodpecker-ci.org/woodpecker/v2/shared/constant" ) +type forgeFetcher struct { +} + +func NewForge() Service { + return &forgeFetcher{} +} + +func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ([]*types.FileMeta, error) { + configFetcher := NewConfigFetcher(forge, 5*time.Second, user, repo, pipeline) + return configFetcher.Fetch(ctx) +} + type ConfigFetcher struct { - forge Forge - user *model.User - repo *model.Repo - pipeline *model.Pipeline - configExtension config.Extension - timeout time.Duration + forge forge.Forge + user *model.User + repo *model.Repo + pipeline *model.Pipeline + timeout time.Duration } -func NewConfigFetcher(forge Forge, timeout time.Duration, configExtension config.Extension, user *model.User, repo *model.Repo, pipeline *model.Pipeline) *ConfigFetcher { +func NewConfigFetcher(forge forge.Forge, timeout time.Duration, user *model.User, repo *model.Repo, pipeline *model.Pipeline) *ConfigFetcher { return &ConfigFetcher{ - forge: forge, - user: user, - repo: repo, - pipeline: pipeline, - configExtension: configExtension, - timeout: timeout, + forge: forge, + user: user, + repo: repo, + pipeline: pipeline, + timeout: timeout, } } @@ -63,30 +73,29 @@ func (cf *ConfigFetcher) Fetch(ctx context.Context) (files []*types.FileMeta, er continue } - if cf.configExtension != nil { - fetchCtx, cancel := context.WithTimeout(ctx, cf.timeout) - defer cancel() // ok here as we only try http fetching once, returning on fail and success - - log.Trace().Msgf("configFetcher[%s]: getting config from external http service", cf.repo.FullName) - netrc, err := cf.forge.Netrc(cf.user, cf.repo) - if err != nil { - return nil, fmt.Errorf("could not get Netrc data from forge: %w", err) - } - - newConfigs, useOld, err := cf.configExtension.FetchConfig(fetchCtx, cf.repo, cf.pipeline, files, netrc) - if err != nil { - log.Error().Err(err).Msg("could not fetch config via http") - return nil, fmt.Errorf("could not fetch config via http: %w", err) - } - - if !useOld { - return newConfigs, nil - } - } - - return + // if cf.configExtension != nil { + // fetchCtx, cancel := context.WithTimeout(ctx, cf.timeout) + // defer cancel() // ok here as we only try http fetching once, returning on fail and success + + // log.Trace().Msgf("configFetcher[%s]: getting config from external http service", cf.repo.FullName) + // netrc, err := cf.forge.Netrc(cf.user, cf.repo) + // if err != nil { + // return nil, fmt.Errorf("could not get Netrc data from forge: %w", err) + // } + + // newConfigs, useOld, err := cf.configExtension.FetchConfig(fetchCtx, cf.repo, cf.pipeline, files, netrc) + // if err != nil { + // log.Error().Err(err).Msg("could not fetch config via http") + // return nil, fmt.Errorf("could not fetch config via http: %w", err) + // } + + // if !useOld { + // return newConfigs, nil + // } + // } } - return + + return files, err } // fetch config by timeout @@ -100,9 +109,9 @@ func (cf *ConfigFetcher) fetch(c context.Context, config string) ([]*types.FileM // could be adapted to allow the user to supply a list like we do in the defaults configs := []string{config} - fileMeta, err := cf.getFirstAvailableConfig(ctx, configs) + fileMetas, err := cf.getFirstAvailableConfig(ctx, configs) if err == nil { - return fileMeta, err + return fileMetas, err } return nil, fmt.Errorf("user defined config '%s' not found: %w", config, err) @@ -110,16 +119,16 @@ func (cf *ConfigFetcher) fetch(c context.Context, config string) ([]*types.FileM log.Trace().Msgf("configFetcher[%s]: user did not define own config, following default procedure", cf.repo.FullName) // for the order see shared/constants/constants.go - fileMeta, err := cf.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:]) + fileMetas, err := cf.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:]) if err == nil { - return fileMeta, err + return fileMetas, err } select { case <-ctx.Done(): return nil, ctx.Err() default: - return []*types.FileMeta{}, fmt.Errorf("configFetcher: fallback did not find config: %w", err) + return nil, fmt.Errorf("configFetcher: fallback did not find config: %w", err) } } diff --git a/server/forge/configFetcher_test.go b/server/plugins/config/forge_test.go similarity index 98% rename from server/forge/configFetcher_test.go rename to server/plugins/config/forge_test.go index 36b25d9c58..a346c65e21 100644 --- a/server/forge/configFetcher_test.go +++ b/server/plugins/config/forge_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package forge_test +package config_test import ( "context" @@ -31,7 +31,6 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/forge/mocks" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" @@ -313,7 +312,7 @@ func TestFetch(t *testing.T) { f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("file not found")) f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("directory not found")) - configFetcher := forge.NewConfigFetcher( + configFetcher := config.NewConfigFetcher( f, time.Second*3, nil, @@ -521,7 +520,7 @@ func TestFetchFromConfigService(t *testing.T) { f.On("Netrc", mock.Anything, mock.Anything).Return(&model.Netrc{Machine: "mock", Login: "mock", Password: "mock"}, nil) - configFetcher := forge.NewConfigFetcher( + configFetcher := config.NewConfigFetcher( f, time.Second*3, configAPI, diff --git a/server/plugins/config/http.go b/server/plugins/config/http.go index d548cdb2c5..a4ce230d71 100644 --- a/server/plugins/config/http.go +++ b/server/plugins/config/http.go @@ -19,7 +19,7 @@ import ( "crypto" "fmt" - forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" + "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/utils" ) @@ -29,55 +29,43 @@ type http struct { privateKey crypto.PrivateKey } -// Same as forge.FileMeta but with json tags and string data -type config struct { - Name string `json:"name"` - Data string `json:"data"` -} - type requestStructure struct { Repo *model.Repo `json:"repo"` Pipeline *model.Pipeline `json:"pipeline"` - Configuration []*config `json:"configs"` + Configuration []*ConfigData `json:"configs"` Netrc *model.Netrc `json:"netrc"` } type responseStructure struct { - Configs []config `json:"configs"` + Configs []*ConfigData `json:"configs"` } func NewHTTP(endpoint string, privateKey crypto.PrivateKey) Service { return &http{endpoint, privateKey} } -func (cp *http) FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge_types.FileMeta, netrc *model.Netrc) (configData []*forge_types.FileMeta, useOld bool, err error) { - currentConfigs := make([]*config, len(currentFileMeta)) - for i, pipe := range currentFileMeta { - currentConfigs[i] = &config{Name: pipe.Name, Data: string(pipe.Data)} - } +func (h *http) Fetch(ctx context.Context, forge *forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline, netrc *model.Netrc) ([]*ConfigData, error) { + // currentConfigs := make([]*configData, len(currentFileMeta)) + // for i, pipe := range currentFileMeta { + // currentConfigs[i] = &configData{Name: pipe.Name, Data: string(pipe.Data)} + // } response := new(responseStructure) body := requestStructure{ - Repo: repo, - Pipeline: pipeline, - Configuration: currentConfigs, - Netrc: netrc, + Repo: repo, + Pipeline: pipeline, + // Configuration: currentConfigs, + Netrc: netrc, } - status, err := utils.Send(ctx, "POST", cp.endpoint, cp.privateKey, body, response) + status, err := utils.Send(ctx, "POST", h.endpoint, h.privateKey, body, response) if err != nil && status != 204 { - return nil, false, fmt.Errorf("failed to fetch config via http (%d) %w", status, err) + return nil, fmt.Errorf("failed to fetch config via http (%d) %w", status, err) } - var newFileMeta []*forge_types.FileMeta if status != 200 { - newFileMeta = make([]*forge_types.FileMeta, 0) - } else { - newFileMeta = make([]*forge_types.FileMeta, len(response.Configs)) - for i, pipe := range response.Configs { - newFileMeta[i] = &forge_types.FileMeta{Name: pipe.Name, Data: []byte(pipe.Data)} - } + return []*ConfigData{}, nil } - return newFileMeta, status == 204, nil + return response.Configs, nil } diff --git a/server/plugins/config/service.go b/server/plugins/config/service.go index 326cb50a61..0912091a5e 100644 --- a/server/plugins/config/service.go +++ b/server/plugins/config/service.go @@ -17,10 +17,17 @@ package config import ( "context" - forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" + "go.woodpecker-ci.org/woodpecker/v2/server/forge" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" ) +// ConfigData same as forge.FileMeta but with json tags and string data +type ConfigData struct { + Name string `json:"name"` + Data string `json:"data"` +} + type Service interface { - FetchConfig(ctx context.Context, repo *model.Repo, pipeline *model.Pipeline, currentFileMeta []*forge_types.FileMeta, netrc *model.Netrc) (configData []*forge_types.FileMeta, useOld bool, err error) + Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (configData []*types.FileMeta, err error) } diff --git a/server/plugins/manager.go b/server/plugins/manager.go index 410e241b1e..19dd567c76 100644 --- a/server/plugins/manager.go +++ b/server/plugins/manager.go @@ -1,4 +1,4 @@ -package extensions +package plugins import ( "crypto" @@ -17,12 +17,12 @@ type Manager struct { secret secret.Service registry registry.Service config config.Service - environ environments.Service + environment environments.Service signaturePrivateKey crypto.PrivateKey signaturePublicKey crypto.PublicKey } -func NewManager(store store.Store, forge forge.Forge, c *cli.Context) (*Manager, error) { +func NewManager(c *cli.Context, store store.Store, forge forge.Forge) (*Manager, error) { signaturePrivateKey, signaturePublicKey, err := setupSignatureKeys(store) if err != nil { return nil, err @@ -39,7 +39,7 @@ func NewManager(store store.Store, forge forge.Forge, c *cli.Context) (*Manager, secret: setupSecretExtension(store), registry: setupRegistryExtension(store, c.String("docker-config")), config: config, - environ: environments.Parse(c.StringSlice("environment")), + environment: environments.Parse(c.StringSlice("environment")), }, nil } @@ -47,30 +47,26 @@ func (e *Manager) SignaturePublicKey() crypto.PublicKey { return e.signaturePublicKey } -func (e *Manager) SecretAddonFromRepo(repo *model.Repo) secret.Service { - // if repo.SecretEndpoint != "" { - // return secret.NewHTTP(repo.SecretEndpoint, e.signaturePrivateKey) - // } - +func (e *Manager) SecretServiceFromRepo(_ *model.Repo) secret.Service { return e.secret } -func (e *Manager) RegistryAddonFromRepo(repo *model.Repo) registry.Service { - // if repo.SecretEndpoint != "" { - // return registry.NewHTTP(repo.SecretEndpoint, e.signaturePrivateKey) - // } +func (e *Manager) SecretService() secret.Service { + return e.secret +} +func (e *Manager) RegistryServiceFromRepo(_ *model.Repo) registry.Service { return e.registry } -func (e *Manager) ConfigAddonFromRepo(repo *model.Repo) config.Service { - // if repo.ConfigEndpoint != "" { - // return config.NewHTTP(repo.ConfigEndpoint, e.signaturePrivateKey) - // } +func (e *Manager) RegistryService() registry.Service { + return e.registry +} +func (e *Manager) ConfigServiceFromRepo(_ *model.Repo) config.Service { return e.config } -func (e *Manager) EnvironAddonFromRepo(repo *model.Repo) environments.Service { - return e.environ +func (e *Manager) EnvironmentService() environments.Service { + return e.environment } diff --git a/server/plugins/setup.go b/server/plugins/setup.go index 045e8163ab..303baf7c86 100644 --- a/server/plugins/setup.go +++ b/server/plugins/setup.go @@ -1,4 +1,4 @@ -package extensions +package plugins import ( "crypto" From d824a1c6337b7ac299e06d446439f18313329cdb Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Wed, 7 Feb 2024 23:23:52 +0100 Subject: [PATCH 04/28] cleanup --- .../registry/{combine.go => combined.go} | 0 server/plugins/setup.go | 42 ------------------- 2 files changed, 42 deletions(-) rename server/plugins/registry/{combine.go => combined.go} (100%) diff --git a/server/plugins/registry/combine.go b/server/plugins/registry/combined.go similarity index 100% rename from server/plugins/registry/combine.go rename to server/plugins/registry/combined.go diff --git a/server/plugins/setup.go b/server/plugins/setup.go index 303baf7c86..6f11165197 100644 --- a/server/plugins/setup.go +++ b/server/plugins/setup.go @@ -85,45 +85,3 @@ func setupSignatureKeys(_store store.Store) (crypto.PrivateKey, crypto.PublicKey privateKey := ed25519.PrivateKey(privKeyStr) return privateKey, privateKey.Public(), nil } - -// func setupSecretService(c *cli.Context, s model.SecretStore) (model.SecretService, error) { -// addonService, err := addon.Load[model.SecretService](c.StringSlice("addons"), addonTypes.TypeSecretService) -// if err != nil { -// return nil, err -// } -// if addonService != nil { -// return addonService.Value, nil -// } - -// return secrets.New(c.Context, s), nil -// } - -// func setupRegistryService(c *cli.Context, s store.Store) (model.RegistryService, error) { -// addonService, err := addon.Load[model.RegistryService](c.StringSlice("addons"), addonTypes.TypeRegistryService) -// if err != nil { -// return nil, err -// } -// if addonService != nil { -// return addonService.Value, nil -// } - -// if c.String("docker-config") != "" { -// return registry.Combined( -// registry.New(s), -// registry.Filesystem(c.String("docker-config")), -// ), nil -// } -// return registry.New(s), nil -// } - -// func setupEnvironService(c *cli.Context, _ store.Store) (model.EnvironService, error) { -// addonService, err := addon.Load[model.EnvironService](c.StringSlice("addons"), addonTypes.TypeEnvironmentService) -// if err != nil { -// return nil, err -// } -// if addonService != nil { -// return addonService.Value, nil -// } - -// return environments.Parse(c.StringSlice("environment")), nil -// } From 80e8fcd376c4a369f179cc671194a5e1e34b7dbf Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Wed, 7 Feb 2024 23:43:31 +0100 Subject: [PATCH 05/28] cleanup --- cmd/server/server.go | 1 - server/config.go | 1 - server/pipeline/create.go | 5 +- server/plugins/config/combined.go | 84 ++++++++++++++++++++ server/plugins/config/forge.go | 95 ++++------------------- server/plugins/config/http.go | 37 ++++++--- server/plugins/config/service.go | 8 +- server/plugins/environments/parse_test.go | 14 ++++ server/plugins/environments/service.go | 14 ++++ server/plugins/manager.go | 20 ++++- server/plugins/registry/service.go | 14 ++++ server/plugins/secret/service.go | 14 ++++ server/plugins/setup.go | 21 ++++- 13 files changed, 223 insertions(+), 105 deletions(-) create mode 100644 server/plugins/config/combined.go diff --git a/cmd/server/server.go b/cmd/server/server.go index ce093f6a4c..3481fef687 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -276,7 +276,6 @@ func run(c *cli.Context) error { func setupEvilGlobals(c *cli.Context, s store.Store, f forge.Forge) error { // forge server.Config.Services.Forge = f - server.Config.Services.Timeout = c.Duration("forge-timeout") // services server.Config.Services.Queue = setupQueue(c, s) diff --git a/server/config.go b/server/config.go index fb600f6e39..c92286a81a 100644 --- a/server/config.go +++ b/server/config.go @@ -36,7 +36,6 @@ var Config = struct { Queue queue.Queue Logs logging.Log Forge forge.Forge - Timeout time.Duration Membership cache.MembershipService Manager *plugins.Manager } diff --git a/server/pipeline/create.go b/server/pipeline/create.go index abede02367..4e41b6100e 100644 --- a/server/pipeline/create.go +++ b/server/pipeline/create.go @@ -34,6 +34,7 @@ var skipPipelineRegex = regexp.MustCompile(`\[(?i:ci *skip|skip *ci)\]`) // Create a new pipeline and start it func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline *model.Pipeline) (*model.Pipeline, error) { + _forge := server.Config.Services.Forge repoUser, err := _store.GetUser(repo.UserID) if err != nil { msg := fmt.Sprintf("failure to find repo owner via id '%d'", repo.UserID) @@ -54,7 +55,7 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline // If the forge has a refresh token, the current access token // may be stale. Therefore, we should refresh prior to dispatching // the pipeline. - forge.Refresh(ctx, server.Config.Services.Forge, _store, repoUser) + forge.Refresh(ctx, _forge, _store, repoUser) // update some pipeline fields pipeline.RepoID = repo.ID @@ -68,9 +69,7 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline } // fetch the pipeline file from the forge - _forge := server.Config.Services.Forge configService := server.Config.Services.Manager.ConfigServiceFromRepo(repo) - // configFetcher := configService.NewConfigFetcher(server.Config.Services.Forge, server.Config.Services.Timeout, server.Config.Services.ConfigService, repoUser, repo, pipeline) forgeYamlConfigs, configFetchErr := configService.Fetch(ctx, _forge, repoUser, repo, pipeline) if errors.Is(configFetchErr, &forge_types.ErrConfigNotFound{}) { log.Debug().Str("repo", repo.FullName).Err(configFetchErr).Msgf("cannot find config '%s' in '%s' with user: '%s'", repo.Config, pipeline.Ref, repoUser.Login) diff --git a/server/plugins/config/combined.go b/server/plugins/config/combined.go new file mode 100644 index 0000000000..356502c103 --- /dev/null +++ b/server/plugins/config/combined.go @@ -0,0 +1,84 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config + +import ( + "context" + "errors" + "strings" + "time" + + "github.com/rs/zerolog/log" + + "go.woodpecker-ci.org/woodpecker/v2/server/forge" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" + "go.woodpecker-ci.org/woodpecker/v2/server/model" +) + +type forgeFetcher struct { + timeout time.Duration +} + +func NewForge(timeout time.Duration) Service { + return &forgeFetcher{ + timeout: timeout, + } +} + +func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ([]*types.FileMeta, error) { + var files []*types.FileMeta + var err error + + // if cf.configExtension != nil { + // fetchCtx, cancel := context.WithTimeout(ctx, cf.timeout) + // defer cancel() // ok here as we only try http fetching once, returning on fail and success + + // log.Trace().Msgf("configFetcher[%s]: getting config from external http service", cf.repo.FullName) + // netrc, err := cf.forge.Netrc(cf.user, cf.repo) + // if err != nil { + // return nil, fmt.Errorf("could not get Netrc data from forge: %w", err) + // } + + // newConfigs, useOld, err := cf.configExtension.FetchConfig(fetchCtx, cf.repo, cf.pipeline, files, netrc) + // if err != nil { + // log.Error().Err(err).Msg("could not fetch config via http") + // return nil, fmt.Errorf("could not fetch config via http: %w", err) + // } + + // if !useOld { + // return newConfigs, nil + // } + // } + + // try to fetch 3 times + configFetcher := &forgeFetcherContext{ + forge: forge, + user: user, + repo: repo, + pipeline: pipeline, + timeout: f.timeout, + } + for i := 0; i < 3; i++ { + files, err = configFetcher.fetch(ctx, strings.TrimSpace(repo.Config)) + if err != nil { + log.Trace().Err(err).Msgf("%d. try failed", i+1) + } + if errors.Is(err, context.DeadlineExceeded) { + continue + } + } + + return files, err +} diff --git a/server/plugins/config/forge.go b/server/plugins/config/forge.go index 8675ce27ee..745a07373b 100644 --- a/server/plugins/config/forge.go +++ b/server/plugins/config/forge.go @@ -1,4 +1,4 @@ -// Copyright 2022 Woodpecker Authors +// Copyright 2024 Woodpecker Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -29,19 +29,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/shared/constant" ) -type forgeFetcher struct { -} - -func NewForge() Service { - return &forgeFetcher{} -} - -func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ([]*types.FileMeta, error) { - configFetcher := NewConfigFetcher(forge, 5*time.Second, user, repo, pipeline) - return configFetcher.Fetch(ctx) -} - -type ConfigFetcher struct { +type forgeFetcherContext struct { forge forge.Forge user *model.User repo *model.Repo @@ -49,67 +37,18 @@ type ConfigFetcher struct { timeout time.Duration } -func NewConfigFetcher(forge forge.Forge, timeout time.Duration, user *model.User, repo *model.Repo, pipeline *model.Pipeline) *ConfigFetcher { - return &ConfigFetcher{ - forge: forge, - user: user, - repo: repo, - pipeline: pipeline, - timeout: timeout, - } -} - -// Fetch pipeline config from source forge -func (cf *ConfigFetcher) Fetch(ctx context.Context) (files []*types.FileMeta, err error) { - log.Trace().Msgf("start fetching config for '%s'", cf.repo.FullName) - - // try to fetch 3 times - for i := 0; i < 3; i++ { - files, err = cf.fetch(ctx, strings.TrimSpace(cf.repo.Config)) - if err != nil { - log.Trace().Err(err).Msgf("%d. try failed", i+1) - } - if errors.Is(err, context.DeadlineExceeded) { - continue - } - - // if cf.configExtension != nil { - // fetchCtx, cancel := context.WithTimeout(ctx, cf.timeout) - // defer cancel() // ok here as we only try http fetching once, returning on fail and success - - // log.Trace().Msgf("configFetcher[%s]: getting config from external http service", cf.repo.FullName) - // netrc, err := cf.forge.Netrc(cf.user, cf.repo) - // if err != nil { - // return nil, fmt.Errorf("could not get Netrc data from forge: %w", err) - // } - - // newConfigs, useOld, err := cf.configExtension.FetchConfig(fetchCtx, cf.repo, cf.pipeline, files, netrc) - // if err != nil { - // log.Error().Err(err).Msg("could not fetch config via http") - // return nil, fmt.Errorf("could not fetch config via http: %w", err) - // } - - // if !useOld { - // return newConfigs, nil - // } - // } - } - - return files, err -} - // fetch config by timeout -func (cf *ConfigFetcher) fetch(c context.Context, config string) ([]*types.FileMeta, error) { - ctx, cancel := context.WithTimeout(c, cf.timeout) +func (f *forgeFetcherContext) fetch(c context.Context, config string) ([]*types.FileMeta, error) { + ctx, cancel := context.WithTimeout(c, f.timeout) defer cancel() if len(config) > 0 { - log.Trace().Msgf("configFetcher[%s]: use user config '%s'", cf.repo.FullName, config) + log.Trace().Msgf("configFetcher[%s]: use user config '%s'", f.repo.FullName, config) // could be adapted to allow the user to supply a list like we do in the defaults configs := []string{config} - fileMetas, err := cf.getFirstAvailableConfig(ctx, configs) + fileMetas, err := f.getFirstAvailableConfig(ctx, configs) if err == nil { return fileMetas, err } @@ -117,9 +56,9 @@ func (cf *ConfigFetcher) fetch(c context.Context, config string) ([]*types.FileM return nil, fmt.Errorf("user defined config '%s' not found: %w", config, err) } - log.Trace().Msgf("configFetcher[%s]: user did not define own config, following default procedure", cf.repo.FullName) + log.Trace().Msgf("configFetcher[%s]: user did not define own config, following default procedure", f.repo.FullName) // for the order see shared/constants/constants.go - fileMetas, err := cf.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:]) + fileMetas, err := f.getFirstAvailableConfig(ctx, constant.DefaultConfigOrder[:]) if err == nil { return fileMetas, err } @@ -144,11 +83,11 @@ func filterPipelineFiles(files []*types.FileMeta) []*types.FileMeta { return res } -func (cf *ConfigFetcher) checkPipelineFile(c context.Context, config string) ([]*types.FileMeta, error) { - file, err := cf.forge.File(c, cf.user, cf.repo, cf.pipeline, config) +func (f *forgeFetcherContext) checkPipelineFile(c context.Context, config string) ([]*types.FileMeta, error) { + file, err := f.forge.File(c, f.user, f.repo, f.pipeline, config) if err == nil && len(file) != 0 { - log.Trace().Msgf("configFetcher[%s]: found file '%s'", cf.repo.FullName, config) + log.Trace().Msgf("configFetcher[%s]: found file '%s'", f.repo.FullName, config) return []*types.FileMeta{{ Name: config, @@ -159,30 +98,30 @@ func (cf *ConfigFetcher) checkPipelineFile(c context.Context, config string) ([] return nil, err } -func (cf *ConfigFetcher) getFirstAvailableConfig(c context.Context, configs []string) ([]*types.FileMeta, error) { +func (f *forgeFetcherContext) getFirstAvailableConfig(c context.Context, configs []string) ([]*types.FileMeta, error) { var forgeErr []error for _, fileOrFolder := range configs { if strings.HasSuffix(fileOrFolder, "/") { // config is a folder - files, err := cf.forge.Dir(c, cf.user, cf.repo, cf.pipeline, strings.TrimSuffix(fileOrFolder, "/")) + files, err := f.forge.Dir(c, f.user, f.repo, f.pipeline, strings.TrimSuffix(fileOrFolder, "/")) // if folder is not supported we will get a "Not implemented" error and continue if err != nil { if !(errors.Is(err, types.ErrNotImplemented) || errors.Is(err, &types.ErrConfigNotFound{})) { - log.Error().Err(err).Str("repo", cf.repo.FullName).Str("user", cf.user.Login).Msg("could not get folder from forge") + log.Error().Err(err).Str("repo", f.repo.FullName).Str("user", f.user.Login).Msg("could not get folder from forge") forgeErr = append(forgeErr, err) } continue } files = filterPipelineFiles(files) if len(files) != 0 { - log.Trace().Msgf("configFetcher[%s]: found %d files in '%s'", cf.repo.FullName, len(files), fileOrFolder) + log.Trace().Msgf("configFetcher[%s]: found %d files in '%s'", f.repo.FullName, len(files), fileOrFolder) return files, nil } } // config is a file - if fileMeta, err := cf.checkPipelineFile(c, fileOrFolder); err == nil { - log.Trace().Msgf("configFetcher[%s]: found file: '%s'", cf.repo.FullName, fileOrFolder) + if fileMeta, err := f.checkPipelineFile(c, fileOrFolder); err == nil { + log.Trace().Msgf("configFetcher[%s]: found file: '%s'", f.repo.FullName, fileOrFolder) return fileMeta, nil } else if !errors.Is(err, &types.ErrConfigNotFound{}) { forgeErr = append(forgeErr, err) diff --git a/server/plugins/config/http.go b/server/plugins/config/http.go index a4ce230d71..a26ebfc74a 100644 --- a/server/plugins/config/http.go +++ b/server/plugins/config/http.go @@ -1,4 +1,4 @@ -// Copyright 2022 Woodpecker Authors +// Copyright 2024 Woodpecker Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -20,6 +20,7 @@ import ( "fmt" "go.woodpecker-ci.org/woodpecker/v2/server/forge" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/utils" ) @@ -29,32 +30,43 @@ type http struct { privateKey crypto.PrivateKey } +// configData same as forge.FileMeta but with json tags and string data +type configData struct { + Name string `json:"name"` + Data string `json:"data"` +} + type requestStructure struct { - Repo *model.Repo `json:"repo"` - Pipeline *model.Pipeline `json:"pipeline"` - Configuration []*ConfigData `json:"configs"` - Netrc *model.Netrc `json:"netrc"` + Repo *model.Repo `json:"repo"` + Pipeline *model.Pipeline `json:"pipeline"` + Netrc *model.Netrc `json:"netrc"` + // Configuration []*configData `json:"configs"` } type responseStructure struct { - Configs []*ConfigData `json:"configs"` + Configs []*configData `json:"configs"` } func NewHTTP(endpoint string, privateKey crypto.PrivateKey) Service { return &http{endpoint, privateKey} } -func (h *http) Fetch(ctx context.Context, forge *forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline, netrc *model.Netrc) ([]*ConfigData, error) { +func (h *http) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ([]*types.FileMeta, error) { // currentConfigs := make([]*configData, len(currentFileMeta)) // for i, pipe := range currentFileMeta { // currentConfigs[i] = &configData{Name: pipe.Name, Data: string(pipe.Data)} // } + netrc, err := forge.Netrc(user, repo) + if err != nil { + return nil, fmt.Errorf("could not get Netrc data from forge: %w", err) + } + response := new(responseStructure) body := requestStructure{ Repo: repo, Pipeline: pipeline, - // Configuration: currentConfigs, + // Configuration: currentConfigs, // TODO: provide the current config to the http service somehow Netrc: netrc, } @@ -64,8 +76,13 @@ func (h *http) Fetch(ctx context.Context, forge *forge.Forge, user *model.User, } if status != 200 { - return []*ConfigData{}, nil + return []*types.FileMeta{}, nil + } + + fileMetas := make([]*types.FileMeta, len(response.Configs)) + for i, config := range response.Configs { + fileMetas[i] = &types.FileMeta{Name: config.Name, Data: []byte(config.Data)} } - return response.Configs, nil + return fileMetas, nil } diff --git a/server/plugins/config/service.go b/server/plugins/config/service.go index 0912091a5e..89bbefc108 100644 --- a/server/plugins/config/service.go +++ b/server/plugins/config/service.go @@ -1,4 +1,4 @@ -// Copyright 2022 Woodpecker Authors +// Copyright 2024 Woodpecker Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -22,12 +22,6 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/model" ) -// ConfigData same as forge.FileMeta but with json tags and string data -type ConfigData struct { - Name string `json:"name"` - Data string `json:"data"` -} - type Service interface { Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (configData []*types.FileMeta, err error) } diff --git a/server/plugins/environments/parse_test.go b/server/plugins/environments/parse_test.go index 9e1feacc49..d393fe1b72 100644 --- a/server/plugins/environments/parse_test.go +++ b/server/plugins/environments/parse_test.go @@ -1,3 +1,17 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package environments import ( diff --git a/server/plugins/environments/service.go b/server/plugins/environments/service.go index 1d9ef80b92..642f2d1ae3 100644 --- a/server/plugins/environments/service.go +++ b/server/plugins/environments/service.go @@ -1,3 +1,17 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package environments import "go.woodpecker-ci.org/woodpecker/v2/server/model" diff --git a/server/plugins/manager.go b/server/plugins/manager.go index 19dd567c76..8c8c8c11c1 100644 --- a/server/plugins/manager.go +++ b/server/plugins/manager.go @@ -1,10 +1,24 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package plugins import ( "crypto" "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/forge" + "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/environments" @@ -22,13 +36,13 @@ type Manager struct { signaturePublicKey crypto.PublicKey } -func NewManager(c *cli.Context, store store.Store, forge forge.Forge) (*Manager, error) { +func NewManager(c *cli.Context, store store.Store) (*Manager, error) { signaturePrivateKey, signaturePublicKey, err := setupSignatureKeys(store) if err != nil { return nil, err } - config, err := setupConfigService(c, store, signaturePrivateKey) + config, err := setupConfigService(c, signaturePrivateKey) if err != nil { return nil, err } diff --git a/server/plugins/registry/service.go b/server/plugins/registry/service.go index 03f269a82c..252ce9d9a9 100644 --- a/server/plugins/registry/service.go +++ b/server/plugins/registry/service.go @@ -1,3 +1,17 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package registry import "go.woodpecker-ci.org/woodpecker/v2/server/model" diff --git a/server/plugins/secret/service.go b/server/plugins/secret/service.go index d8a9c3bc72..24568c195b 100644 --- a/server/plugins/secret/service.go +++ b/server/plugins/secret/service.go @@ -1,3 +1,17 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package secret import "go.woodpecker-ci.org/woodpecker/v2/server/model" diff --git a/server/plugins/setup.go b/server/plugins/setup.go index 6f11165197..aba7e4a001 100644 --- a/server/plugins/setup.go +++ b/server/plugins/setup.go @@ -1,3 +1,17 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + package plugins import ( @@ -10,6 +24,7 @@ import ( "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/registry" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secret" @@ -43,7 +58,7 @@ func setupSecretExtension(store store.Store) secret.Service { return secret.NewDB(store) } -func setupConfigService(c *cli.Context, store store.Store, privateSignatureKey crypto.PrivateKey) (config.Service, error) { +func setupConfigService(c *cli.Context, privateSignatureKey crypto.PrivateKey) (config.Service, error) { addonExt, err := addon.Load[config.Service](c.StringSlice("addons"), addonTypes.TypeConfigService) if err != nil { return nil, err @@ -56,7 +71,9 @@ func setupConfigService(c *cli.Context, store store.Store, privateSignatureKey c return config.NewHTTP(endpoint, privateSignatureKey), nil } - return nil, nil + timeout := c.Duration("forge-timeout") + config := config.NewForge(timeout) + return config, nil } // setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for extensions) From f2e43ee9e901bdfad25ae6404665c0762056cab5 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Wed, 7 Feb 2024 23:47:18 +0100 Subject: [PATCH 06/28] undo unrelated change --- server/model/repo.go | 6 ------ 1 file changed, 6 deletions(-) diff --git a/server/model/repo.go b/server/model/repo.go index e2748a8c1c..34d50bb508 100644 --- a/server/model/repo.go +++ b/server/model/repo.go @@ -49,9 +49,6 @@ type Repo struct { Perm *Perm `json:"-" xorm:"-"` CancelPreviousPipelineEvents []WebhookEvent `json:"cancel_previous_pipeline_events" xorm:"json 'cancel_previous_pipeline_events'"` NetrcOnlyTrusted bool `json:"netrc_only_trusted" xorm:"NOT NULL DEFAULT true 'netrc_only_trusted'"` - SecretEndpoint string `json:"secret_endpoint" xorm:"varchar(500) 'secret_endpoint'"` - RegistryEndpoint string `json:"registry_endpoint" xorm:"varchar(500) 'registry_endpoint'"` - ConfigEndpoint string `json:"config_endpoint" xorm:"varchar(500) 'config_endpoint'"` } // @name Repo // TableName return database table name for xorm @@ -117,9 +114,6 @@ type RepoPatch struct { AllowPull *bool `json:"allow_pr,omitempty"` CancelPreviousPipelineEvents *[]WebhookEvent `json:"cancel_previous_pipeline_events"` NetrcOnlyTrusted *bool `json:"netrc_only_trusted"` - ConfigEndpoint *string `json:"config_endpoint,omitempty"` - SecretEndpoint *string `json:"secret_endpoint,omitempty"` - RegistryEndpoint *string `json:"registry_endpoint,omitempty"` } // @name RepoPatch type ForgeRemoteID string From b094c9c685f9a55bae0bfa11a387bf9291366395 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Wed, 7 Feb 2024 23:50:04 +0100 Subject: [PATCH 07/28] rename folder --- server/plugins/{environments => environment}/parse.go | 2 +- .../plugins/{environments => environment}/parse_test.go | 2 +- server/plugins/{environments => environment}/service.go | 2 +- server/plugins/manager.go | 8 ++++---- 4 files changed, 7 insertions(+), 7 deletions(-) rename server/plugins/{environments => environment}/parse.go (98%) rename server/plugins/{environments => environment}/parse_test.go (98%) rename server/plugins/{environments => environment}/service.go (97%) diff --git a/server/plugins/environments/parse.go b/server/plugins/environment/parse.go similarity index 98% rename from server/plugins/environments/parse.go rename to server/plugins/environment/parse.go index a4990c3450..36de3cb5b1 100644 --- a/server/plugins/environments/parse.go +++ b/server/plugins/environment/parse.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package environments +package environment import ( "strings" diff --git a/server/plugins/environments/parse_test.go b/server/plugins/environment/parse_test.go similarity index 98% rename from server/plugins/environments/parse_test.go rename to server/plugins/environment/parse_test.go index d393fe1b72..97e7575698 100644 --- a/server/plugins/environments/parse_test.go +++ b/server/plugins/environment/parse_test.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package environments +package environment import ( "testing" diff --git a/server/plugins/environments/service.go b/server/plugins/environment/service.go similarity index 97% rename from server/plugins/environments/service.go rename to server/plugins/environment/service.go index 642f2d1ae3..f8ad148215 100644 --- a/server/plugins/environments/service.go +++ b/server/plugins/environment/service.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package environments +package environment import "go.woodpecker-ci.org/woodpecker/v2/server/model" diff --git a/server/plugins/manager.go b/server/plugins/manager.go index 8c8c8c11c1..5f1e83f790 100644 --- a/server/plugins/manager.go +++ b/server/plugins/manager.go @@ -21,7 +21,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/environments" + "go.woodpecker-ci.org/woodpecker/v2/server/plugins/environment" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/registry" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secret" "go.woodpecker-ci.org/woodpecker/v2/server/store" @@ -31,7 +31,7 @@ type Manager struct { secret secret.Service registry registry.Service config config.Service - environment environments.Service + environment environment.Service signaturePrivateKey crypto.PrivateKey signaturePublicKey crypto.PublicKey } @@ -53,7 +53,7 @@ func NewManager(c *cli.Context, store store.Store) (*Manager, error) { secret: setupSecretExtension(store), registry: setupRegistryExtension(store, c.String("docker-config")), config: config, - environment: environments.Parse(c.StringSlice("environment")), + environment: environment.Parse(c.StringSlice("environment")), }, nil } @@ -81,6 +81,6 @@ func (e *Manager) ConfigServiceFromRepo(_ *model.Repo) config.Service { return e.config } -func (e *Manager) EnvironmentService() environments.Service { +func (e *Manager) EnvironmentService() environment.Service { return e.environment } From dc0be46bf91e75892bbb59be49955bf1c73d36f2 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Wed, 7 Feb 2024 23:52:09 +0100 Subject: [PATCH 08/28] fixes --- cmd/server/server.go | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/cmd/server/server.go b/cmd/server/server.go index 3481fef687..4f30b8f2ed 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -43,9 +43,6 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/logging" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/plugins" - - // "go.woodpecker-ci.org/woodpecker/v2/server/plugins/encryption" - // encryptedStore "go.woodpecker-ci.org/woodpecker/v2/server/plugins/encryption/wrapper/store" "go.woodpecker-ci.org/woodpecker/v2/server/plugins/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/pubsub" "go.woodpecker-ci.org/woodpecker/v2/server/router" @@ -283,7 +280,7 @@ func setupEvilGlobals(c *cli.Context, s store.Store, f forge.Forge) error { server.Config.Services.Pubsub = pubsub.New() server.Config.Services.Membership = setupMembershipService(c, f) - pluginManager, err := plugins.NewManager(c, s, f) + pluginManager, err := plugins.NewManager(c, s) if err != nil { return fmt.Errorf("could not setup plugin manager: %w", err) } From 59f237a829cfce0d2b3338e5fb8847c5e851ca5b Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Wed, 7 Feb 2024 23:58:45 +0100 Subject: [PATCH 09/28] cleanup --- server/plugins/config/combined.go | 69 ----------------------------- server/plugins/config/forge.go | 33 ++++++++++++++ server/plugins/config/forge_test.go | 16 +++---- 3 files changed, 41 insertions(+), 77 deletions(-) diff --git a/server/plugins/config/combined.go b/server/plugins/config/combined.go index 356502c103..9806ce55fb 100644 --- a/server/plugins/config/combined.go +++ b/server/plugins/config/combined.go @@ -13,72 +13,3 @@ // limitations under the License. package config - -import ( - "context" - "errors" - "strings" - "time" - - "github.com/rs/zerolog/log" - - "go.woodpecker-ci.org/woodpecker/v2/server/forge" - "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" - "go.woodpecker-ci.org/woodpecker/v2/server/model" -) - -type forgeFetcher struct { - timeout time.Duration -} - -func NewForge(timeout time.Duration) Service { - return &forgeFetcher{ - timeout: timeout, - } -} - -func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ([]*types.FileMeta, error) { - var files []*types.FileMeta - var err error - - // if cf.configExtension != nil { - // fetchCtx, cancel := context.WithTimeout(ctx, cf.timeout) - // defer cancel() // ok here as we only try http fetching once, returning on fail and success - - // log.Trace().Msgf("configFetcher[%s]: getting config from external http service", cf.repo.FullName) - // netrc, err := cf.forge.Netrc(cf.user, cf.repo) - // if err != nil { - // return nil, fmt.Errorf("could not get Netrc data from forge: %w", err) - // } - - // newConfigs, useOld, err := cf.configExtension.FetchConfig(fetchCtx, cf.repo, cf.pipeline, files, netrc) - // if err != nil { - // log.Error().Err(err).Msg("could not fetch config via http") - // return nil, fmt.Errorf("could not fetch config via http: %w", err) - // } - - // if !useOld { - // return newConfigs, nil - // } - // } - - // try to fetch 3 times - configFetcher := &forgeFetcherContext{ - forge: forge, - user: user, - repo: repo, - pipeline: pipeline, - timeout: f.timeout, - } - for i := 0; i < 3; i++ { - files, err = configFetcher.fetch(ctx, strings.TrimSpace(repo.Config)) - if err != nil { - log.Trace().Err(err).Msgf("%d. try failed", i+1) - } - if errors.Is(err, context.DeadlineExceeded) { - continue - } - } - - return files, err -} diff --git a/server/plugins/config/forge.go b/server/plugins/config/forge.go index 745a07373b..a5f8fe1180 100644 --- a/server/plugins/config/forge.go +++ b/server/plugins/config/forge.go @@ -29,6 +29,39 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/shared/constant" ) +type forgeFetcher struct { + timeout time.Duration +} + +func NewForge(timeout time.Duration) Service { + return &forgeFetcher{ + timeout: timeout, + } +} + +func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (files []*types.FileMeta, err error) { + configFetcher := &forgeFetcherContext{ + forge: forge, + user: user, + repo: repo, + pipeline: pipeline, + timeout: f.timeout, + } + + // try to fetch 3 times + for i := 0; i < 3; i++ { + files, err = configFetcher.fetch(ctx, strings.TrimSpace(repo.Config)) + if err != nil { + log.Trace().Err(err).Msgf("%d. try failed", i+1) + } + if errors.Is(err, context.DeadlineExceeded) { + continue + } + } + + return +} + type forgeFetcherContext struct { forge forge.Forge user *model.User diff --git a/server/plugins/config/forge_test.go b/server/plugins/config/forge_test.go index a346c65e21..710de1f9c6 100644 --- a/server/plugins/config/forge_test.go +++ b/server/plugins/config/forge_test.go @@ -312,15 +312,16 @@ func TestFetch(t *testing.T) { f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("file not found")) f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("directory not found")) - configFetcher := config.NewConfigFetcher( + configFetcher := config.NewForge( + time.Second * 3, + ) + files, err := configFetcher.Fetch( + context.Background(), f, - time.Second*3, - nil, &model.User{Token: "xxx"}, repo, &model.Pipeline{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"}, ) - files, err := configFetcher.Fetch(context.Background()) if tt.expectedError && err == nil { t.Fatal("expected an error") } else if !tt.expectedError && err != nil { @@ -520,15 +521,14 @@ func TestFetchFromConfigService(t *testing.T) { f.On("Netrc", mock.Anything, mock.Anything).Return(&model.Netrc{Machine: "mock", Login: "mock", Password: "mock"}, nil) - configFetcher := config.NewConfigFetcher( + configFetcher := config.NewForge(time.Second * 3) + files, err := configFetcher.Fetch( + context.Background(), f, - time.Second*3, - configAPI, &model.User{Token: "xxx"}, repo, &model.Pipeline{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"}, ) - files, err := configFetcher.Fetch(context.Background()) if tt.expectedError && err == nil { t.Fatal("expected an error") } else if !tt.expectedError && err != nil { From ffe4c90f2899f7e41754d2e6865f793690436e5e Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Thu, 8 Feb 2024 00:04:56 +0100 Subject: [PATCH 10/28] rename plugins to services --- cmd/server/server.go | 10 +- .../75-addons/20-creating-addons.md | 8 +- server/config.go | 6 +- .../{plugins => services}/config/combined.go | 0 server/services/config/combined_test.go | 244 ++++++++++++++++++ server/{plugins => services}/config/forge.go | 0 .../config/forge_test.go | 216 +--------------- server/{plugins => services}/config/http.go | 2 +- .../{plugins => services}/config/service.go | 0 .../{plugins => services}/encryption/aes.go | 0 .../encryption/aes_builder.go | 0 .../encryption/aes_encryption.go | 0 .../encryption/aes_state.go | 0 .../encryption/aes_test.go | 0 .../encryption/constants.go | 0 .../encryption/encryption.go | 0 .../encryption/encryption_builder.go | 0 .../encryption/no_encryption.go | 0 .../{plugins => services}/encryption/tink.go | 0 .../encryption/tink_builder.go | 0 .../encryption/tink_keyset.go | 0 .../encryption/tink_keyset_watcher.go | 0 .../encryption/tink_state.go | 0 .../encryption/wrapper/store/constants.go | 0 .../encryption/wrapper/store/secret_store.go | 0 .../wrapper/store/secret_store_wrapper.go | 0 .../environment/parse.go | 0 .../environment/parse_test.go | 0 .../environment/service.go | 0 server/{plugins => services}/manager.go | 10 +- .../permissions/admins.go | 0 .../permissions/admins_test.go | 0 .../{plugins => services}/permissions/orgs.go | 0 .../permissions/orgs_test.go | 0 .../permissions/repo_owners.go | 0 .../permissions/repo_owners_test.go | 0 .../registry/combined.go | 0 server/{plugins => services}/registry/db.go | 0 .../registry/filesystem.go | 0 .../{plugins => services}/registry/service.go | 0 server/{plugins => services}/secret/db.go | 0 .../{plugins => services}/secret/db_test.go | 2 +- .../{plugins => services}/secret/service.go | 0 server/{plugins => services}/setup.go | 8 +- server/{plugins => services}/utils/http.go | 0 .../{plugins => services}/utils/http_test.go | 2 +- 46 files changed, 269 insertions(+), 239 deletions(-) rename server/{plugins => services}/config/combined.go (100%) create mode 100644 server/services/config/combined_test.go rename server/{plugins => services}/config/forge.go (100%) rename server/{plugins => services}/config/forge_test.go (59%) rename server/{plugins => services}/config/http.go (97%) rename server/{plugins => services}/config/service.go (100%) rename server/{plugins => services}/encryption/aes.go (100%) rename server/{plugins => services}/encryption/aes_builder.go (100%) rename server/{plugins => services}/encryption/aes_encryption.go (100%) rename server/{plugins => services}/encryption/aes_state.go (100%) rename server/{plugins => services}/encryption/aes_test.go (100%) rename server/{plugins => services}/encryption/constants.go (100%) rename server/{plugins => services}/encryption/encryption.go (100%) rename server/{plugins => services}/encryption/encryption_builder.go (100%) rename server/{plugins => services}/encryption/no_encryption.go (100%) rename server/{plugins => services}/encryption/tink.go (100%) rename server/{plugins => services}/encryption/tink_builder.go (100%) rename server/{plugins => services}/encryption/tink_keyset.go (100%) rename server/{plugins => services}/encryption/tink_keyset_watcher.go (100%) rename server/{plugins => services}/encryption/tink_state.go (100%) rename server/{plugins => services}/encryption/wrapper/store/constants.go (100%) rename server/{plugins => services}/encryption/wrapper/store/secret_store.go (100%) rename server/{plugins => services}/encryption/wrapper/store/secret_store_wrapper.go (100%) rename server/{plugins => services}/environment/parse.go (100%) rename server/{plugins => services}/environment/parse_test.go (100%) rename server/{plugins => services}/environment/service.go (100%) rename server/{plugins => services}/manager.go (89%) rename server/{plugins => services}/permissions/admins.go (100%) rename server/{plugins => services}/permissions/admins_test.go (100%) rename server/{plugins => services}/permissions/orgs.go (100%) rename server/{plugins => services}/permissions/orgs_test.go (100%) rename server/{plugins => services}/permissions/repo_owners.go (100%) rename server/{plugins => services}/permissions/repo_owners_test.go (100%) rename server/{plugins => services}/registry/combined.go (100%) rename server/{plugins => services}/registry/db.go (100%) rename server/{plugins => services}/registry/filesystem.go (100%) rename server/{plugins => services}/registry/service.go (100%) rename server/{plugins => services}/secret/db.go (100%) rename server/{plugins => services}/secret/db_test.go (97%) rename server/{plugins => services}/secret/service.go (100%) rename server/{plugins => services}/setup.go (94%) rename server/{plugins => services}/utils/http.go (100%) rename server/{plugins => services}/utils/http_test.go (96%) diff --git a/cmd/server/server.go b/cmd/server/server.go index 4f30b8f2ed..02813287b7 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -42,11 +42,11 @@ import ( woodpeckerGrpcServer "go.woodpecker-ci.org/woodpecker/v2/server/grpc" "go.woodpecker-ci.org/woodpecker/v2/server/logging" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/pubsub" "go.woodpecker-ci.org/woodpecker/v2/server/router" "go.woodpecker-ci.org/woodpecker/v2/server/router/middleware" + "go.woodpecker-ci.org/woodpecker/v2/server/services" + "go.woodpecker-ci.org/woodpecker/v2/server/services/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/store" "go.woodpecker-ci.org/woodpecker/v2/server/web" "go.woodpecker-ci.org/woodpecker/v2/shared/constant" @@ -280,11 +280,11 @@ func setupEvilGlobals(c *cli.Context, s store.Store, f forge.Forge) error { server.Config.Services.Pubsub = pubsub.New() server.Config.Services.Membership = setupMembershipService(c, f) - pluginManager, err := plugins.NewManager(c, s) + servicesManager, err := services.NewManager(c, s) if err != nil { - return fmt.Errorf("could not setup plugin manager: %w", err) + return fmt.Errorf("could not setup services manager: %w", err) } - server.Config.Services.Manager = pluginManager + server.Config.Services.Manager = servicesManager // authentication server.Config.Pipeline.AuthenticatePublicRepos = c.Bool("authenticate-public-repos") diff --git a/docs/docs/30-administration/75-addons/20-creating-addons.md b/docs/docs/30-administration/75-addons/20-creating-addons.md index 0fd6d2fe07..25b90bc430 100644 --- a/docs/docs/30-administration/75-addons/20-creating-addons.md +++ b/docs/docs/30-administration/75-addons/20-creating-addons.md @@ -23,10 +23,10 @@ Directly import Woodpecker's Go package (`go.woodpecker-ci.org/woodpecker/woodpe | -------------------- | -------------------------------------------------------------------------------- | | `Forge` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge` | | `Backend` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/pipeline/backend/types".Backend` | -| `ConfigService` | `"go.woodpecker-ci.org/woodpecker/v2/server/plugins/config".Extension` | -| `SecretService` | `"go.woodpecker-ci.org/woodpecker/v2/server/model".SecretService` | -| `EnvironmentService` | `"go.woodpecker-ci.org/woodpecker/v2/server/model".EnvironmentService` | -| `RegistryService` | `"go.woodpecker-ci.org/woodpecker/v2/server/model".RegistryService` | +| `ConfigService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/config".Extension` | +| `SecretService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/secret".SecretService` | +| `EnvironmentService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/environment".EnvironmentService` | +| `RegistryService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/registry".RegistryService` | ### Using configurations diff --git a/server/config.go b/server/config.go index c92286a81a..5015c5483b 100644 --- a/server/config.go +++ b/server/config.go @@ -24,10 +24,10 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/logging" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/pubsub" "go.woodpecker-ci.org/woodpecker/v2/server/queue" + "go.woodpecker-ci.org/woodpecker/v2/server/services" + "go.woodpecker-ci.org/woodpecker/v2/server/services/permissions" ) var Config = struct { @@ -37,7 +37,7 @@ var Config = struct { Logs logging.Log Forge forge.Forge Membership cache.MembershipService - Manager *plugins.Manager + Manager *services.Manager } Storage struct { // Users model.UserStore diff --git a/server/plugins/config/combined.go b/server/services/config/combined.go similarity index 100% rename from server/plugins/config/combined.go rename to server/services/config/combined.go diff --git a/server/services/config/combined_test.go b/server/services/config/combined_test.go new file mode 100644 index 0000000000..61ecee7f31 --- /dev/null +++ b/server/services/config/combined_test.go @@ -0,0 +1,244 @@ +// Copyright 2024 Woodpecker Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package config_test + +import ( + "context" + "crypto/ed25519" + "crypto/rand" + "encoding/json" + "fmt" + "io" + "net/http" + "net/http/httptest" + "path/filepath" + "testing" + "time" + + "github.com/go-ap/httpsig" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/mock" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/mocks" + forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" + "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/config" +) + +func TestFetchFromConfigService(t *testing.T) { + t.Parallel() + + type file struct { + name string + data []byte + } + + dummyData := []byte("TEST") + + testTable := []struct { + name string + repoConfig string + files []file + expectedFileNames []string + expectedError bool + }{ + { + name: "External Fetch empty repo", + repoConfig: "", + files: []file{}, + expectedFileNames: []string{"override1", "override2", "override3"}, + expectedError: false, + }, + { + name: "Default config - Additional sub-folders", + repoConfig: "", + files: []file{{ + name: ".woodpecker/test.yml", + data: dummyData, + }, { + name: ".woodpecker/sub-folder/config.yml", + data: dummyData, + }}, + expectedFileNames: []string{"override1", "override2", "override3"}, + expectedError: false, + }, + { + name: "Fetch empty", + repoConfig: " ", + files: []file{{ + name: ".woodpecker/.keep", + data: dummyData, + }, { + name: ".woodpecker.yml", + data: nil, + }, { + name: ".woodpecker.yaml", + data: dummyData, + }}, + expectedFileNames: []string{}, + expectedError: true, + }, + { + name: "Use old config", + repoConfig: ".my-ci-folder/", + files: []file{{ + name: ".woodpecker/test.yml", + data: dummyData, + }, { + name: ".woodpecker.yml", + data: dummyData, + }, { + name: ".woodpecker.yaml", + data: dummyData, + }, { + name: ".my-ci-folder/test.yml", + data: dummyData, + }}, + expectedFileNames: []string{ + ".my-ci-folder/test.yml", + }, + expectedError: false, + }, + } + + pubEd25519Key, privEd25519Key, err := ed25519.GenerateKey(rand.Reader) + if err != nil { + t.Fatal("can't generate ed25519 key pair") + } + + fixtureHandler := func(w http.ResponseWriter, r *http.Request) { + // check signature + pubKeyID := "woodpecker-ci-plugins" + + keystore := httpsig.NewMemoryKeyStore() + keystore.SetKey(pubKeyID, pubEd25519Key) + + verifier := httpsig.NewVerifier(keystore) + verifier.SetRequiredHeaders([]string{"(request-target)", "date"}) + + keyID, err := verifier.Verify(r) + if err != nil { + http.Error(w, "Invalid signature", http.StatusBadRequest) + return + } + + if keyID != pubKeyID { + http.Error(w, "Used wrong key", http.StatusBadRequest) + return + } + + type config struct { + Name string `json:"name"` + Data string `json:"data"` + } + + type incoming struct { + Repo *model.Repo `json:"repo"` + Build *model.Pipeline `json:"pipeline"` + Configuration []*config `json:"config"` + } + + var req incoming + body, err := io.ReadAll(r.Body) + if err != nil { + http.Error(w, "can't read body", http.StatusBadRequest) + return + } + err = json.Unmarshal(body, &req) + if err != nil { + http.Error(w, "Failed to parse JSON"+err.Error(), http.StatusBadRequest) + return + } + + if req.Repo.Name == "Fetch empty" { + w.WriteHeader(404) + return + } + + if req.Repo.Name == "Use old config" { + w.WriteHeader(204) + return + } + + fmt.Fprint(w, `{ + "configs": [ + { + "name": "override1", + "data": "some new pipelineconfig \n pipe, pipe, pipe" + }, + { + "name": "override2", + "data": "some new pipelineconfig \n pipe, pipe, pipe" + }, + { + "name": "override3", + "data": "some new pipelineconfig \n pipe, pipe, pipe" + } + ] +}`) + } + + ts := httptest.NewServer(http.HandlerFunc(fixtureHandler)) + defer ts.Close() + configAPI := config.NewHTTP(ts.URL, privEd25519Key) + + for _, tt := range testTable { + t.Run(tt.name, func(t *testing.T) { + repo := &model.Repo{Owner: "laszlocph", Name: tt.name, Config: tt.repoConfig} // Using test name as repo name to provide different responses in mock server + + f := new(mocks.Forge) + dirs := map[string][]*forge_types.FileMeta{} + for _, file := range tt.files { + f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, file.name).Return(file.data, nil) + path := filepath.Dir(file.name) + if path != "." { + dirs[path] = append(dirs[path], &forge_types.FileMeta{ + Name: file.name, + Data: file.data, + }) + } + } + + for path, files := range dirs { + f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, path).Return(files, nil) + } + + // if the previous mocks do not match return not found errors + f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("file not found")) + f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("directory not found")) + + f.On("Netrc", mock.Anything, mock.Anything).Return(&model.Netrc{Machine: "mock", Login: "mock", Password: "mock"}, nil) + + configFetcher := config.NewForge(time.Second * 3) + files, err := configFetcher.Fetch( + context.Background(), + f, + &model.User{Token: "xxx"}, + repo, + &model.Pipeline{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"}, + ) + if tt.expectedError && err == nil { + t.Fatal("expected an error") + } else if !tt.expectedError && err != nil { + t.Fatal("error fetching config:", err) + } + + matchingFiles := make([]string, len(files)) + for i := range files { + matchingFiles[i] = files[i].Name + } + assert.ElementsMatch(t, tt.expectedFileNames, matchingFiles, "expected some other pipeline files") + }) + } +} diff --git a/server/plugins/config/forge.go b/server/services/config/forge.go similarity index 100% rename from server/plugins/config/forge.go rename to server/services/config/forge.go diff --git a/server/plugins/config/forge_test.go b/server/services/config/forge_test.go similarity index 59% rename from server/plugins/config/forge_test.go rename to server/services/config/forge_test.go index 710de1f9c6..9826c52616 100644 --- a/server/plugins/config/forge_test.go +++ b/server/services/config/forge_test.go @@ -16,25 +16,18 @@ package config_test import ( "context" - "crypto/ed25519" - "crypto/rand" - "encoding/json" "fmt" - "io" - "net/http" - "net/http/httptest" "path/filepath" "testing" "time" - "github.com/go-ap/httpsig" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" "go.woodpecker-ci.org/woodpecker/v2/server/forge/mocks" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" + "go.woodpecker-ci.org/woodpecker/v2/server/services/config" ) func TestFetch(t *testing.T) { @@ -336,210 +329,3 @@ func TestFetch(t *testing.T) { }) } } - -func TestFetchFromConfigService(t *testing.T) { - t.Parallel() - - type file struct { - name string - data []byte - } - - dummyData := []byte("TEST") - - testTable := []struct { - name string - repoConfig string - files []file - expectedFileNames []string - expectedError bool - }{ - { - name: "External Fetch empty repo", - repoConfig: "", - files: []file{}, - expectedFileNames: []string{"override1", "override2", "override3"}, - expectedError: false, - }, - { - name: "Default config - Additional sub-folders", - repoConfig: "", - files: []file{{ - name: ".woodpecker/test.yml", - data: dummyData, - }, { - name: ".woodpecker/sub-folder/config.yml", - data: dummyData, - }}, - expectedFileNames: []string{"override1", "override2", "override3"}, - expectedError: false, - }, - { - name: "Fetch empty", - repoConfig: " ", - files: []file{{ - name: ".woodpecker/.keep", - data: dummyData, - }, { - name: ".woodpecker.yml", - data: nil, - }, { - name: ".woodpecker.yaml", - data: dummyData, - }}, - expectedFileNames: []string{}, - expectedError: true, - }, - { - name: "Use old config", - repoConfig: ".my-ci-folder/", - files: []file{{ - name: ".woodpecker/test.yml", - data: dummyData, - }, { - name: ".woodpecker.yml", - data: dummyData, - }, { - name: ".woodpecker.yaml", - data: dummyData, - }, { - name: ".my-ci-folder/test.yml", - data: dummyData, - }}, - expectedFileNames: []string{ - ".my-ci-folder/test.yml", - }, - expectedError: false, - }, - } - - pubEd25519Key, privEd25519Key, err := ed25519.GenerateKey(rand.Reader) - if err != nil { - t.Fatal("can't generate ed25519 key pair") - } - - fixtureHandler := func(w http.ResponseWriter, r *http.Request) { - // check signature - pubKeyID := "woodpecker-ci-plugins" - - keystore := httpsig.NewMemoryKeyStore() - keystore.SetKey(pubKeyID, pubEd25519Key) - - verifier := httpsig.NewVerifier(keystore) - verifier.SetRequiredHeaders([]string{"(request-target)", "date"}) - - keyID, err := verifier.Verify(r) - if err != nil { - http.Error(w, "Invalid signature", http.StatusBadRequest) - return - } - - if keyID != pubKeyID { - http.Error(w, "Used wrong key", http.StatusBadRequest) - return - } - - type config struct { - Name string `json:"name"` - Data string `json:"data"` - } - - type incoming struct { - Repo *model.Repo `json:"repo"` - Build *model.Pipeline `json:"pipeline"` - Configuration []*config `json:"config"` - } - - var req incoming - body, err := io.ReadAll(r.Body) - if err != nil { - http.Error(w, "can't read body", http.StatusBadRequest) - return - } - err = json.Unmarshal(body, &req) - if err != nil { - http.Error(w, "Failed to parse JSON"+err.Error(), http.StatusBadRequest) - return - } - - if req.Repo.Name == "Fetch empty" { - w.WriteHeader(404) - return - } - - if req.Repo.Name == "Use old config" { - w.WriteHeader(204) - return - } - - fmt.Fprint(w, `{ - "configs": [ - { - "name": "override1", - "data": "some new pipelineconfig \n pipe, pipe, pipe" - }, - { - "name": "override2", - "data": "some new pipelineconfig \n pipe, pipe, pipe" - }, - { - "name": "override3", - "data": "some new pipelineconfig \n pipe, pipe, pipe" - } - ] -}`) - } - - ts := httptest.NewServer(http.HandlerFunc(fixtureHandler)) - defer ts.Close() - configAPI := config.NewHTTP(ts.URL, privEd25519Key) - - for _, tt := range testTable { - t.Run(tt.name, func(t *testing.T) { - repo := &model.Repo{Owner: "laszlocph", Name: tt.name, Config: tt.repoConfig} // Using test name as repo name to provide different responses in mock server - - f := new(mocks.Forge) - dirs := map[string][]*forge_types.FileMeta{} - for _, file := range tt.files { - f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, file.name).Return(file.data, nil) - path := filepath.Dir(file.name) - if path != "." { - dirs[path] = append(dirs[path], &forge_types.FileMeta{ - Name: file.name, - Data: file.data, - }) - } - } - - for path, files := range dirs { - f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, path).Return(files, nil) - } - - // if the previous mocks do not match return not found errors - f.On("File", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("file not found")) - f.On("Dir", mock.Anything, mock.Anything, mock.Anything, mock.Anything, mock.Anything).Return(nil, fmt.Errorf("directory not found")) - - f.On("Netrc", mock.Anything, mock.Anything).Return(&model.Netrc{Machine: "mock", Login: "mock", Password: "mock"}, nil) - - configFetcher := config.NewForge(time.Second * 3) - files, err := configFetcher.Fetch( - context.Background(), - f, - &model.User{Token: "xxx"}, - repo, - &model.Pipeline{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"}, - ) - if tt.expectedError && err == nil { - t.Fatal("expected an error") - } else if !tt.expectedError && err != nil { - t.Fatal("error fetching config:", err) - } - - matchingFiles := make([]string, len(files)) - for i := range files { - matchingFiles[i] = files[i].Name - } - assert.ElementsMatch(t, tt.expectedFileNames, matchingFiles, "expected some other pipeline files") - }) - } -} diff --git a/server/plugins/config/http.go b/server/services/config/http.go similarity index 97% rename from server/plugins/config/http.go rename to server/services/config/http.go index a26ebfc74a..9df33873dc 100644 --- a/server/plugins/config/http.go +++ b/server/services/config/http.go @@ -22,7 +22,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/utils" + "go.woodpecker-ci.org/woodpecker/v2/server/services/utils" ) type http struct { diff --git a/server/plugins/config/service.go b/server/services/config/service.go similarity index 100% rename from server/plugins/config/service.go rename to server/services/config/service.go diff --git a/server/plugins/encryption/aes.go b/server/services/encryption/aes.go similarity index 100% rename from server/plugins/encryption/aes.go rename to server/services/encryption/aes.go diff --git a/server/plugins/encryption/aes_builder.go b/server/services/encryption/aes_builder.go similarity index 100% rename from server/plugins/encryption/aes_builder.go rename to server/services/encryption/aes_builder.go diff --git a/server/plugins/encryption/aes_encryption.go b/server/services/encryption/aes_encryption.go similarity index 100% rename from server/plugins/encryption/aes_encryption.go rename to server/services/encryption/aes_encryption.go diff --git a/server/plugins/encryption/aes_state.go b/server/services/encryption/aes_state.go similarity index 100% rename from server/plugins/encryption/aes_state.go rename to server/services/encryption/aes_state.go diff --git a/server/plugins/encryption/aes_test.go b/server/services/encryption/aes_test.go similarity index 100% rename from server/plugins/encryption/aes_test.go rename to server/services/encryption/aes_test.go diff --git a/server/plugins/encryption/constants.go b/server/services/encryption/constants.go similarity index 100% rename from server/plugins/encryption/constants.go rename to server/services/encryption/constants.go diff --git a/server/plugins/encryption/encryption.go b/server/services/encryption/encryption.go similarity index 100% rename from server/plugins/encryption/encryption.go rename to server/services/encryption/encryption.go diff --git a/server/plugins/encryption/encryption_builder.go b/server/services/encryption/encryption_builder.go similarity index 100% rename from server/plugins/encryption/encryption_builder.go rename to server/services/encryption/encryption_builder.go diff --git a/server/plugins/encryption/no_encryption.go b/server/services/encryption/no_encryption.go similarity index 100% rename from server/plugins/encryption/no_encryption.go rename to server/services/encryption/no_encryption.go diff --git a/server/plugins/encryption/tink.go b/server/services/encryption/tink.go similarity index 100% rename from server/plugins/encryption/tink.go rename to server/services/encryption/tink.go diff --git a/server/plugins/encryption/tink_builder.go b/server/services/encryption/tink_builder.go similarity index 100% rename from server/plugins/encryption/tink_builder.go rename to server/services/encryption/tink_builder.go diff --git a/server/plugins/encryption/tink_keyset.go b/server/services/encryption/tink_keyset.go similarity index 100% rename from server/plugins/encryption/tink_keyset.go rename to server/services/encryption/tink_keyset.go diff --git a/server/plugins/encryption/tink_keyset_watcher.go b/server/services/encryption/tink_keyset_watcher.go similarity index 100% rename from server/plugins/encryption/tink_keyset_watcher.go rename to server/services/encryption/tink_keyset_watcher.go diff --git a/server/plugins/encryption/tink_state.go b/server/services/encryption/tink_state.go similarity index 100% rename from server/plugins/encryption/tink_state.go rename to server/services/encryption/tink_state.go diff --git a/server/plugins/encryption/wrapper/store/constants.go b/server/services/encryption/wrapper/store/constants.go similarity index 100% rename from server/plugins/encryption/wrapper/store/constants.go rename to server/services/encryption/wrapper/store/constants.go diff --git a/server/plugins/encryption/wrapper/store/secret_store.go b/server/services/encryption/wrapper/store/secret_store.go similarity index 100% rename from server/plugins/encryption/wrapper/store/secret_store.go rename to server/services/encryption/wrapper/store/secret_store.go diff --git a/server/plugins/encryption/wrapper/store/secret_store_wrapper.go b/server/services/encryption/wrapper/store/secret_store_wrapper.go similarity index 100% rename from server/plugins/encryption/wrapper/store/secret_store_wrapper.go rename to server/services/encryption/wrapper/store/secret_store_wrapper.go diff --git a/server/plugins/environment/parse.go b/server/services/environment/parse.go similarity index 100% rename from server/plugins/environment/parse.go rename to server/services/environment/parse.go diff --git a/server/plugins/environment/parse_test.go b/server/services/environment/parse_test.go similarity index 100% rename from server/plugins/environment/parse_test.go rename to server/services/environment/parse_test.go diff --git a/server/plugins/environment/service.go b/server/services/environment/service.go similarity index 100% rename from server/plugins/environment/service.go rename to server/services/environment/service.go diff --git a/server/plugins/manager.go b/server/services/manager.go similarity index 89% rename from server/plugins/manager.go rename to server/services/manager.go index 5f1e83f790..0c022cd1ec 100644 --- a/server/plugins/manager.go +++ b/server/services/manager.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package plugins +package services import ( "crypto" @@ -20,10 +20,10 @@ import ( "github.com/urfave/cli/v2" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/environment" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/registry" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secret" + "go.woodpecker-ci.org/woodpecker/v2/server/services/config" + "go.woodpecker-ci.org/woodpecker/v2/server/services/environment" + "go.woodpecker-ci.org/woodpecker/v2/server/services/registry" + "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) diff --git a/server/plugins/permissions/admins.go b/server/services/permissions/admins.go similarity index 100% rename from server/plugins/permissions/admins.go rename to server/services/permissions/admins.go diff --git a/server/plugins/permissions/admins_test.go b/server/services/permissions/admins_test.go similarity index 100% rename from server/plugins/permissions/admins_test.go rename to server/services/permissions/admins_test.go diff --git a/server/plugins/permissions/orgs.go b/server/services/permissions/orgs.go similarity index 100% rename from server/plugins/permissions/orgs.go rename to server/services/permissions/orgs.go diff --git a/server/plugins/permissions/orgs_test.go b/server/services/permissions/orgs_test.go similarity index 100% rename from server/plugins/permissions/orgs_test.go rename to server/services/permissions/orgs_test.go diff --git a/server/plugins/permissions/repo_owners.go b/server/services/permissions/repo_owners.go similarity index 100% rename from server/plugins/permissions/repo_owners.go rename to server/services/permissions/repo_owners.go diff --git a/server/plugins/permissions/repo_owners_test.go b/server/services/permissions/repo_owners_test.go similarity index 100% rename from server/plugins/permissions/repo_owners_test.go rename to server/services/permissions/repo_owners_test.go diff --git a/server/plugins/registry/combined.go b/server/services/registry/combined.go similarity index 100% rename from server/plugins/registry/combined.go rename to server/services/registry/combined.go diff --git a/server/plugins/registry/db.go b/server/services/registry/db.go similarity index 100% rename from server/plugins/registry/db.go rename to server/services/registry/db.go diff --git a/server/plugins/registry/filesystem.go b/server/services/registry/filesystem.go similarity index 100% rename from server/plugins/registry/filesystem.go rename to server/services/registry/filesystem.go diff --git a/server/plugins/registry/service.go b/server/services/registry/service.go similarity index 100% rename from server/plugins/registry/service.go rename to server/services/registry/service.go diff --git a/server/plugins/secret/db.go b/server/services/secret/db.go similarity index 100% rename from server/plugins/secret/db.go rename to server/services/secret/db.go diff --git a/server/plugins/secret/db_test.go b/server/services/secret/db_test.go similarity index 97% rename from server/plugins/secret/db_test.go rename to server/services/secret/db_test.go index 979d4124ed..6f6099f57f 100644 --- a/server/plugins/secret/db_test.go +++ b/server/services/secret/db_test.go @@ -21,7 +21,7 @@ import ( "github.com/stretchr/testify/mock" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secret" + "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" mocks_store "go.woodpecker-ci.org/woodpecker/v2/server/store/mocks" ) diff --git a/server/plugins/secret/service.go b/server/services/secret/service.go similarity index 100% rename from server/plugins/secret/service.go rename to server/services/secret/service.go diff --git a/server/plugins/setup.go b/server/services/setup.go similarity index 94% rename from server/plugins/setup.go rename to server/services/setup.go index aba7e4a001..e156d1f41e 100644 --- a/server/plugins/setup.go +++ b/server/services/setup.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package plugins +package services import ( "crypto" @@ -25,9 +25,9 @@ import ( "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/config" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/registry" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/secret" + "go.woodpecker-ci.org/woodpecker/v2/server/services/config" + "go.woodpecker-ci.org/woodpecker/v2/server/services/registry" + "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" "go.woodpecker-ci.org/woodpecker/v2/server/store" "go.woodpecker-ci.org/woodpecker/v2/server/store/types" "go.woodpecker-ci.org/woodpecker/v2/shared/addon" diff --git a/server/plugins/utils/http.go b/server/services/utils/http.go similarity index 100% rename from server/plugins/utils/http.go rename to server/services/utils/http.go diff --git a/server/plugins/utils/http_test.go b/server/services/utils/http_test.go similarity index 96% rename from server/plugins/utils/http_test.go rename to server/services/utils/http_test.go index 723a20a6a4..272be5203d 100644 --- a/server/plugins/utils/http_test.go +++ b/server/services/utils/http_test.go @@ -25,7 +25,7 @@ import ( "github.com/go-ap/httpsig" "github.com/stretchr/testify/assert" - "go.woodpecker-ci.org/woodpecker/v2/server/plugins/utils" + "go.woodpecker-ci.org/woodpecker/v2/server/services/utils" ) func TestSign(t *testing.T) { From b87ceb23a4ac2b4ce3165b19d75db0212d200788 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Thu, 8 Feb 2024 00:18:22 +0100 Subject: [PATCH 11/28] cleanup --- server/services/config/combined.go | 49 +++++++++++++++++++++++++ server/services/config/combined_test.go | 6 ++- server/services/config/forge.go | 4 +- server/services/config/http.go | 29 ++++++++------- server/services/setup.go | 10 +++-- 5 files changed, 77 insertions(+), 21 deletions(-) diff --git a/server/services/config/combined.go b/server/services/config/combined.go index 9806ce55fb..b15ab96ea7 100644 --- a/server/services/config/combined.go +++ b/server/services/config/combined.go @@ -13,3 +13,52 @@ // limitations under the License. package config + +import ( + "context" + + "github.com/rs/zerolog/log" + + "go.woodpecker-ci.org/woodpecker/v2/server/forge" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" + "go.woodpecker-ci.org/woodpecker/v2/server/model" +) + +type combined struct { + forgeService Service + httpService Service +} + +func NewCombined(forgeService, httpService Service) Service { + return &combined{ + forgeService: forgeService, + httpService: httpService, + } +} + +func (c *combined) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (files []*types.FileMeta, err error) { + files, err = c.forgeService.Fetch(ctx, forge, user, repo, pipeline) + if err != nil { + return nil, err + } + + if c.httpService != nil { + // TODO(anbraten): This is a hack to get the current configs into the http service + _httpService, ok := c.httpService.(*http) + if !ok { + log.Err(err).Msg("http service is not of type http") + return files, nil + } + _httpService.currentConfigs = files + + httpFiles, err := c.httpService.Fetch(ctx, forge, user, repo, pipeline) + if err != nil { + log.Err(err).Msg("failed to fetch config from http service using forge config instead") + return files, nil + } + + files = httpFiles + } + + return files, nil +} diff --git a/server/services/config/combined_test.go b/server/services/config/combined_test.go index 61ecee7f31..b4389e6573 100644 --- a/server/services/config/combined_test.go +++ b/server/services/config/combined_test.go @@ -30,6 +30,7 @@ import ( "github.com/go-ap/httpsig" "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "go.woodpecker-ci.org/woodpecker/v2/server/forge/mocks" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" @@ -191,7 +192,7 @@ func TestFetchFromConfigService(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(fixtureHandler)) defer ts.Close() - configAPI := config.NewHTTP(ts.URL, privEd25519Key) + httpFetcher := config.NewHTTP(ts.URL, privEd25519Key, nil) for _, tt := range testTable { t.Run(tt.name, func(t *testing.T) { @@ -220,7 +221,8 @@ func TestFetchFromConfigService(t *testing.T) { f.On("Netrc", mock.Anything, mock.Anything).Return(&model.Netrc{Machine: "mock", Login: "mock", Password: "mock"}, nil) - configFetcher := config.NewForge(time.Second * 3) + forgeFetcher := config.NewForge(time.Second * 3) + configFetcher := config.NewCombined(forgeFetcher, httpFetcher) files, err := configFetcher.Fetch( context.Background(), f, diff --git a/server/services/config/forge.go b/server/services/config/forge.go index a5f8fe1180..f1965e30ab 100644 --- a/server/services/config/forge.go +++ b/server/services/config/forge.go @@ -40,7 +40,7 @@ func NewForge(timeout time.Duration) Service { } func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (files []*types.FileMeta, err error) { - configFetcher := &forgeFetcherContext{ + ffc := &forgeFetcherContext{ forge: forge, user: user, repo: repo, @@ -50,7 +50,7 @@ func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model // try to fetch 3 times for i := 0; i < 3; i++ { - files, err = configFetcher.fetch(ctx, strings.TrimSpace(repo.Config)) + files, err = ffc.fetch(ctx, strings.TrimSpace(repo.Config)) if err != nil { log.Trace().Err(err).Msgf("%d. try failed", i+1) } diff --git a/server/services/config/http.go b/server/services/config/http.go index 9df33873dc..307fa62828 100644 --- a/server/services/config/http.go +++ b/server/services/config/http.go @@ -26,8 +26,9 @@ import ( ) type http struct { - endpoint string - privateKey crypto.PrivateKey + endpoint string + privateKey crypto.PrivateKey + currentConfigs []*types.FileMeta } // configData same as forge.FileMeta but with json tags and string data @@ -40,22 +41,24 @@ type requestStructure struct { Repo *model.Repo `json:"repo"` Pipeline *model.Pipeline `json:"pipeline"` Netrc *model.Netrc `json:"netrc"` - // Configuration []*configData `json:"configs"` + + // @deprecated use netrc data to fetch the config by yourself instead + DeprecatedConfiguration []*configData `json:"configs"` // TODO: remove at some point } type responseStructure struct { Configs []*configData `json:"configs"` } -func NewHTTP(endpoint string, privateKey crypto.PrivateKey) Service { - return &http{endpoint, privateKey} +func NewHTTP(endpoint string, privateKey crypto.PrivateKey, currentConfigs []*types.FileMeta) Service { + return &http{endpoint, privateKey, currentConfigs} } func (h *http) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ([]*types.FileMeta, error) { - // currentConfigs := make([]*configData, len(currentFileMeta)) - // for i, pipe := range currentFileMeta { - // currentConfigs[i] = &configData{Name: pipe.Name, Data: string(pipe.Data)} - // } + currentConfigs := make([]*configData, len(h.currentConfigs)) + for i, pipe := range h.currentConfigs { + currentConfigs[i] = &configData{Name: pipe.Name, Data: string(pipe.Data)} + } netrc, err := forge.Netrc(user, repo) if err != nil { @@ -64,10 +67,10 @@ func (h *http) Fetch(ctx context.Context, forge forge.Forge, user *model.User, r response := new(responseStructure) body := requestStructure{ - Repo: repo, - Pipeline: pipeline, - // Configuration: currentConfigs, // TODO: provide the current config to the http service somehow - Netrc: netrc, + Repo: repo, + Pipeline: pipeline, + DeprecatedConfiguration: currentConfigs, + Netrc: netrc, } status, err := utils.Send(ctx, "POST", h.endpoint, h.privateKey, body, response) diff --git a/server/services/setup.go b/server/services/setup.go index e156d1f41e..845d936005 100644 --- a/server/services/setup.go +++ b/server/services/setup.go @@ -67,13 +67,15 @@ func setupConfigService(c *cli.Context, privateSignatureKey crypto.PrivateKey) ( return addonExt.Value, nil } + timeout := c.Duration("forge-timeout") + configFetcher := config.NewForge(timeout) + if endpoint := c.String("config-service-endpoint"); endpoint != "" { - return config.NewHTTP(endpoint, privateSignatureKey), nil + httpFetcher := config.NewHTTP(endpoint, privateSignatureKey, nil) + return config.NewCombined(configFetcher, httpFetcher), nil } - timeout := c.Duration("forge-timeout") - config := config.NewForge(timeout) - return config, nil + return configFetcher, nil } // setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for extensions) From a85fdf077d309c57c1833e1c27b1e029cd29ac09 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Thu, 8 Feb 2024 00:21:07 +0100 Subject: [PATCH 12/28] improve --- server/services/config/combined.go | 33 ++++++++---------------------- 1 file changed, 9 insertions(+), 24 deletions(-) diff --git a/server/services/config/combined.go b/server/services/config/combined.go index b15ab96ea7..8312b98735 100644 --- a/server/services/config/combined.go +++ b/server/services/config/combined.go @@ -25,40 +25,25 @@ import ( ) type combined struct { - forgeService Service - httpService Service + serives []Service } -func NewCombined(forgeService, httpService Service) Service { - return &combined{ - forgeService: forgeService, - httpService: httpService, - } +func NewCombined(services ...Service) Service { + return &combined{serives: services} } func (c *combined) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (files []*types.FileMeta, err error) { - files, err = c.forgeService.Fetch(ctx, forge, user, repo, pipeline) - if err != nil { - return nil, err - } - - if c.httpService != nil { + for _, s := range c.serives { // TODO(anbraten): This is a hack to get the current configs into the http service - _httpService, ok := c.httpService.(*http) + _s, ok := s.(*http) if !ok { log.Err(err).Msg("http service is not of type http") - return files, nil - } - _httpService.currentConfigs = files - - httpFiles, err := c.httpService.Fetch(ctx, forge, user, repo, pipeline) - if err != nil { - log.Err(err).Msg("failed to fetch config from http service using forge config instead") - return files, nil + } else { + _s.currentConfigs = files } - files = httpFiles + files, err = s.Fetch(ctx, forge, user, repo, pipeline) } - return files, nil + return files, err } From 80c4e3b0a3db0bcbe190dcdfd54e3d8c1fb940c2 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Thu, 8 Feb 2024 00:33:12 +0100 Subject: [PATCH 13/28] prettier --- .../75-addons/20-creating-addons.md | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/docs/docs/30-administration/75-addons/20-creating-addons.md b/docs/docs/30-administration/75-addons/20-creating-addons.md index 25b90bc430..048ab875ea 100644 --- a/docs/docs/30-administration/75-addons/20-creating-addons.md +++ b/docs/docs/30-administration/75-addons/20-creating-addons.md @@ -19,14 +19,14 @@ Directly import Woodpecker's Go package (`go.woodpecker-ci.org/woodpecker/woodpe ### Return types -| Addon type | Return type | -| -------------------- | -------------------------------------------------------------------------------- | -| `Forge` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge` | -| `Backend` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/pipeline/backend/types".Backend` | -| `ConfigService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/config".Extension` | -| `SecretService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/secret".SecretService` | -| `EnvironmentService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/environment".EnvironmentService` | -| `RegistryService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/registry".RegistryService` | +| Addon type | Return type | +| -------------------- | ------------------------------------------------------------------------------------- | +| `Forge` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge` | +| `Backend` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/pipeline/backend/types".Backend` | +| `ConfigService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/config".Extension` | +| `SecretService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/secret".SecretService` | +| `EnvironmentService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/environment".EnvironmentService` | +| `RegistryService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/registry".RegistryService` | ### Using configurations From 78275f1852e0417d4fea9d7bfeea42e841602c35 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Thu, 8 Feb 2024 09:18:36 +0100 Subject: [PATCH 14/28] remove addon types --- server/services/manager.go | 7 +------ server/services/setup.go | 16 +++------------- shared/addon/types/types.go | 8 ++------ 3 files changed, 6 insertions(+), 25 deletions(-) diff --git a/server/services/manager.go b/server/services/manager.go index 0c022cd1ec..fc90e929e4 100644 --- a/server/services/manager.go +++ b/server/services/manager.go @@ -42,17 +42,12 @@ func NewManager(c *cli.Context, store store.Store) (*Manager, error) { return nil, err } - config, err := setupConfigService(c, signaturePrivateKey) - if err != nil { - return nil, err - } - return &Manager{ signaturePrivateKey: signaturePrivateKey, signaturePublicKey: signaturePublicKey, secret: setupSecretExtension(store), registry: setupRegistryExtension(store, c.String("docker-config")), - config: config, + config: setupConfigService(c, signaturePrivateKey), environment: environment.Parse(c.StringSlice("environment")), }, nil } diff --git a/server/services/setup.go b/server/services/setup.go index 845d936005..46e8e9fd67 100644 --- a/server/services/setup.go +++ b/server/services/setup.go @@ -30,8 +30,6 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" "go.woodpecker-ci.org/woodpecker/v2/server/store" "go.woodpecker-ci.org/woodpecker/v2/server/store/types" - "go.woodpecker-ci.org/woodpecker/v2/shared/addon" - addonTypes "go.woodpecker-ci.org/woodpecker/v2/shared/addon/types" ) func setupRegistryExtension(store store.Store, dockerConfig string) registry.Service { @@ -58,24 +56,16 @@ func setupSecretExtension(store store.Store) secret.Service { return secret.NewDB(store) } -func setupConfigService(c *cli.Context, privateSignatureKey crypto.PrivateKey) (config.Service, error) { - addonExt, err := addon.Load[config.Service](c.StringSlice("addons"), addonTypes.TypeConfigService) - if err != nil { - return nil, err - } - if addonExt != nil { - return addonExt.Value, nil - } - +func setupConfigService(c *cli.Context, privateSignatureKey crypto.PrivateKey) config.Service { timeout := c.Duration("forge-timeout") configFetcher := config.NewForge(timeout) if endpoint := c.String("config-service-endpoint"); endpoint != "" { httpFetcher := config.NewHTTP(endpoint, privateSignatureKey, nil) - return config.NewCombined(configFetcher, httpFetcher), nil + return config.NewCombined(configFetcher, httpFetcher) } - return configFetcher, nil + return configFetcher } // setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for extensions) diff --git a/shared/addon/types/types.go b/shared/addon/types/types.go index 158df423b1..1a70daffeb 100644 --- a/shared/addon/types/types.go +++ b/shared/addon/types/types.go @@ -3,10 +3,6 @@ package types type Type string const ( - TypeForge Type = "forge" - TypeBackend Type = "backend" - TypeConfigService Type = "config_service" - TypeSecretService Type = "secret_service" - TypeEnvironmentService Type = "environment_service" - TypeRegistryService Type = "registry_service" + TypeForge Type = "forge" + TypeBackend Type = "backend" ) From cb25cc0451b2d63866a30429645d6af421fdc84e Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Thu, 8 Feb 2024 10:23:09 +0100 Subject: [PATCH 15/28] remove unused config --- server/config.go | 9 --------- 1 file changed, 9 deletions(-) diff --git a/server/config.go b/server/config.go index 5015c5483b..84b56c29ad 100644 --- a/server/config.go +++ b/server/config.go @@ -39,15 +39,6 @@ var Config = struct { Membership cache.MembershipService Manager *services.Manager } - Storage struct { - // Users model.UserStore - // Repos model.RepoStore - // Builds model.BuildStore - // Logs model.LogStore - Steps model.StepStore - // Registries model.RegistryStore - // Secrets model.SecretStore - } Server struct { Key string Cert string From 88d3732cd99f1b351ddecce8e7731f533158098f Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Thu, 8 Feb 2024 11:03:06 +0100 Subject: [PATCH 16/28] rename to extensions --- cmd/server/server.go | 3 +-- .../30-administration/75-addons/20-creating-addons.md | 4 ---- server/config.go | 6 +++--- server/{services => extensions}/config/combined.go | 0 .../{services => extensions}/config/combined_test.go | 2 +- server/{services => extensions}/config/forge.go | 0 server/{services => extensions}/config/forge_test.go | 2 +- server/{services => extensions}/config/http.go | 2 +- server/{services => extensions}/config/service.go | 0 server/{services => extensions}/encryption/aes.go | 0 .../{services => extensions}/encryption/aes_builder.go | 0 .../encryption/aes_encryption.go | 0 .../{services => extensions}/encryption/aes_state.go | 0 server/{services => extensions}/encryption/aes_test.go | 0 .../{services => extensions}/encryption/constants.go | 0 .../{services => extensions}/encryption/encryption.go | 0 .../encryption/encryption_builder.go | 0 .../encryption/no_encryption.go | 0 server/{services => extensions}/encryption/tink.go | 0 .../encryption/tink_builder.go | 0 .../{services => extensions}/encryption/tink_keyset.go | 0 .../encryption/tink_keyset_watcher.go | 0 .../{services => extensions}/encryption/tink_state.go | 0 .../encryption/wrapper/store/constants.go | 0 .../encryption/wrapper/store/secret_store.go | 0 .../encryption/wrapper/store/secret_store_wrapper.go | 0 server/{services => extensions}/environment/parse.go | 0 .../{services => extensions}/environment/parse_test.go | 0 server/{services => extensions}/environment/service.go | 0 server/{services => extensions}/manager.go | 10 +++++----- server/{services => extensions}/permissions/admins.go | 0 .../permissions/admins_test.go | 0 server/{services => extensions}/permissions/orgs.go | 0 .../{services => extensions}/permissions/orgs_test.go | 0 .../permissions/repo_owners.go | 0 .../permissions/repo_owners_test.go | 0 server/{services => extensions}/registry/combined.go | 0 server/{services => extensions}/registry/db.go | 0 server/{services => extensions}/registry/filesystem.go | 0 server/{services => extensions}/registry/service.go | 0 server/{services => extensions}/secret/db.go | 0 server/{services => extensions}/secret/db_test.go | 2 +- server/{services => extensions}/secret/service.go | 0 server/{services => extensions}/setup.go | 8 ++++---- server/{services => extensions}/utils/http.go | 0 server/{services => extensions}/utils/http_test.go | 2 +- 46 files changed, 18 insertions(+), 23 deletions(-) rename server/{services => extensions}/config/combined.go (100%) rename server/{services => extensions}/config/combined_test.go (99%) rename server/{services => extensions}/config/forge.go (100%) rename server/{services => extensions}/config/forge_test.go (99%) rename server/{services => extensions}/config/http.go (97%) rename server/{services => extensions}/config/service.go (100%) rename server/{services => extensions}/encryption/aes.go (100%) rename server/{services => extensions}/encryption/aes_builder.go (100%) rename server/{services => extensions}/encryption/aes_encryption.go (100%) rename server/{services => extensions}/encryption/aes_state.go (100%) rename server/{services => extensions}/encryption/aes_test.go (100%) rename server/{services => extensions}/encryption/constants.go (100%) rename server/{services => extensions}/encryption/encryption.go (100%) rename server/{services => extensions}/encryption/encryption_builder.go (100%) rename server/{services => extensions}/encryption/no_encryption.go (100%) rename server/{services => extensions}/encryption/tink.go (100%) rename server/{services => extensions}/encryption/tink_builder.go (100%) rename server/{services => extensions}/encryption/tink_keyset.go (100%) rename server/{services => extensions}/encryption/tink_keyset_watcher.go (100%) rename server/{services => extensions}/encryption/tink_state.go (100%) rename server/{services => extensions}/encryption/wrapper/store/constants.go (100%) rename server/{services => extensions}/encryption/wrapper/store/secret_store.go (100%) rename server/{services => extensions}/encryption/wrapper/store/secret_store_wrapper.go (100%) rename server/{services => extensions}/environment/parse.go (100%) rename server/{services => extensions}/environment/parse_test.go (100%) rename server/{services => extensions}/environment/service.go (100%) rename server/{services => extensions}/manager.go (88%) rename server/{services => extensions}/permissions/admins.go (100%) rename server/{services => extensions}/permissions/admins_test.go (100%) rename server/{services => extensions}/permissions/orgs.go (100%) rename server/{services => extensions}/permissions/orgs_test.go (100%) rename server/{services => extensions}/permissions/repo_owners.go (100%) rename server/{services => extensions}/permissions/repo_owners_test.go (100%) rename server/{services => extensions}/registry/combined.go (100%) rename server/{services => extensions}/registry/db.go (100%) rename server/{services => extensions}/registry/filesystem.go (100%) rename server/{services => extensions}/registry/service.go (100%) rename server/{services => extensions}/secret/db.go (100%) rename server/{services => extensions}/secret/db_test.go (97%) rename server/{services => extensions}/secret/service.go (100%) rename server/{services => extensions}/setup.go (93%) rename server/{services => extensions}/utils/http.go (100%) rename server/{services => extensions}/utils/http_test.go (96%) diff --git a/cmd/server/server.go b/cmd/server/server.go index 02813287b7..836709817a 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -38,6 +38,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc/proto" "go.woodpecker-ci.org/woodpecker/v2/server" "go.woodpecker-ci.org/woodpecker/v2/server/cron" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/forge" woodpeckerGrpcServer "go.woodpecker-ci.org/woodpecker/v2/server/grpc" "go.woodpecker-ci.org/woodpecker/v2/server/logging" @@ -45,8 +46,6 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/pubsub" "go.woodpecker-ci.org/woodpecker/v2/server/router" "go.woodpecker-ci.org/woodpecker/v2/server/router/middleware" - "go.woodpecker-ci.org/woodpecker/v2/server/services" - "go.woodpecker-ci.org/woodpecker/v2/server/services/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/store" "go.woodpecker-ci.org/woodpecker/v2/server/web" "go.woodpecker-ci.org/woodpecker/v2/shared/constant" diff --git a/docs/docs/30-administration/75-addons/20-creating-addons.md b/docs/docs/30-administration/75-addons/20-creating-addons.md index 048ab875ea..998b437b5e 100644 --- a/docs/docs/30-administration/75-addons/20-creating-addons.md +++ b/docs/docs/30-administration/75-addons/20-creating-addons.md @@ -23,10 +23,6 @@ Directly import Woodpecker's Go package (`go.woodpecker-ci.org/woodpecker/woodpe | -------------------- | ------------------------------------------------------------------------------------- | | `Forge` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge` | | `Backend` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/pipeline/backend/types".Backend` | -| `ConfigService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/config".Extension` | -| `SecretService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/secret".SecretService` | -| `EnvironmentService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/environment".EnvironmentService` | -| `RegistryService` | `"go.woodpecker-ci.org/woodpecker/v2/server/services/registry".RegistryService` | ### Using configurations diff --git a/server/config.go b/server/config.go index 84b56c29ad..208c452384 100644 --- a/server/config.go +++ b/server/config.go @@ -21,13 +21,13 @@ import ( "time" "go.woodpecker-ci.org/woodpecker/v2/server/cache" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/logging" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/pubsub" "go.woodpecker-ci.org/woodpecker/v2/server/queue" - "go.woodpecker-ci.org/woodpecker/v2/server/services" - "go.woodpecker-ci.org/woodpecker/v2/server/services/permissions" ) var Config = struct { @@ -37,7 +37,7 @@ var Config = struct { Logs logging.Log Forge forge.Forge Membership cache.MembershipService - Manager *services.Manager + Manager *extensions.Manager } Server struct { Key string diff --git a/server/services/config/combined.go b/server/extensions/config/combined.go similarity index 100% rename from server/services/config/combined.go rename to server/extensions/config/combined.go diff --git a/server/services/config/combined_test.go b/server/extensions/config/combined_test.go similarity index 99% rename from server/services/config/combined_test.go rename to server/extensions/config/combined_test.go index b4389e6573..56060f7d6e 100644 --- a/server/services/config/combined_test.go +++ b/server/extensions/config/combined_test.go @@ -31,10 +31,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/config" "go.woodpecker-ci.org/woodpecker/v2/server/forge/mocks" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/services/config" ) func TestFetchFromConfigService(t *testing.T) { diff --git a/server/services/config/forge.go b/server/extensions/config/forge.go similarity index 100% rename from server/services/config/forge.go rename to server/extensions/config/forge.go diff --git a/server/services/config/forge_test.go b/server/extensions/config/forge_test.go similarity index 99% rename from server/services/config/forge_test.go rename to server/extensions/config/forge_test.go index 9826c52616..05ec0fa147 100644 --- a/server/services/config/forge_test.go +++ b/server/extensions/config/forge_test.go @@ -24,10 +24,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/config" "go.woodpecker-ci.org/woodpecker/v2/server/forge/mocks" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/services/config" ) func TestFetch(t *testing.T) { diff --git a/server/services/config/http.go b/server/extensions/config/http.go similarity index 97% rename from server/services/config/http.go rename to server/extensions/config/http.go index 307fa62828..34027b01d8 100644 --- a/server/services/config/http.go +++ b/server/extensions/config/http.go @@ -19,10 +19,10 @@ import ( "crypto" "fmt" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/utils" "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/services/utils" ) type http struct { diff --git a/server/services/config/service.go b/server/extensions/config/service.go similarity index 100% rename from server/services/config/service.go rename to server/extensions/config/service.go diff --git a/server/services/encryption/aes.go b/server/extensions/encryption/aes.go similarity index 100% rename from server/services/encryption/aes.go rename to server/extensions/encryption/aes.go diff --git a/server/services/encryption/aes_builder.go b/server/extensions/encryption/aes_builder.go similarity index 100% rename from server/services/encryption/aes_builder.go rename to server/extensions/encryption/aes_builder.go diff --git a/server/services/encryption/aes_encryption.go b/server/extensions/encryption/aes_encryption.go similarity index 100% rename from server/services/encryption/aes_encryption.go rename to server/extensions/encryption/aes_encryption.go diff --git a/server/services/encryption/aes_state.go b/server/extensions/encryption/aes_state.go similarity index 100% rename from server/services/encryption/aes_state.go rename to server/extensions/encryption/aes_state.go diff --git a/server/services/encryption/aes_test.go b/server/extensions/encryption/aes_test.go similarity index 100% rename from server/services/encryption/aes_test.go rename to server/extensions/encryption/aes_test.go diff --git a/server/services/encryption/constants.go b/server/extensions/encryption/constants.go similarity index 100% rename from server/services/encryption/constants.go rename to server/extensions/encryption/constants.go diff --git a/server/services/encryption/encryption.go b/server/extensions/encryption/encryption.go similarity index 100% rename from server/services/encryption/encryption.go rename to server/extensions/encryption/encryption.go diff --git a/server/services/encryption/encryption_builder.go b/server/extensions/encryption/encryption_builder.go similarity index 100% rename from server/services/encryption/encryption_builder.go rename to server/extensions/encryption/encryption_builder.go diff --git a/server/services/encryption/no_encryption.go b/server/extensions/encryption/no_encryption.go similarity index 100% rename from server/services/encryption/no_encryption.go rename to server/extensions/encryption/no_encryption.go diff --git a/server/services/encryption/tink.go b/server/extensions/encryption/tink.go similarity index 100% rename from server/services/encryption/tink.go rename to server/extensions/encryption/tink.go diff --git a/server/services/encryption/tink_builder.go b/server/extensions/encryption/tink_builder.go similarity index 100% rename from server/services/encryption/tink_builder.go rename to server/extensions/encryption/tink_builder.go diff --git a/server/services/encryption/tink_keyset.go b/server/extensions/encryption/tink_keyset.go similarity index 100% rename from server/services/encryption/tink_keyset.go rename to server/extensions/encryption/tink_keyset.go diff --git a/server/services/encryption/tink_keyset_watcher.go b/server/extensions/encryption/tink_keyset_watcher.go similarity index 100% rename from server/services/encryption/tink_keyset_watcher.go rename to server/extensions/encryption/tink_keyset_watcher.go diff --git a/server/services/encryption/tink_state.go b/server/extensions/encryption/tink_state.go similarity index 100% rename from server/services/encryption/tink_state.go rename to server/extensions/encryption/tink_state.go diff --git a/server/services/encryption/wrapper/store/constants.go b/server/extensions/encryption/wrapper/store/constants.go similarity index 100% rename from server/services/encryption/wrapper/store/constants.go rename to server/extensions/encryption/wrapper/store/constants.go diff --git a/server/services/encryption/wrapper/store/secret_store.go b/server/extensions/encryption/wrapper/store/secret_store.go similarity index 100% rename from server/services/encryption/wrapper/store/secret_store.go rename to server/extensions/encryption/wrapper/store/secret_store.go diff --git a/server/services/encryption/wrapper/store/secret_store_wrapper.go b/server/extensions/encryption/wrapper/store/secret_store_wrapper.go similarity index 100% rename from server/services/encryption/wrapper/store/secret_store_wrapper.go rename to server/extensions/encryption/wrapper/store/secret_store_wrapper.go diff --git a/server/services/environment/parse.go b/server/extensions/environment/parse.go similarity index 100% rename from server/services/environment/parse.go rename to server/extensions/environment/parse.go diff --git a/server/services/environment/parse_test.go b/server/extensions/environment/parse_test.go similarity index 100% rename from server/services/environment/parse_test.go rename to server/extensions/environment/parse_test.go diff --git a/server/services/environment/service.go b/server/extensions/environment/service.go similarity index 100% rename from server/services/environment/service.go rename to server/extensions/environment/service.go diff --git a/server/services/manager.go b/server/extensions/manager.go similarity index 88% rename from server/services/manager.go rename to server/extensions/manager.go index fc90e929e4..51f6b5bd57 100644 --- a/server/services/manager.go +++ b/server/extensions/manager.go @@ -12,18 +12,18 @@ // See the License for the specific language governing permissions and // limitations under the License. -package services +package extensions import ( "crypto" "github.com/urfave/cli/v2" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/config" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/environment" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/registry" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/secret" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/services/config" - "go.woodpecker-ci.org/woodpecker/v2/server/services/environment" - "go.woodpecker-ci.org/woodpecker/v2/server/services/registry" - "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) diff --git a/server/services/permissions/admins.go b/server/extensions/permissions/admins.go similarity index 100% rename from server/services/permissions/admins.go rename to server/extensions/permissions/admins.go diff --git a/server/services/permissions/admins_test.go b/server/extensions/permissions/admins_test.go similarity index 100% rename from server/services/permissions/admins_test.go rename to server/extensions/permissions/admins_test.go diff --git a/server/services/permissions/orgs.go b/server/extensions/permissions/orgs.go similarity index 100% rename from server/services/permissions/orgs.go rename to server/extensions/permissions/orgs.go diff --git a/server/services/permissions/orgs_test.go b/server/extensions/permissions/orgs_test.go similarity index 100% rename from server/services/permissions/orgs_test.go rename to server/extensions/permissions/orgs_test.go diff --git a/server/services/permissions/repo_owners.go b/server/extensions/permissions/repo_owners.go similarity index 100% rename from server/services/permissions/repo_owners.go rename to server/extensions/permissions/repo_owners.go diff --git a/server/services/permissions/repo_owners_test.go b/server/extensions/permissions/repo_owners_test.go similarity index 100% rename from server/services/permissions/repo_owners_test.go rename to server/extensions/permissions/repo_owners_test.go diff --git a/server/services/registry/combined.go b/server/extensions/registry/combined.go similarity index 100% rename from server/services/registry/combined.go rename to server/extensions/registry/combined.go diff --git a/server/services/registry/db.go b/server/extensions/registry/db.go similarity index 100% rename from server/services/registry/db.go rename to server/extensions/registry/db.go diff --git a/server/services/registry/filesystem.go b/server/extensions/registry/filesystem.go similarity index 100% rename from server/services/registry/filesystem.go rename to server/extensions/registry/filesystem.go diff --git a/server/services/registry/service.go b/server/extensions/registry/service.go similarity index 100% rename from server/services/registry/service.go rename to server/extensions/registry/service.go diff --git a/server/services/secret/db.go b/server/extensions/secret/db.go similarity index 100% rename from server/services/secret/db.go rename to server/extensions/secret/db.go diff --git a/server/services/secret/db_test.go b/server/extensions/secret/db_test.go similarity index 97% rename from server/services/secret/db_test.go rename to server/extensions/secret/db_test.go index 6f6099f57f..09ceb0aee6 100644 --- a/server/services/secret/db_test.go +++ b/server/extensions/secret/db_test.go @@ -20,8 +20,8 @@ import ( "github.com/franela/goblin" "github.com/stretchr/testify/mock" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/secret" "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" mocks_store "go.woodpecker-ci.org/woodpecker/v2/server/store/mocks" ) diff --git a/server/services/secret/service.go b/server/extensions/secret/service.go similarity index 100% rename from server/services/secret/service.go rename to server/extensions/secret/service.go diff --git a/server/services/setup.go b/server/extensions/setup.go similarity index 93% rename from server/services/setup.go rename to server/extensions/setup.go index 46e8e9fd67..570197997d 100644 --- a/server/services/setup.go +++ b/server/extensions/setup.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package services +package extensions import ( "crypto" @@ -25,9 +25,9 @@ import ( "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/services/config" - "go.woodpecker-ci.org/woodpecker/v2/server/services/registry" - "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/config" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/registry" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/secret" "go.woodpecker-ci.org/woodpecker/v2/server/store" "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) diff --git a/server/services/utils/http.go b/server/extensions/utils/http.go similarity index 100% rename from server/services/utils/http.go rename to server/extensions/utils/http.go diff --git a/server/services/utils/http_test.go b/server/extensions/utils/http_test.go similarity index 96% rename from server/services/utils/http_test.go rename to server/extensions/utils/http_test.go index 272be5203d..32e2a83929 100644 --- a/server/services/utils/http_test.go +++ b/server/extensions/utils/http_test.go @@ -25,7 +25,7 @@ import ( "github.com/go-ap/httpsig" "github.com/stretchr/testify/assert" - "go.woodpecker-ci.org/woodpecker/v2/server/services/utils" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/utils" ) func TestSign(t *testing.T) { From 71c122dcddd039295e49b80a34d436dd3afbf7a7 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Thu, 8 Feb 2024 18:53:32 +0100 Subject: [PATCH 17/28] prettier --- docs/docs/30-administration/75-addons/20-creating-addons.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/docs/docs/30-administration/75-addons/20-creating-addons.md b/docs/docs/30-administration/75-addons/20-creating-addons.md index afee46e0b9..283c456f45 100644 --- a/docs/docs/30-administration/75-addons/20-creating-addons.md +++ b/docs/docs/30-administration/75-addons/20-creating-addons.md @@ -19,9 +19,9 @@ Directly import Woodpecker's Go package (`go.woodpecker-ci.org/woodpecker/woodpe ### Return types -| Addon type | Return type | -| -------------------- | ---------------------------------------------------------------------- | -| `Forge` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge` | +| Addon type | Return type | +| ---------- | -------------------------------------------------------------------- | +| `Forge` | `"go.woodpecker-ci.org/woodpecker/woodpecker/v2/server/forge".Forge` | ### Using configurations From b38b722e3fec4cdf42a08f4b39cade032eb3174e Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Thu, 8 Feb 2024 20:49:51 +0100 Subject: [PATCH 18/28] update readme --- docs/docs/30-administration/75-addons/00-overview.md | 4 ---- 1 file changed, 4 deletions(-) diff --git a/docs/docs/30-administration/75-addons/00-overview.md b/docs/docs/30-administration/75-addons/00-overview.md index 31c93e7824..747dc4b366 100644 --- a/docs/docs/30-administration/75-addons/00-overview.md +++ b/docs/docs/30-administration/75-addons/00-overview.md @@ -13,10 +13,6 @@ To adapt Woodpecker to your needs beyond the [configuration](../10-server-config Addons can be used for: - Forges -- Config services -- Secret services -- Environment services -- Registry services ## Restrictions From 535ee6b1c15a614ca685f586761d4a124203dee9 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:05:13 +0100 Subject: [PATCH 19/28] service => extension --- server/api/global_secret.go | 22 +++++++++---------- server/api/org_secret.go | 22 +++++++++---------- server/api/registry.go | 22 +++++++++---------- server/api/repo_secret.go | 22 +++++++++---------- server/extensions/config/combined.go | 12 +++++----- server/extensions/config/combined_test.go | 2 +- .../config/{service.go => extension.go} | 2 +- server/extensions/config/forge.go | 10 ++++++--- server/extensions/config/http.go | 2 +- server/extensions/encryption/aes.go | 12 +++++----- server/extensions/encryption/aes_builder.go | 12 +++++----- .../extensions/encryption/aes_encryption.go | 6 ++--- server/extensions/encryption/aes_state.go | 16 +++++++------- server/extensions/encryption/aes_test.go | 4 ++-- server/extensions/encryption/constants.go | 12 +++++----- server/extensions/encryption/encryption.go | 10 ++++----- .../encryption/encryption_builder.go | 14 ++++++------ server/extensions/encryption/no_encryption.go | 10 ++++----- server/extensions/encryption/tink.go | 12 +++++----- server/extensions/encryption/tink_builder.go | 12 +++++----- server/extensions/encryption/tink_keyset.go | 4 ++-- .../encryption/tink_keyset_watcher.go | 4 ++-- server/extensions/encryption/tink_state.go | 22 +++++++++---------- .../encryption/types}/encryption.go | 20 ++++++++--------- .../wrapper/store/secret_store_wrapper.go | 11 +++++----- .../environment/{service.go => extension.go} | 4 ++-- server/extensions/environment/parse.go | 4 ++-- server/extensions/environment/parse_test.go | 16 +++++++------- server/extensions/manager.go | 22 +++++++++---------- server/extensions/registry/combined.go | 6 ++--- server/extensions/registry/db.go | 4 ++-- .../registry/{service.go => extension.go} | 8 +++---- server/extensions/registry/filesystem.go | 2 +- server/extensions/secret/db.go | 4 ++-- .../secret/{service.go => extension.go} | 4 ++-- server/extensions/setup.go | 12 +++++----- server/pipeline/create.go | 2 +- server/pipeline/items.go | 6 ++--- server/pipeline/restart.go | 2 +- 39 files changed, 199 insertions(+), 194 deletions(-) rename server/extensions/config/{service.go => extension.go} (97%) rename server/{model => extensions/encryption/types}/encryption.go (68%) rename server/extensions/environment/{service.go => extension.go} (88%) rename server/extensions/registry/{service.go => extension.go} (85%) rename server/extensions/secret/{service.go => extension.go} (95%) diff --git a/server/api/global_secret.go b/server/api/global_secret.go index b01de72e5b..a9421ccd55 100644 --- a/server/api/global_secret.go +++ b/server/api/global_secret.go @@ -35,8 +35,8 @@ import ( // @Param page query int false "for response pagination, page offset number" default(1) // @Param perPage query int false "for response pagination, max items per page" default(50) func GetGlobalSecretList(c *gin.Context) { - secretService := server.Config.Services.Manager.SecretService() - list, err := secretService.GlobalSecretList(session.Pagination(c)) + secretExtension := server.Config.Services.Manager.SecretExtension() + list, err := secretExtension.GlobalSecretList(session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting global secret list. %s", err) return @@ -60,8 +60,8 @@ func GetGlobalSecretList(c *gin.Context) { // @Param secret path string true "the secret's name" func GetGlobalSecret(c *gin.Context) { name := c.Param("secret") - secretService := server.Config.Services.Manager.SecretService() - secret, err := secretService.GlobalSecretFind(name) + secretExtension := server.Config.Services.Manager.SecretExtension() + secret, err := secretExtension.GlobalSecretFind(name) if err != nil { handleDBError(c, err) return @@ -95,8 +95,8 @@ func PostGlobalSecret(c *gin.Context) { return } - secretService := server.Config.Services.Manager.SecretService() - if err := secretService.GlobalSecretCreate(secret); err != nil { + secretExtension := server.Config.Services.Manager.SecretExtension() + if err := secretExtension.GlobalSecretCreate(secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting global secret %q. %s", in.Name, err) return } @@ -123,8 +123,8 @@ func PatchGlobalSecret(c *gin.Context) { return } - secretService := server.Config.Services.Manager.SecretService() - secret, err := secretService.GlobalSecretFind(name) + secretExtension := server.Config.Services.Manager.SecretExtension() + secret, err := secretExtension.GlobalSecretFind(name) if err != nil { handleDBError(c, err) return @@ -144,7 +144,7 @@ func PatchGlobalSecret(c *gin.Context) { return } - if err := secretService.GlobalSecretUpdate(secret); err != nil { + if err := secretExtension.GlobalSecretUpdate(secret); err != nil { c.String(http.StatusInternalServerError, "Error updating global secret %q. %s", in.Name, err) return } @@ -162,8 +162,8 @@ func PatchGlobalSecret(c *gin.Context) { // @Param secret path string true "the secret's name" func DeleteGlobalSecret(c *gin.Context) { name := c.Param("secret") - secretService := server.Config.Services.Manager.SecretService() - if err := secretService.GlobalSecretDelete(name); err != nil { + secretExtension := server.Config.Services.Manager.SecretExtension() + if err := secretExtension.GlobalSecretDelete(name); err != nil { handleDBError(c, err) return } diff --git a/server/api/org_secret.go b/server/api/org_secret.go index 579820bc98..bd70c4ddd8 100644 --- a/server/api/org_secret.go +++ b/server/api/org_secret.go @@ -44,8 +44,8 @@ func GetOrgSecret(c *gin.Context) { return } - secretService := server.Config.Services.Manager.SecretService() - secret, err := secretService.OrgSecretFind(orgID, name) + secretExtension := server.Config.Services.Manager.SecretExtension() + secret, err := secretExtension.OrgSecretFind(orgID, name) if err != nil { handleDBError(c, err) return @@ -71,8 +71,8 @@ func GetOrgSecretList(c *gin.Context) { return } - secretService := server.Config.Services.Manager.SecretService() - list, err := secretService.OrgSecretList(orgID, session.Pagination(c)) + secretExtension := server.Config.Services.Manager.SecretExtension() + list, err := secretExtension.OrgSecretList(orgID, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting secret list for %q. %s", orgID, err) return @@ -119,8 +119,8 @@ func PostOrgSecret(c *gin.Context) { return } - secretService := server.Config.Services.Manager.SecretService() - if err := secretService.OrgSecretCreate(orgID, secret); err != nil { + secretExtension := server.Config.Services.Manager.SecretExtension() + if err := secretExtension.OrgSecretCreate(orgID, secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting org %q secret %q. %s", orgID, in.Name, err) return } @@ -153,8 +153,8 @@ func PatchOrgSecret(c *gin.Context) { return } - secretService := server.Config.Services.Manager.SecretService() - secret, err := secretService.OrgSecretFind(orgID, name) + secretExtension := server.Config.Services.Manager.SecretExtension() + secret, err := secretExtension.OrgSecretFind(orgID, name) if err != nil { handleDBError(c, err) return @@ -174,7 +174,7 @@ func PatchOrgSecret(c *gin.Context) { return } - if err := secretService.OrgSecretUpdate(orgID, secret); err != nil { + if err := secretExtension.OrgSecretUpdate(orgID, secret); err != nil { c.String(http.StatusInternalServerError, "Error updating org %q secret %q. %s", orgID, in.Name, err) return } @@ -199,8 +199,8 @@ func DeleteOrgSecret(c *gin.Context) { return } - secretService := server.Config.Services.Manager.SecretService() - if err := secretService.OrgSecretDelete(orgID, name); err != nil { + secretExtension := server.Config.Services.Manager.SecretExtension() + if err := secretExtension.OrgSecretDelete(orgID, name); err != nil { handleDBError(c, err) return } diff --git a/server/api/registry.go b/server/api/registry.go index 60e6648ff8..f08dbd41ec 100644 --- a/server/api/registry.go +++ b/server/api/registry.go @@ -38,8 +38,8 @@ func GetRegistry(c *gin.Context) { repo := session.Repo(c) name := c.Param("registry") - registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) - registry, err := registryService.RegistryFind(repo, name) + registryExtension := server.Config.Services.Manager.RegistryExtensionFromRepo(repo) + registry, err := registryExtension.RegistryFind(repo, name) if err != nil { handleDBError(c, err) return @@ -76,8 +76,8 @@ func PostRegistry(c *gin.Context) { return } - registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) - if err := registryService.RegistryCreate(repo, registry); err != nil { + registryExtension := server.Config.Services.Manager.RegistryExtensionFromRepo(repo) + if err := registryExtension.RegistryCreate(repo, registry); err != nil { c.String(http.StatusInternalServerError, "Error inserting registry %q. %s", in.Address, err) return } @@ -108,8 +108,8 @@ func PatchRegistry(c *gin.Context) { return } - registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) - registry, err := registryService.RegistryFind(repo, name) + registryExtension := server.Config.Services.Manager.RegistryExtensionFromRepo(repo) + registry, err := registryExtension.RegistryFind(repo, name) if err != nil { handleDBError(c, err) return @@ -125,7 +125,7 @@ func PatchRegistry(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error updating registry. %s", err) return } - if err := registryService.RegistryUpdate(repo, registry); err != nil { + if err := registryExtension.RegistryUpdate(repo, registry); err != nil { c.String(http.StatusInternalServerError, "Error updating registry %q. %s", in.Address, err) return } @@ -145,8 +145,8 @@ func PatchRegistry(c *gin.Context) { // @Param perPage query int false "for response pagination, max items per page" default(50) func GetRegistryList(c *gin.Context) { repo := session.Repo(c) - registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) - list, err := registryService.RegistryList(repo, session.Pagination(c)) + registryExtension := server.Config.Services.Manager.RegistryExtensionFromRepo(repo) + list, err := registryExtension.RegistryList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting registry list. %s", err) return @@ -173,8 +173,8 @@ func DeleteRegistry(c *gin.Context) { repo := session.Repo(c) name := c.Param("registry") - registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) - err := registryService.RegistryDelete(repo, name) + registryExtension := server.Config.Services.Manager.RegistryExtensionFromRepo(repo) + err := registryExtension.RegistryDelete(repo, name) if err != nil { handleDBError(c, err) return diff --git a/server/api/repo_secret.go b/server/api/repo_secret.go index 2a1b963483..4ef271f230 100644 --- a/server/api/repo_secret.go +++ b/server/api/repo_secret.go @@ -39,8 +39,8 @@ func GetSecret(c *gin.Context) { repo := session.Repo(c) name := c.Param("secret") - secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) - secret, err := secretService.SecretFind(repo, name) + secretExtension := server.Config.Services.Manager.SecretExtensionFromRepo(repo) + secret, err := secretExtension.SecretFind(repo, name) if err != nil { handleDBError(c, err) return @@ -78,8 +78,8 @@ func PostSecret(c *gin.Context) { return } - secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) - if err := secretService.SecretCreate(repo, secret); err != nil { + secretExtension := server.Config.Services.Manager.SecretExtensionFromRepo(repo) + if err := secretExtension.SecretCreate(repo, secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting secret %q. %s", in.Name, err) return } @@ -110,8 +110,8 @@ func PatchSecret(c *gin.Context) { return } - secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) - secret, err := secretService.SecretFind(repo, name) + secretExtension := server.Config.Services.Manager.SecretExtensionFromRepo(repo) + secret, err := secretExtension.SecretFind(repo, name) if err != nil { handleDBError(c, err) return @@ -130,7 +130,7 @@ func PatchSecret(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error updating secret. %s", err) return } - if err := secretService.SecretUpdate(repo, secret); err != nil { + if err := secretExtension.SecretUpdate(repo, secret); err != nil { c.String(http.StatusInternalServerError, "Error updating secret %q. %s", in.Name, err) return } @@ -150,8 +150,8 @@ func PatchSecret(c *gin.Context) { // @Param perPage query int false "for response pagination, max items per page" default(50) func GetSecretList(c *gin.Context) { repo := session.Repo(c) - secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) - list, err := secretService.SecretList(repo, session.Pagination(c)) + secretExtension := server.Config.Services.Manager.SecretExtensionFromRepo(repo) + list, err := secretExtension.SecretList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting secret list. %s", err) return @@ -178,8 +178,8 @@ func DeleteSecret(c *gin.Context) { repo := session.Repo(c) name := c.Param("secret") - secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) - if err := secretService.SecretDelete(repo, name); err != nil { + secretExtension := server.Config.Services.Manager.SecretExtensionFromRepo(repo) + if err := secretExtension.SecretDelete(repo, name); err != nil { handleDBError(c, err) return } diff --git a/server/extensions/config/combined.go b/server/extensions/config/combined.go index 8312b98735..2b8f5739fa 100644 --- a/server/extensions/config/combined.go +++ b/server/extensions/config/combined.go @@ -25,19 +25,19 @@ import ( ) type combined struct { - serives []Service + extensions []Extension } -func NewCombined(services ...Service) Service { - return &combined{serives: services} +func NewCombined(extensions ...Extension) Extension { + return &combined{extensions: extensions} } func (c *combined) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (files []*types.FileMeta, err error) { - for _, s := range c.serives { - // TODO(anbraten): This is a hack to get the current configs into the http service + for _, s := range c.extensions { + // TODO(anbraten): This is a hack to get the current configs into the http extension _s, ok := s.(*http) if !ok { - log.Err(err).Msg("http service is not of type http") + log.Err(err).Msg("http extension is not of type http") } else { _s.currentConfigs = files } diff --git a/server/extensions/config/combined_test.go b/server/extensions/config/combined_test.go index 56060f7d6e..e93d2b9c5a 100644 --- a/server/extensions/config/combined_test.go +++ b/server/extensions/config/combined_test.go @@ -37,7 +37,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/model" ) -func TestFetchFromConfigService(t *testing.T) { +func TestFetchFromConfigExtension(t *testing.T) { t.Parallel() type file struct { diff --git a/server/extensions/config/service.go b/server/extensions/config/extension.go similarity index 97% rename from server/extensions/config/service.go rename to server/extensions/config/extension.go index 89bbefc108..fb038f73e1 100644 --- a/server/extensions/config/service.go +++ b/server/extensions/config/extension.go @@ -22,6 +22,6 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/model" ) -type Service interface { +type Extension interface { Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (configData []*types.FileMeta, err error) } diff --git a/server/extensions/config/forge.go b/server/extensions/config/forge.go index f1965e30ab..a3294e15f6 100644 --- a/server/extensions/config/forge.go +++ b/server/extensions/config/forge.go @@ -29,11 +29,15 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/shared/constant" ) +const ( + forgeFetchingRetryCount = 3 +) + type forgeFetcher struct { timeout time.Duration } -func NewForge(timeout time.Duration) Service { +func NewForge(timeout time.Duration) Extension { return &forgeFetcher{ timeout: timeout, } @@ -48,8 +52,8 @@ func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model timeout: f.timeout, } - // try to fetch 3 times - for i := 0; i < 3; i++ { + // try to fetch multiple times + for i := 0; i < forgeFetchingRetryCount; i++ { files, err = ffc.fetch(ctx, strings.TrimSpace(repo.Config)) if err != nil { log.Trace().Err(err).Msgf("%d. try failed", i+1) diff --git a/server/extensions/config/http.go b/server/extensions/config/http.go index 34027b01d8..65c554a29f 100644 --- a/server/extensions/config/http.go +++ b/server/extensions/config/http.go @@ -50,7 +50,7 @@ type responseStructure struct { Configs []*configData `json:"configs"` } -func NewHTTP(endpoint string, privateKey crypto.PrivateKey, currentConfigs []*types.FileMeta) Service { +func NewHTTP(endpoint string, privateKey crypto.PrivateKey, currentConfigs []*types.FileMeta) Extension { return &http{endpoint, privateKey, currentConfigs} } diff --git a/server/extensions/encryption/aes.go b/server/extensions/encryption/aes.go index a0766f4bee..192b899579 100644 --- a/server/extensions/encryption/aes.go +++ b/server/extensions/encryption/aes.go @@ -21,18 +21,18 @@ import ( "github.com/google/tink/go/subtle/random" - "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) -type aesEncryptionService struct { +type aesEncryptionExtension struct { cipher cipher.AEAD keyID string store store.Store - clients []model.EncryptionClient + clients []types.EncryptionClient } -func (svc *aesEncryptionService) Encrypt(plaintext, associatedData string) (string, error) { +func (svc *aesEncryptionExtension) Encrypt(plaintext, associatedData string) (string, error) { msg := []byte(plaintext) aad := []byte(associatedData) @@ -46,7 +46,7 @@ func (svc *aesEncryptionService) Encrypt(plaintext, associatedData string) (stri return base64.StdEncoding.EncodeToString(result), nil } -func (svc *aesEncryptionService) Decrypt(ciphertext, associatedData string) (string, error) { +func (svc *aesEncryptionExtension) Decrypt(ciphertext, associatedData string) (string, error) { bytes, err := base64.StdEncoding.DecodeString(ciphertext) if err != nil { return "", fmt.Errorf(errTemplateBase64DecryptionFailed, err) @@ -62,6 +62,6 @@ func (svc *aesEncryptionService) Decrypt(ciphertext, associatedData string) (str return string(plaintext), nil } -func (svc *aesEncryptionService) Disable() error { +func (svc *aesEncryptionExtension) Disable() error { return svc.disable() } diff --git a/server/extensions/encryption/aes_builder.go b/server/extensions/encryption/aes_builder.go index bc0bbe9455..3bb9f58991 100644 --- a/server/extensions/encryption/aes_builder.go +++ b/server/extensions/encryption/aes_builder.go @@ -20,28 +20,28 @@ import ( "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) type aesConfiguration struct { password string store store.Store - clients []model.EncryptionClient + clients []types.EncryptionClient } -func newAES(ctx *cli.Context, s store.Store) model.EncryptionServiceBuilder { +func newAES(ctx *cli.Context, s store.Store) types.EncryptionExtensionBuilder { key := ctx.String(rawKeyConfigFlag) return &aesConfiguration{key, s, nil} } -func (c aesConfiguration) WithClients(clients []model.EncryptionClient) model.EncryptionServiceBuilder { +func (c aesConfiguration) WithClients(clients []types.EncryptionClient) types.EncryptionExtensionBuilder { c.clients = clients return c } -func (c aesConfiguration) Build() (model.EncryptionService, error) { - svc := &aesEncryptionService{ +func (c aesConfiguration) Build() (types.EncryptionExtension, error) { + svc := &aesEncryptionExtension{ cipher: nil, store: c.store, clients: c.clients, diff --git a/server/extensions/encryption/aes_encryption.go b/server/extensions/encryption/aes_encryption.go index b7b5016ffb..1c2447ecc7 100644 --- a/server/extensions/encryption/aes_encryption.go +++ b/server/extensions/encryption/aes_encryption.go @@ -26,7 +26,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) -func (svc *aesEncryptionService) loadCipher(password string) error { +func (svc *aesEncryptionExtension) loadCipher(password string) error { key, err := svc.hash([]byte(password)) if err != nil { return fmt.Errorf(errTemplateAesFailedGeneratingKey, err) @@ -50,7 +50,7 @@ func (svc *aesEncryptionService) loadCipher(password string) error { return nil } -func (svc *aesEncryptionService) validateKey() error { +func (svc *aesEncryptionExtension) validateKey() error { ciphertextSample, err := svc.store.ServerConfigGet(ciphertextSampleConfigKey) if errors.Is(err, types.RecordNotExist) { return errEncryptionNotEnabled @@ -67,7 +67,7 @@ func (svc *aesEncryptionService) validateKey() error { return nil } -func (svc *aesEncryptionService) hash(data []byte) ([]byte, error) { +func (svc *aesEncryptionExtension) hash(data []byte) ([]byte, error) { result := make([]byte, 32) sha := sha3.NewShake256() diff --git a/server/extensions/encryption/aes_state.go b/server/extensions/encryption/aes_state.go index 470e80f5e7..296d8832ac 100644 --- a/server/extensions/encryption/aes_state.go +++ b/server/extensions/encryption/aes_state.go @@ -20,9 +20,9 @@ import ( "github.com/rs/zerolog/log" ) -func (svc *aesEncryptionService) initClients() error { +func (svc *aesEncryptionExtension) initClients() error { for _, client := range svc.clients { - err := client.SetEncryptionService(svc) + err := client.SetEncryptionExtension(svc) if err != nil { return fmt.Errorf(errTemplateFailedInitializingClients, err) } @@ -31,7 +31,7 @@ func (svc *aesEncryptionService) initClients() error { return nil } -func (svc *aesEncryptionService) enable() error { +func (svc *aesEncryptionExtension) enable() error { err := svc.callbackOnEnable() if err != nil { return fmt.Errorf(errTemplateFailedEnablingEncryption, err) @@ -44,7 +44,7 @@ func (svc *aesEncryptionService) enable() error { return nil } -func (svc *aesEncryptionService) disable() error { +func (svc *aesEncryptionExtension) disable() error { err := svc.callbackOnDisable() if err != nil { return fmt.Errorf(errTemplateFailedDisablingEncryption, err) @@ -57,7 +57,7 @@ func (svc *aesEncryptionService) disable() error { return nil } -func (svc *aesEncryptionService) updateCiphertextSample() error { +func (svc *aesEncryptionExtension) updateCiphertextSample() error { ciphertext, err := svc.Encrypt(svc.keyID, keyIDAssociatedData) if err != nil { return fmt.Errorf(errTemplateFailedUpdatingServerConfig, err) @@ -70,7 +70,7 @@ func (svc *aesEncryptionService) updateCiphertextSample() error { return nil } -func (svc *aesEncryptionService) deleteCiphertextSample() error { +func (svc *aesEncryptionExtension) deleteCiphertextSample() error { err := svc.store.ServerConfigDelete(ciphertextSampleConfigKey) if err != nil { err = fmt.Errorf(errTemplateFailedUpdatingServerConfig, err) @@ -78,7 +78,7 @@ func (svc *aesEncryptionService) deleteCiphertextSample() error { return err } -func (svc *aesEncryptionService) callbackOnEnable() error { +func (svc *aesEncryptionExtension) callbackOnEnable() error { for _, client := range svc.clients { err := client.EnableEncryption() if err != nil { @@ -89,7 +89,7 @@ func (svc *aesEncryptionService) callbackOnEnable() error { return nil } -func (svc *aesEncryptionService) callbackOnDisable() error { +func (svc *aesEncryptionExtension) callbackOnDisable() error { for _, client := range svc.clients { err := client.MigrateEncryption(&noEncryption{}) if err != nil { diff --git a/server/extensions/encryption/aes_test.go b/server/extensions/encryption/aes_test.go index 1a2bc5abb1..18524c3fcd 100644 --- a/server/extensions/encryption/aes_test.go +++ b/server/extensions/encryption/aes_test.go @@ -22,7 +22,7 @@ import ( ) func TestShortMessageLongKey(t *testing.T) { - aes := &aesEncryptionService{} + aes := &aesEncryptionExtension{} err := aes.loadCipher(string(random.GetRandomBytes(32))) assert.NoError(t, err) @@ -36,7 +36,7 @@ func TestShortMessageLongKey(t *testing.T) { } func TestLongMessageShortKey(t *testing.T) { - aes := &aesEncryptionService{} + aes := &aesEncryptionExtension{} err := aes.loadCipher(string(random.GetRandomBytes(12))) assert.NoError(t, err) diff --git a/server/extensions/encryption/constants.go b/server/extensions/encryption/constants.go index 5a0f28500c..6d3b5a2321 100644 --- a/server/extensions/encryption/constants.go +++ b/server/extensions/encryption/constants.go @@ -41,7 +41,7 @@ var ( const ( // error wrapping templates errTemplateFailedInitializingUnencrypted = "failed initializing server in unencrypted mode: %w" - errTemplateFailedInitializing = "failed initializing encryption service: %w" + errTemplateFailedInitializing = "failed initializing encryption extension: %w" errTemplateFailedEnablingEncryption = "failed enabling encryption: %w" errTemplateFailedRotatingEncryption = "failed rotating encryption: %w" errTemplateFailedDisablingEncryption = "failed disabling encryption: %w" @@ -55,7 +55,7 @@ const ( // error messages errMessageTemplateUnsupportedKeyType = "unsupported encryption key type: %s" - errMessageCantUseBothServices = "cannot use raw encryption key and tink keyset at the same time" + errMessageCantUseBothExtensions = "cannot use raw encryption key and tink keyset at the same time" errMessageNoKeysProvided = "encryption enabled but no keys provided" errMessageFailedRotatingEncryption = "failed rotating encryption" @@ -64,9 +64,9 @@ const ( logMessageEncryptionDisabled = "encryption disabled" logMessageEncryptionKeyRegistered = "registered new encryption key" logMessageClientsInitialized = "initialized encryption on registered clients" - logMessageClientsEnabled = "enabled encryption on registered services" - logMessageClientsRotated = "updated encryption key on registered services" - logMessageClientsDecrypted = "disabled encryption on registered services" + logMessageClientsEnabled = "enabled encryption on registered extension" + logMessageClientsRotated = "updated encryption key on registered extension" + logMessageClientsDecrypted = "disabled encryption on registered extension" ) // tink @@ -84,7 +84,7 @@ const ( errMessageTinkKeysetFileWatchFailed = "failed watching encryption keyset file changes" // log message templates - logTemplateTinkKeysetFileChanged = "changes detected in encryption keyset file: '%s'. Encryption service will be reloaded" + logTemplateTinkKeysetFileChanged = "changes detected in encryption keyset file: '%s'. Encryption extension will be reloaded" logTemplateTinkLoadingKeyset = "loading encryption keyset from file: %s" logTemplateTinkFailedClosingKeysetFile = "could not close keyset file: %s" ) diff --git a/server/extensions/encryption/encryption.go b/server/extensions/encryption/encryption.go index 34e9189cd1..ea45bd1080 100644 --- a/server/extensions/encryption/encryption.go +++ b/server/extensions/encryption/encryption.go @@ -19,21 +19,21 @@ import ( "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) type builder struct { store store.Store ctx *cli.Context - clients []model.EncryptionClient + clients []types.EncryptionClient } -func Encryption(ctx *cli.Context, s store.Store) model.EncryptionBuilder { +func Encryption(ctx *cli.Context, s store.Store) types.EncryptionBuilder { return &builder{store: s, ctx: ctx} } -func (b builder) WithClient(client model.EncryptionClient) model.EncryptionBuilder { +func (b builder) WithClient(client types.EncryptionClient) types.EncryptionBuilder { b.clients = append(b.clients, client) return b } @@ -57,7 +57,7 @@ func (b builder) Build() error { return fmt.Errorf(errTemplateFailedInitializingUnencrypted, err) } } - svc, err := b.getService(keyType) + svc, err := b.getExtension(keyType) if err != nil { return fmt.Errorf(errTemplateFailedInitializing, err) } diff --git a/server/extensions/encryption/encryption_builder.go b/server/extensions/encryption/encryption_builder.go index f14a8824a6..9adcc85c4e 100644 --- a/server/extensions/encryption/encryption_builder.go +++ b/server/extensions/encryption/encryption_builder.go @@ -18,16 +18,16 @@ import ( "errors" "fmt" - "go.woodpecker-ci.org/woodpecker/v2/server/model" - "go.woodpecker-ci.org/woodpecker/v2/server/store/types" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" + storeTypes "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) -func (b builder) getService(keyType string) (model.EncryptionService, error) { +func (b builder) getExtension(keyType string) (types.EncryptionExtension, error) { if keyType == keyTypeNone { return nil, errors.New(errMessageNoKeysProvided) } - builder, err := b.serviceBuilder(keyType) + builder, err := b.extensionBuilder(keyType) if err != nil { return nil, err } @@ -41,7 +41,7 @@ func (b builder) getService(keyType string) (model.EncryptionService, error) { func (b builder) isEnabled() (bool, error) { _, err := b.store.ServerConfigGet(ciphertextSampleConfigKey) - if err != nil && !errors.Is(err, types.RecordNotExist) { + if err != nil && !errors.Is(err, storeTypes.RecordNotExist) { return false, fmt.Errorf(errTemplateFailedLoadingServerConfig, err) } return err == nil, nil @@ -52,7 +52,7 @@ func (b builder) detectKeyType() (string, error) { tinkKeysetPresent := b.ctx.IsSet(tinkKeysetFilepathConfigFlag) switch { case rawKeyPresent && tinkKeysetPresent: - return "", errors.New(errMessageCantUseBothServices) + return "", errors.New(errMessageCantUseBothExtensions) case rawKeyPresent: return keyTypeRaw, nil case tinkKeysetPresent: @@ -61,7 +61,7 @@ func (b builder) detectKeyType() (string, error) { return keyTypeNone, nil } -func (b builder) serviceBuilder(keyType string) (model.EncryptionServiceBuilder, error) { +func (b builder) extensionBuilder(keyType string) (types.EncryptionExtensionBuilder, error) { switch { case keyType == keyTypeTink: return newTink(b.ctx, b.store), nil diff --git a/server/extensions/encryption/no_encryption.go b/server/extensions/encryption/no_encryption.go index 0d13d5ad45..b381a8aa5b 100644 --- a/server/extensions/encryption/no_encryption.go +++ b/server/extensions/encryption/no_encryption.go @@ -14,21 +14,21 @@ package encryption -import "go.woodpecker-ci.org/woodpecker/v2/server/model" +import "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" type noEncryptionBuilder struct { - clients []model.EncryptionClient + clients []types.EncryptionClient } -func (b noEncryptionBuilder) WithClients(clients []model.EncryptionClient) model.EncryptionServiceBuilder { +func (b noEncryptionBuilder) WithClients(clients []types.EncryptionClient) types.EncryptionExtensionBuilder { b.clients = clients return b } -func (b noEncryptionBuilder) Build() (model.EncryptionService, error) { +func (b noEncryptionBuilder) Build() (types.EncryptionExtension, error) { svc := &noEncryption{} for _, client := range b.clients { - err := client.SetEncryptionService(svc) + err := client.SetEncryptionExtension(svc) if err != nil { return nil, err } diff --git a/server/extensions/encryption/tink.go b/server/extensions/encryption/tink.go index 22111b802d..a7781a43c8 100644 --- a/server/extensions/encryption/tink.go +++ b/server/extensions/encryption/tink.go @@ -21,20 +21,20 @@ import ( "github.com/fsnotify/fsnotify" "github.com/google/tink/go/tink" - "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) -type tinkEncryptionService struct { +type tinkEncryptionExtension struct { keysetFilePath string primaryKeyID string encryption tink.AEAD store store.Store keysetFileWatcher *fsnotify.Watcher - clients []model.EncryptionClient + clients []types.EncryptionClient } -func (svc *tinkEncryptionService) Encrypt(plaintext, associatedData string) (string, error) { +func (svc *tinkEncryptionExtension) Encrypt(plaintext, associatedData string) (string, error) { msg := []byte(plaintext) aad := []byte(associatedData) ciphertext, err := svc.encryption.Encrypt(msg, aad) @@ -44,7 +44,7 @@ func (svc *tinkEncryptionService) Encrypt(plaintext, associatedData string) (str return base64.StdEncoding.EncodeToString(ciphertext), nil } -func (svc *tinkEncryptionService) Decrypt(ciphertext, associatedData string) (string, error) { +func (svc *tinkEncryptionExtension) Decrypt(ciphertext, associatedData string) (string, error) { ct, err := base64.StdEncoding.DecodeString(ciphertext) if err != nil { return "", fmt.Errorf(errTemplateBase64DecryptionFailed, err) @@ -57,6 +57,6 @@ func (svc *tinkEncryptionService) Decrypt(ciphertext, associatedData string) (st return string(plaintext), nil } -func (svc *tinkEncryptionService) Disable() error { +func (svc *tinkEncryptionExtension) Disable() error { return svc.disable() } diff --git a/server/extensions/encryption/tink_builder.go b/server/extensions/encryption/tink_builder.go index a7fba536e1..432be21fa9 100644 --- a/server/extensions/encryption/tink_builder.go +++ b/server/extensions/encryption/tink_builder.go @@ -20,28 +20,28 @@ import ( "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) type tinkConfiguration struct { keysetFilePath string store store.Store - clients []model.EncryptionClient + clients []types.EncryptionClient } -func newTink(ctx *cli.Context, s store.Store) model.EncryptionServiceBuilder { +func newTink(ctx *cli.Context, s store.Store) types.EncryptionExtensionBuilder { filepath := ctx.String(tinkKeysetFilepathConfigFlag) return &tinkConfiguration{filepath, s, nil} } -func (c tinkConfiguration) WithClients(clients []model.EncryptionClient) model.EncryptionServiceBuilder { +func (c tinkConfiguration) WithClients(clients []types.EncryptionClient) types.EncryptionExtensionBuilder { c.clients = clients return c } -func (c tinkConfiguration) Build() (model.EncryptionService, error) { - svc := &tinkEncryptionService{ +func (c tinkConfiguration) Build() (types.EncryptionExtension, error) { + svc := &tinkEncryptionExtension{ keysetFilePath: c.keysetFilePath, primaryKeyID: "", encryption: nil, diff --git a/server/extensions/encryption/tink_keyset.go b/server/extensions/encryption/tink_keyset.go index 482b178140..90762b1f54 100644 --- a/server/extensions/encryption/tink_keyset.go +++ b/server/extensions/encryption/tink_keyset.go @@ -28,7 +28,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) -func (svc *tinkEncryptionService) loadKeyset() error { +func (svc *tinkEncryptionExtension) loadKeyset() error { log.Warn().Msgf(logTemplateTinkLoadingKeyset, svc.keysetFilePath) file, err := os.Open(svc.keysetFilePath) if err != nil { @@ -56,7 +56,7 @@ func (svc *tinkEncryptionService) loadKeyset() error { return nil } -func (svc *tinkEncryptionService) validateKeyset() error { +func (svc *tinkEncryptionExtension) validateKeyset() error { ciphertextSample, err := svc.store.ServerConfigGet(ciphertextSampleConfigKey) if errors.Is(err, types.RecordNotExist) { return errEncryptionNotEnabled diff --git a/server/extensions/encryption/tink_keyset_watcher.go b/server/extensions/encryption/tink_keyset_watcher.go index 149278d559..0ef9a94d7f 100644 --- a/server/extensions/encryption/tink_keyset_watcher.go +++ b/server/extensions/encryption/tink_keyset_watcher.go @@ -22,7 +22,7 @@ import ( ) // Watch keyset file events to detect key rotations and hot reload keys -func (svc *tinkEncryptionService) initFileWatcher() error { +func (svc *tinkEncryptionExtension) initFileWatcher() error { watcher, err := fsnotify.NewWatcher() if err != nil { return fmt.Errorf(errTemplateTinkFailedSubscribeKeysetFileChanges, err) @@ -37,7 +37,7 @@ func (svc *tinkEncryptionService) initFileWatcher() error { return nil } -func (svc *tinkEncryptionService) handleFileEvents() { +func (svc *tinkEncryptionExtension) handleFileEvents() { for { select { case event, ok := <-svc.keysetFileWatcher.Events: diff --git a/server/extensions/encryption/tink_state.go b/server/extensions/encryption/tink_state.go index f15bf180a0..5b928a452a 100644 --- a/server/extensions/encryption/tink_state.go +++ b/server/extensions/encryption/tink_state.go @@ -21,7 +21,7 @@ import ( "github.com/rs/zerolog/log" ) -func (svc *tinkEncryptionService) enable() error { +func (svc *tinkEncryptionExtension) enable() error { if err := svc.callbackOnEnable(); err != nil { return fmt.Errorf(errTemplateFailedEnablingEncryption, err) } @@ -34,7 +34,7 @@ func (svc *tinkEncryptionService) enable() error { return nil } -func (svc *tinkEncryptionService) disable() error { +func (svc *tinkEncryptionExtension) disable() error { if err := svc.callbackOnDisable(); err != nil { return fmt.Errorf(errTemplateFailedDisablingEncryption, err) } @@ -47,8 +47,8 @@ func (svc *tinkEncryptionService) disable() error { return nil } -func (svc *tinkEncryptionService) rotate() error { - newSvc := &tinkEncryptionService{ +func (svc *tinkEncryptionExtension) rotate() error { + newSvc := &tinkEncryptionExtension{ keysetFilePath: svc.keysetFilePath, primaryKeyID: "", encryption: nil, @@ -79,7 +79,7 @@ func (svc *tinkEncryptionService) rotate() error { return nil } -func (svc *tinkEncryptionService) updateCiphertextSample() error { +func (svc *tinkEncryptionExtension) updateCiphertextSample() error { ciphertext, err := svc.Encrypt(svc.primaryKeyID, keyIDAssociatedData) if err != nil { return fmt.Errorf(errTemplateFailedUpdatingServerConfig, err) @@ -93,16 +93,16 @@ func (svc *tinkEncryptionService) updateCiphertextSample() error { return nil } -func (svc *tinkEncryptionService) deleteCiphertextSample() error { +func (svc *tinkEncryptionExtension) deleteCiphertextSample() error { if err := svc.store.ServerConfigDelete(ciphertextSampleConfigKey); err != nil { return fmt.Errorf(errTemplateFailedUpdatingServerConfig, err) } return nil } -func (svc *tinkEncryptionService) initClients() error { +func (svc *tinkEncryptionExtension) initClients() error { for _, client := range svc.clients { - if err := client.SetEncryptionService(svc); err != nil { + if err := client.SetEncryptionExtension(svc); err != nil { return err } } @@ -110,7 +110,7 @@ func (svc *tinkEncryptionService) initClients() error { return nil } -func (svc *tinkEncryptionService) callbackOnEnable() error { +func (svc *tinkEncryptionExtension) callbackOnEnable() error { for _, client := range svc.clients { if err := client.EnableEncryption(); err != nil { return err @@ -120,7 +120,7 @@ func (svc *tinkEncryptionService) callbackOnEnable() error { return nil } -func (svc *tinkEncryptionService) callbackOnRotation() error { +func (svc *tinkEncryptionExtension) callbackOnRotation() error { for _, client := range svc.clients { if err := client.MigrateEncryption(svc); err != nil { return err @@ -130,7 +130,7 @@ func (svc *tinkEncryptionService) callbackOnRotation() error { return nil } -func (svc *tinkEncryptionService) callbackOnDisable() error { +func (svc *tinkEncryptionExtension) callbackOnDisable() error { for _, client := range svc.clients { if err := client.MigrateEncryption(&noEncryption{}); err != nil { return err diff --git a/server/model/encryption.go b/server/extensions/encryption/types/encryption.go similarity index 68% rename from server/model/encryption.go rename to server/extensions/encryption/types/encryption.go index d3d9878509..e81fb88554 100644 --- a/server/model/encryption.go +++ b/server/extensions/encryption/types/encryption.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package model +package types // EncryptionBuilder is user API to obtain correctly configured encryption type EncryptionBuilder interface { @@ -20,22 +20,22 @@ type EncryptionBuilder interface { Build() error } -type EncryptionServiceBuilder interface { - WithClients(clients []EncryptionClient) EncryptionServiceBuilder - Build() (EncryptionService, error) +type EncryptionExtensionBuilder interface { + WithClients(clients []EncryptionClient) EncryptionExtensionBuilder + Build() (EncryptionExtension, error) } -type EncryptionService interface { +type EncryptionExtension interface { Encrypt(plaintext, associatedData string) (string, error) Decrypt(ciphertext, associatedData string) (string, error) Disable() error } type EncryptionClient interface { - // SetEncryptionService should be used only by EncryptionServiceBuilder - SetEncryptionService(encryption EncryptionService) error - // EnableEncryption should encrypt all service data + // SetEncryptionExtension should be used only by EncryptionExtensionBuilder + SetEncryptionExtension(encryption EncryptionExtension) error + // EnableEncryption should encrypt all extension data EnableEncryption() error - // MigrateEncryption should decrypt all existing data and encrypt it with new encryption service - MigrateEncryption(newEncryption EncryptionService) error + // MigrateEncryption should decrypt all existing data and encrypt it with new encryption extension + MigrateEncryption(newEncryption EncryptionExtension) error } diff --git a/server/extensions/encryption/wrapper/store/secret_store_wrapper.go b/server/extensions/encryption/wrapper/store/secret_store_wrapper.go index d1ce549b72..66c3a1cbc5 100644 --- a/server/extensions/encryption/wrapper/store/secret_store_wrapper.go +++ b/server/extensions/encryption/wrapper/store/secret_store_wrapper.go @@ -21,12 +21,13 @@ import ( "github.com/rs/zerolog/log" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" ) type EncryptedSecretStore struct { store model.SecretStore - encryption model.EncryptionService + encryption types.EncryptionExtension } // ensure wrapper match interface @@ -37,11 +38,11 @@ func NewSecretStore(secretStore model.SecretStore) *EncryptedSecretStore { return &wrapper } -func (wrapper *EncryptedSecretStore) SetEncryptionService(service model.EncryptionService) error { +func (wrapper *EncryptedSecretStore) SetEncryptionExtension(extension types.EncryptionExtension) error { if wrapper.encryption != nil { return errors.New(errMessageInitSeveralTimes) } - wrapper.encryption = service + wrapper.encryption = extension return nil } @@ -63,7 +64,7 @@ func (wrapper *EncryptedSecretStore) EnableEncryption() error { return nil } -func (wrapper *EncryptedSecretStore) MigrateEncryption(newEncryptionService model.EncryptionService) error { +func (wrapper *EncryptedSecretStore) MigrateEncryption(newEncryptionExtension types.EncryptionExtension) error { log.Warn().Msg(logMessageMigratingSecretsEncryption) secrets, err := wrapper.store.SecretListAll() if err != nil { @@ -72,7 +73,7 @@ func (wrapper *EncryptedSecretStore) MigrateEncryption(newEncryptionService mode if err := wrapper.decryptList(secrets); err != nil { return err } - wrapper.encryption = newEncryptionService + wrapper.encryption = newEncryptionExtension for _, secret := range secrets { if err := wrapper.encrypt(secret); err != nil { return err diff --git a/server/extensions/environment/service.go b/server/extensions/environment/extension.go similarity index 88% rename from server/extensions/environment/service.go rename to server/extensions/environment/extension.go index f8ad148215..2ba7f2f705 100644 --- a/server/extensions/environment/service.go +++ b/server/extensions/environment/extension.go @@ -16,7 +16,7 @@ package environment import "go.woodpecker-ci.org/woodpecker/v2/server/model" -// Service defines a service for managing environment variables. -type Service interface { +// Extension defines a extension for managing environment variables. +type Extension interface { EnvironList(*model.Repo) ([]*model.Environ, error) } diff --git a/server/extensions/environment/parse.go b/server/extensions/environment/parse.go index 36de3cb5b1..a8fa63b4ed 100644 --- a/server/extensions/environment/parse.go +++ b/server/extensions/environment/parse.go @@ -26,8 +26,8 @@ type builtin struct { globals []*model.Environ } -// Parse returns a model.EnvironService based on a string slice where key and value are separated by a ":" delimiter. -func Parse(params []string) Service { +// Parse returns a model.EnvironExtension based on a string slice where key and value are separated by a ":" delimiter. +func Parse(params []string) Extension { var globals []*model.Environ for _, item := range params { diff --git a/server/extensions/environment/parse_test.go b/server/extensions/environment/parse_test.go index 97e7575698..5a96b5ec7f 100644 --- a/server/extensions/environment/parse_test.go +++ b/server/extensions/environment/parse_test.go @@ -21,25 +21,25 @@ import ( ) func TestParse(t *testing.T) { - service := Parse([]string{}) - env, err := service.EnvironList(nil) + extension := Parse([]string{}) + env, err := extension.EnvironList(nil) assert.NoError(t, err) assert.Empty(t, env) - service = Parse([]string{"ENV:value"}) - env, err = service.EnvironList(nil) + extension = Parse([]string{"ENV:value"}) + env, err = extension.EnvironList(nil) assert.NoError(t, err) assert.Len(t, env, 1) assert.Equal(t, env[0].Name, "ENV") assert.Equal(t, env[0].Value, "value") - service = Parse([]string{"ENV:value", "ENV2:value2"}) - env, err = service.EnvironList(nil) + extension = Parse([]string{"ENV:value", "ENV2:value2"}) + env, err = extension.EnvironList(nil) assert.NoError(t, err) assert.Len(t, env, 2) - service = Parse([]string{"ENV:value", "ENV2:value2", "ENV3_WITHOUT_VALUE"}) - env, err = service.EnvironList(nil) + extension = Parse([]string{"ENV:value", "ENV2:value2", "ENV3_WITHOUT_VALUE"}) + env, err = extension.EnvironList(nil) assert.NoError(t, err) assert.Len(t, env, 2) } diff --git a/server/extensions/manager.go b/server/extensions/manager.go index 51f6b5bd57..4d24b1d3e2 100644 --- a/server/extensions/manager.go +++ b/server/extensions/manager.go @@ -28,10 +28,10 @@ import ( ) type Manager struct { - secret secret.Service - registry registry.Service - config config.Service - environment environment.Service + secret secret.Extension + registry registry.Extension + config config.Extension + environment environment.Extension signaturePrivateKey crypto.PrivateKey signaturePublicKey crypto.PublicKey } @@ -47,7 +47,7 @@ func NewManager(c *cli.Context, store store.Store) (*Manager, error) { signaturePublicKey: signaturePublicKey, secret: setupSecretExtension(store), registry: setupRegistryExtension(store, c.String("docker-config")), - config: setupConfigService(c, signaturePrivateKey), + config: setupConfigExtension(c, signaturePrivateKey), environment: environment.Parse(c.StringSlice("environment")), }, nil } @@ -56,26 +56,26 @@ func (e *Manager) SignaturePublicKey() crypto.PublicKey { return e.signaturePublicKey } -func (e *Manager) SecretServiceFromRepo(_ *model.Repo) secret.Service { +func (e *Manager) SecretExtensionFromRepo(_ *model.Repo) secret.Extension { return e.secret } -func (e *Manager) SecretService() secret.Service { +func (e *Manager) SecretExtension() secret.Extension { return e.secret } -func (e *Manager) RegistryServiceFromRepo(_ *model.Repo) registry.Service { +func (e *Manager) RegistryExtensionFromRepo(_ *model.Repo) registry.Extension { return e.registry } -func (e *Manager) RegistryService() registry.Service { +func (e *Manager) RegistryExtension() registry.Extension { return e.registry } -func (e *Manager) ConfigServiceFromRepo(_ *model.Repo) config.Service { +func (e *Manager) ConfigExtensionFromRepo(_ *model.Repo) config.Extension { return e.config } -func (e *Manager) EnvironmentService() environment.Service { +func (e *Manager) EnvironmentExtension() environment.Extension { return e.environment } diff --git a/server/extensions/registry/combined.go b/server/extensions/registry/combined.go index 4f52bca1a5..20747d5cc3 100644 --- a/server/extensions/registry/combined.go +++ b/server/extensions/registry/combined.go @@ -19,11 +19,11 @@ import ( ) type combined struct { - registries []ReadOnlyService - dbRegistry Service + registries []ReadOnlyExtension + dbRegistry Extension } -func NewCombined(dbRegistry Service, registries ...ReadOnlyService) Service { +func NewCombined(dbRegistry Extension, registries ...ReadOnlyExtension) Extension { registries = append(registries, dbRegistry) return &combined{ registries: registries, diff --git a/server/extensions/registry/db.go b/server/extensions/registry/db.go index 5e7339557f..29be0a5a35 100644 --- a/server/extensions/registry/db.go +++ b/server/extensions/registry/db.go @@ -22,8 +22,8 @@ type db struct { store model.RegistryStore } -// New returns a new local registry service. -func NewDB(store model.RegistryStore) Service { +// New returns a new local registry extension. +func NewDB(store model.RegistryStore) Extension { return &db{store} } diff --git a/server/extensions/registry/service.go b/server/extensions/registry/extension.go similarity index 85% rename from server/extensions/registry/service.go rename to server/extensions/registry/extension.go index 252ce9d9a9..cd39b28b52 100644 --- a/server/extensions/registry/service.go +++ b/server/extensions/registry/extension.go @@ -16,8 +16,8 @@ package registry import "go.woodpecker-ci.org/woodpecker/v2/server/model" -// Service defines a service for managing registries. -type Service interface { +// Extension defines a extension for managing registries. +type Extension interface { RegistryFind(*model.Repo, string) (*model.Registry, error) RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error) RegistryCreate(*model.Repo, *model.Registry) error @@ -25,8 +25,8 @@ type Service interface { RegistryDelete(*model.Repo, string) error } -// ReadOnlyService defines a service for managing registries. -type ReadOnlyService interface { +// ReadOnlyExtension defines a extension for managing registries. +type ReadOnlyExtension interface { RegistryFind(*model.Repo, string) (*model.Registry, error) RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error) } diff --git a/server/extensions/registry/filesystem.go b/server/extensions/registry/filesystem.go index 4d98818cc0..dc224e6f08 100644 --- a/server/extensions/registry/filesystem.go +++ b/server/extensions/registry/filesystem.go @@ -31,7 +31,7 @@ type filesystem struct { path string } -func NewFilesystem(path string) ReadOnlyService { +func NewFilesystem(path string) ReadOnlyExtension { return &filesystem{path} } diff --git a/server/extensions/secret/db.go b/server/extensions/secret/db.go index b3d3202618..999a0e6acb 100644 --- a/server/extensions/secret/db.go +++ b/server/extensions/secret/db.go @@ -22,8 +22,8 @@ type db struct { store model.SecretStore } -// NewDB returns a new local secret service. -func NewDB(store model.SecretStore) Service { +// NewDB returns a new local secret extension. +func NewDB(store model.SecretStore) Extension { return &db{store: store} } diff --git a/server/extensions/secret/service.go b/server/extensions/secret/extension.go similarity index 95% rename from server/extensions/secret/service.go rename to server/extensions/secret/extension.go index 24568c195b..6d56feb795 100644 --- a/server/extensions/secret/service.go +++ b/server/extensions/secret/extension.go @@ -16,8 +16,8 @@ package secret import "go.woodpecker-ci.org/woodpecker/v2/server/model" -// Service defines a service for managing secrets. -type Service interface { +// Extension defines a extension for managing secrets. +type Extension interface { SecretListPipeline(*model.Repo, *model.Pipeline, *model.ListOptions) ([]*model.Secret, error) // Repository secrets SecretFind(*model.Repo, string) (*model.Secret, error) diff --git a/server/extensions/setup.go b/server/extensions/setup.go index 570197997d..040824ecea 100644 --- a/server/extensions/setup.go +++ b/server/extensions/setup.go @@ -32,7 +32,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) -func setupRegistryExtension(store store.Store, dockerConfig string) registry.Service { +func setupRegistryExtension(store store.Store, dockerConfig string) registry.Extension { if dockerConfig != "" { return registry.NewCombined( registry.NewDB(store), @@ -43,24 +43,24 @@ func setupRegistryExtension(store store.Store, dockerConfig string) registry.Ser return registry.NewDB(store) } -func setupSecretExtension(store store.Store) secret.Service { +func setupSecretExtension(store store.Store) secret.Extension { // TODO(1544): fix encrypted store // // encryption // encryptedSecretStore := encryptedStore.NewSecretStore(v) // err := encryption.Encryption(c, v).WithClient(encryptedSecretStore).Build() // if err != nil { - // log.Fatal().Err(err).Msg("could not create encryption service") + // log.Fatal().Err(err).Msg("could not create encryption extension") // } - // server.Config.Services.Secrets = setupSecretService(c, encryptedSecretStore) + // server.Config.Extensions.Secrets = setupSecretExtension(c, encryptedSecretStore) return secret.NewDB(store) } -func setupConfigService(c *cli.Context, privateSignatureKey crypto.PrivateKey) config.Service { +func setupConfigExtension(c *cli.Context, privateSignatureKey crypto.PrivateKey) config.Extension { timeout := c.Duration("forge-timeout") configFetcher := config.NewForge(timeout) - if endpoint := c.String("config-service-endpoint"); endpoint != "" { + if endpoint := c.String("config-extension-endpoint"); endpoint != "" { httpFetcher := config.NewHTTP(endpoint, privateSignatureKey, nil) return config.NewCombined(configFetcher, httpFetcher) } diff --git a/server/pipeline/create.go b/server/pipeline/create.go index 4e41b6100e..d2c4a054c0 100644 --- a/server/pipeline/create.go +++ b/server/pipeline/create.go @@ -69,7 +69,7 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline } // fetch the pipeline file from the forge - configService := server.Config.Services.Manager.ConfigServiceFromRepo(repo) + configService := server.Config.Services.Manager.ConfigExtensionFromRepo(repo) forgeYamlConfigs, configFetchErr := configService.Fetch(ctx, _forge, repoUser, repo, pipeline) if errors.Is(configFetchErr, &forge_types.ErrConfigNotFound{}) { log.Debug().Str("repo", repo.FullName).Err(configFetchErr).Msgf("cannot find config '%s' in '%s' with user: '%s'", repo.Config, pipeline.Ref, repoUser.Login) diff --git a/server/pipeline/items.go b/server/pipeline/items.go index 35206cd117..9f94d1c106 100644 --- a/server/pipeline/items.go +++ b/server/pipeline/items.go @@ -42,13 +42,13 @@ func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *mod log.Error().Err(err).Str("repo", repo.FullName).Msgf("error getting last pipeline before pipeline number '%d'", currentPipeline.Number) } - secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + secretService := server.Config.Services.Manager.SecretExtensionFromRepo(repo) secs, err := secretService.SecretListPipeline(repo, currentPipeline, &model.ListOptions{All: true}) if err != nil { log.Error().Err(err).Msgf("error getting secrets for %s#%d", repo.FullName, currentPipeline.Number) } - registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + registryService := server.Config.Services.Manager.RegistryExtensionFromRepo(repo) regs, err := registryService.RegistryList(repo, &model.ListOptions{All: true}) if err != nil { log.Error().Err(err).Msgf("error getting registry credentials for %s#%d", repo.FullName, currentPipeline.Number) @@ -58,7 +58,7 @@ func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *mod envs = map[string]string{} } - environmentService := server.Config.Services.Manager.EnvironmentService() + environmentService := server.Config.Services.Manager.EnvironmentExtension() if environmentService != nil { globals, _ := environmentService.EnvironList(repo) for _, global := range globals { diff --git a/server/pipeline/restart.go b/server/pipeline/restart.go index 29045bf072..936ed573ee 100644 --- a/server/pipeline/restart.go +++ b/server/pipeline/restart.go @@ -50,7 +50,7 @@ func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipelin } // If the config extension is active we should refetch the config in case something changed - configService := server.Config.Services.Manager.ConfigServiceFromRepo(repo) + configService := server.Config.Services.Manager.ConfigExtensionFromRepo(repo) pipelineFiles, err = configService.Fetch(ctx, forge, user, repo, lastPipeline) if err != nil { return nil, &ErrBadRequest{ From a2df3a04ffbe7202ed94713a11315c3d41ace9bf Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:11:11 +0100 Subject: [PATCH 20/28] add todos --- server/extensions/manager.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/server/extensions/manager.go b/server/extensions/manager.go index 4d24b1d3e2..6d2d2f55d7 100644 --- a/server/extensions/manager.go +++ b/server/extensions/manager.go @@ -57,6 +57,7 @@ func (e *Manager) SignaturePublicKey() crypto.PublicKey { } func (e *Manager) SecretExtensionFromRepo(_ *model.Repo) secret.Extension { + // TODO: decied based on repo property which secret extension to use return e.secret } @@ -65,6 +66,7 @@ func (e *Manager) SecretExtension() secret.Extension { } func (e *Manager) RegistryExtensionFromRepo(_ *model.Repo) registry.Extension { + // TODO: decied based on repo property which registry extension to use return e.registry } @@ -73,6 +75,7 @@ func (e *Manager) RegistryExtension() registry.Extension { } func (e *Manager) ConfigExtensionFromRepo(_ *model.Repo) config.Extension { + // TODO: decied based on repo property which config extension to use return e.config } From b3ef5fb604c45cd3a31c3b85d24e8b80a3cf4da4 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:17:10 +0100 Subject: [PATCH 21/28] adjust todo --- server/extensions/config/combined.go | 4 ++-- server/extensions/config/http.go | 16 ++++++++-------- 2 files changed, 10 insertions(+), 10 deletions(-) diff --git a/server/extensions/config/combined.go b/server/extensions/config/combined.go index 2b8f5739fa..a6da3e33e7 100644 --- a/server/extensions/config/combined.go +++ b/server/extensions/config/combined.go @@ -34,12 +34,12 @@ func NewCombined(extensions ...Extension) Extension { func (c *combined) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (files []*types.FileMeta, err error) { for _, s := range c.extensions { - // TODO(anbraten): This is a hack to get the current configs into the http extension + // TODO(anbraten): This is a hack to get the current configs into the http extension, will be removed when removing deprecatedCurrentConfigs _s, ok := s.(*http) if !ok { log.Err(err).Msg("http extension is not of type http") } else { - _s.currentConfigs = files + _s.deprecatedCurrentConfigs = files } files, err = s.Fetch(ctx, forge, user, repo, pipeline) diff --git a/server/extensions/config/http.go b/server/extensions/config/http.go index 65c554a29f..86e3c11494 100644 --- a/server/extensions/config/http.go +++ b/server/extensions/config/http.go @@ -26,9 +26,9 @@ import ( ) type http struct { - endpoint string - privateKey crypto.PrivateKey - currentConfigs []*types.FileMeta + endpoint string + privateKey crypto.PrivateKey + deprecatedCurrentConfigs []*types.FileMeta } // configData same as forge.FileMeta but with json tags and string data @@ -43,20 +43,20 @@ type requestStructure struct { Netrc *model.Netrc `json:"netrc"` // @deprecated use netrc data to fetch the config by yourself instead - DeprecatedConfiguration []*configData `json:"configs"` // TODO: remove at some point + DeprecatedConfiguration []*configData `json:"configs"` // TODO: remove in next major release } type responseStructure struct { Configs []*configData `json:"configs"` } -func NewHTTP(endpoint string, privateKey crypto.PrivateKey, currentConfigs []*types.FileMeta) Extension { - return &http{endpoint, privateKey, currentConfigs} +func NewHTTP(endpoint string, privateKey crypto.PrivateKey, deprecatedCurrentConfigs []*types.FileMeta) Extension { + return &http{endpoint, privateKey, deprecatedCurrentConfigs} } func (h *http) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ([]*types.FileMeta, error) { - currentConfigs := make([]*configData, len(h.currentConfigs)) - for i, pipe := range h.currentConfigs { + currentConfigs := make([]*configData, len(h.deprecatedCurrentConfigs)) + for i, pipe := range h.deprecatedCurrentConfigs { currentConfigs[i] = &configData{Name: pipe.Name, Data: string(pipe.Data)} } From 6ec6ef80e0079af446b69f20187f749acce616d6 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:22:05 +0100 Subject: [PATCH 22/28] rename to ExtensionsMangager --- cmd/server/server.go | 7 ++++--- server/api/global_secret.go | 10 +++++----- server/api/org_secret.go | 10 +++++----- server/api/registry.go | 10 +++++----- server/api/repo_secret.go | 10 +++++----- server/api/signature_public_key.go | 2 +- server/config.go | 4 ++-- server/pipeline/create.go | 2 +- server/pipeline/items.go | 6 +++--- server/pipeline/restart.go | 2 +- 10 files changed, 32 insertions(+), 31 deletions(-) diff --git a/cmd/server/server.go b/cmd/server/server.go index 836709817a..ad8736f45b 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -38,6 +38,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc/proto" "go.woodpecker-ci.org/woodpecker/v2/server" "go.woodpecker-ci.org/woodpecker/v2/server/cron" + "go.woodpecker-ci.org/woodpecker/v2/server/extensions" "go.woodpecker-ci.org/woodpecker/v2/server/extensions/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/forge" woodpeckerGrpcServer "go.woodpecker-ci.org/woodpecker/v2/server/grpc" @@ -279,11 +280,11 @@ func setupEvilGlobals(c *cli.Context, s store.Store, f forge.Forge) error { server.Config.Services.Pubsub = pubsub.New() server.Config.Services.Membership = setupMembershipService(c, f) - servicesManager, err := services.NewManager(c, s) + extensionsMangager, err := extensions.NewManager(c, s) if err != nil { - return fmt.Errorf("could not setup services manager: %w", err) + return fmt.Errorf("could not setup extensions manager: %w", err) } - server.Config.Services.Manager = servicesManager + server.Config.ExtensionsManager = extensionsMangager // authentication server.Config.Pipeline.AuthenticatePublicRepos = c.Bool("authenticate-public-repos") diff --git a/server/api/global_secret.go b/server/api/global_secret.go index a9421ccd55..acc29794e3 100644 --- a/server/api/global_secret.go +++ b/server/api/global_secret.go @@ -35,7 +35,7 @@ import ( // @Param page query int false "for response pagination, page offset number" default(1) // @Param perPage query int false "for response pagination, max items per page" default(50) func GetGlobalSecretList(c *gin.Context) { - secretExtension := server.Config.Services.Manager.SecretExtension() + secretExtension := server.Config.ExtensionsManager.SecretExtension() list, err := secretExtension.GlobalSecretList(session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting global secret list. %s", err) @@ -60,7 +60,7 @@ func GetGlobalSecretList(c *gin.Context) { // @Param secret path string true "the secret's name" func GetGlobalSecret(c *gin.Context) { name := c.Param("secret") - secretExtension := server.Config.Services.Manager.SecretExtension() + secretExtension := server.Config.ExtensionsManager.SecretExtension() secret, err := secretExtension.GlobalSecretFind(name) if err != nil { handleDBError(c, err) @@ -95,7 +95,7 @@ func PostGlobalSecret(c *gin.Context) { return } - secretExtension := server.Config.Services.Manager.SecretExtension() + secretExtension := server.Config.ExtensionsManager.SecretExtension() if err := secretExtension.GlobalSecretCreate(secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting global secret %q. %s", in.Name, err) return @@ -123,7 +123,7 @@ func PatchGlobalSecret(c *gin.Context) { return } - secretExtension := server.Config.Services.Manager.SecretExtension() + secretExtension := server.Config.ExtensionsManager.SecretExtension() secret, err := secretExtension.GlobalSecretFind(name) if err != nil { handleDBError(c, err) @@ -162,7 +162,7 @@ func PatchGlobalSecret(c *gin.Context) { // @Param secret path string true "the secret's name" func DeleteGlobalSecret(c *gin.Context) { name := c.Param("secret") - secretExtension := server.Config.Services.Manager.SecretExtension() + secretExtension := server.Config.ExtensionsManager.SecretExtension() if err := secretExtension.GlobalSecretDelete(name); err != nil { handleDBError(c, err) return diff --git a/server/api/org_secret.go b/server/api/org_secret.go index bd70c4ddd8..05c102cf32 100644 --- a/server/api/org_secret.go +++ b/server/api/org_secret.go @@ -44,7 +44,7 @@ func GetOrgSecret(c *gin.Context) { return } - secretExtension := server.Config.Services.Manager.SecretExtension() + secretExtension := server.Config.ExtensionsManager.SecretExtension() secret, err := secretExtension.OrgSecretFind(orgID, name) if err != nil { handleDBError(c, err) @@ -71,7 +71,7 @@ func GetOrgSecretList(c *gin.Context) { return } - secretExtension := server.Config.Services.Manager.SecretExtension() + secretExtension := server.Config.ExtensionsManager.SecretExtension() list, err := secretExtension.OrgSecretList(orgID, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting secret list for %q. %s", orgID, err) @@ -119,7 +119,7 @@ func PostOrgSecret(c *gin.Context) { return } - secretExtension := server.Config.Services.Manager.SecretExtension() + secretExtension := server.Config.ExtensionsManager.SecretExtension() if err := secretExtension.OrgSecretCreate(orgID, secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting org %q secret %q. %s", orgID, in.Name, err) return @@ -153,7 +153,7 @@ func PatchOrgSecret(c *gin.Context) { return } - secretExtension := server.Config.Services.Manager.SecretExtension() + secretExtension := server.Config.ExtensionsManager.SecretExtension() secret, err := secretExtension.OrgSecretFind(orgID, name) if err != nil { handleDBError(c, err) @@ -199,7 +199,7 @@ func DeleteOrgSecret(c *gin.Context) { return } - secretExtension := server.Config.Services.Manager.SecretExtension() + secretExtension := server.Config.ExtensionsManager.SecretExtension() if err := secretExtension.OrgSecretDelete(orgID, name); err != nil { handleDBError(c, err) return diff --git a/server/api/registry.go b/server/api/registry.go index f08dbd41ec..0d8ce4a1db 100644 --- a/server/api/registry.go +++ b/server/api/registry.go @@ -38,7 +38,7 @@ func GetRegistry(c *gin.Context) { repo := session.Repo(c) name := c.Param("registry") - registryExtension := server.Config.Services.Manager.RegistryExtensionFromRepo(repo) + registryExtension := server.Config.ExtensionsManager.RegistryExtensionFromRepo(repo) registry, err := registryExtension.RegistryFind(repo, name) if err != nil { handleDBError(c, err) @@ -76,7 +76,7 @@ func PostRegistry(c *gin.Context) { return } - registryExtension := server.Config.Services.Manager.RegistryExtensionFromRepo(repo) + registryExtension := server.Config.ExtensionsManager.RegistryExtensionFromRepo(repo) if err := registryExtension.RegistryCreate(repo, registry); err != nil { c.String(http.StatusInternalServerError, "Error inserting registry %q. %s", in.Address, err) return @@ -108,7 +108,7 @@ func PatchRegistry(c *gin.Context) { return } - registryExtension := server.Config.Services.Manager.RegistryExtensionFromRepo(repo) + registryExtension := server.Config.ExtensionsManager.RegistryExtensionFromRepo(repo) registry, err := registryExtension.RegistryFind(repo, name) if err != nil { handleDBError(c, err) @@ -145,7 +145,7 @@ func PatchRegistry(c *gin.Context) { // @Param perPage query int false "for response pagination, max items per page" default(50) func GetRegistryList(c *gin.Context) { repo := session.Repo(c) - registryExtension := server.Config.Services.Manager.RegistryExtensionFromRepo(repo) + registryExtension := server.Config.ExtensionsManager.RegistryExtensionFromRepo(repo) list, err := registryExtension.RegistryList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting registry list. %s", err) @@ -173,7 +173,7 @@ func DeleteRegistry(c *gin.Context) { repo := session.Repo(c) name := c.Param("registry") - registryExtension := server.Config.Services.Manager.RegistryExtensionFromRepo(repo) + registryExtension := server.Config.ExtensionsManager.RegistryExtensionFromRepo(repo) err := registryExtension.RegistryDelete(repo, name) if err != nil { handleDBError(c, err) diff --git a/server/api/repo_secret.go b/server/api/repo_secret.go index 4ef271f230..e41f2f5b34 100644 --- a/server/api/repo_secret.go +++ b/server/api/repo_secret.go @@ -39,7 +39,7 @@ func GetSecret(c *gin.Context) { repo := session.Repo(c) name := c.Param("secret") - secretExtension := server.Config.Services.Manager.SecretExtensionFromRepo(repo) + secretExtension := server.Config.ExtensionsManager.SecretExtensionFromRepo(repo) secret, err := secretExtension.SecretFind(repo, name) if err != nil { handleDBError(c, err) @@ -78,7 +78,7 @@ func PostSecret(c *gin.Context) { return } - secretExtension := server.Config.Services.Manager.SecretExtensionFromRepo(repo) + secretExtension := server.Config.ExtensionsManager.SecretExtensionFromRepo(repo) if err := secretExtension.SecretCreate(repo, secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting secret %q. %s", in.Name, err) return @@ -110,7 +110,7 @@ func PatchSecret(c *gin.Context) { return } - secretExtension := server.Config.Services.Manager.SecretExtensionFromRepo(repo) + secretExtension := server.Config.ExtensionsManager.SecretExtensionFromRepo(repo) secret, err := secretExtension.SecretFind(repo, name) if err != nil { handleDBError(c, err) @@ -150,7 +150,7 @@ func PatchSecret(c *gin.Context) { // @Param perPage query int false "for response pagination, max items per page" default(50) func GetSecretList(c *gin.Context) { repo := session.Repo(c) - secretExtension := server.Config.Services.Manager.SecretExtensionFromRepo(repo) + secretExtension := server.Config.ExtensionsManager.SecretExtensionFromRepo(repo) list, err := secretExtension.SecretList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting secret list. %s", err) @@ -178,7 +178,7 @@ func DeleteSecret(c *gin.Context) { repo := session.Repo(c) name := c.Param("secret") - secretExtension := server.Config.Services.Manager.SecretExtensionFromRepo(repo) + secretExtension := server.Config.ExtensionsManager.SecretExtensionFromRepo(repo) if err := secretExtension.SecretDelete(repo, name); err != nil { handleDBError(c, err) return diff --git a/server/api/signature_public_key.go b/server/api/signature_public_key.go index b9364166b4..1a779c50b3 100644 --- a/server/api/signature_public_key.go +++ b/server/api/signature_public_key.go @@ -34,7 +34,7 @@ import ( // @Tags System // @Param Authorization header string true "Insert your personal access token" default(Bearer ) func GetSignaturePublicKey(c *gin.Context) { - b, err := x509.MarshalPKIXPublicKey(server.Config.Services.Manager.SignaturePublicKey()) + b, err := x509.MarshalPKIXPublicKey(server.Config.ExtensionsManager.SignaturePublicKey()) if err != nil { log.Error().Err(err).Msg("can't marshal public key") c.AbortWithStatus(http.StatusInternalServerError) diff --git a/server/config.go b/server/config.go index 208c452384..dd3bf45297 100644 --- a/server/config.go +++ b/server/config.go @@ -37,9 +37,9 @@ var Config = struct { Logs logging.Log Forge forge.Forge Membership cache.MembershipService - Manager *extensions.Manager } - Server struct { + ExtensionsManager *extensions.Manager + Server struct { Key string Cert string OAuthHost string diff --git a/server/pipeline/create.go b/server/pipeline/create.go index d2c4a054c0..f669509d2f 100644 --- a/server/pipeline/create.go +++ b/server/pipeline/create.go @@ -69,7 +69,7 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline } // fetch the pipeline file from the forge - configService := server.Config.Services.Manager.ConfigExtensionFromRepo(repo) + configService := server.Config.ExtensionsManager.ConfigExtensionFromRepo(repo) forgeYamlConfigs, configFetchErr := configService.Fetch(ctx, _forge, repoUser, repo, pipeline) if errors.Is(configFetchErr, &forge_types.ErrConfigNotFound{}) { log.Debug().Str("repo", repo.FullName).Err(configFetchErr).Msgf("cannot find config '%s' in '%s' with user: '%s'", repo.Config, pipeline.Ref, repoUser.Login) diff --git a/server/pipeline/items.go b/server/pipeline/items.go index 9f94d1c106..d1955fcd4d 100644 --- a/server/pipeline/items.go +++ b/server/pipeline/items.go @@ -42,13 +42,13 @@ func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *mod log.Error().Err(err).Str("repo", repo.FullName).Msgf("error getting last pipeline before pipeline number '%d'", currentPipeline.Number) } - secretService := server.Config.Services.Manager.SecretExtensionFromRepo(repo) + secretService := server.Config.ExtensionsManager.SecretExtensionFromRepo(repo) secs, err := secretService.SecretListPipeline(repo, currentPipeline, &model.ListOptions{All: true}) if err != nil { log.Error().Err(err).Msgf("error getting secrets for %s#%d", repo.FullName, currentPipeline.Number) } - registryService := server.Config.Services.Manager.RegistryExtensionFromRepo(repo) + registryService := server.Config.ExtensionsManager.RegistryExtensionFromRepo(repo) regs, err := registryService.RegistryList(repo, &model.ListOptions{All: true}) if err != nil { log.Error().Err(err).Msgf("error getting registry credentials for %s#%d", repo.FullName, currentPipeline.Number) @@ -58,7 +58,7 @@ func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *mod envs = map[string]string{} } - environmentService := server.Config.Services.Manager.EnvironmentExtension() + environmentService := server.Config.ExtensionsManager.EnvironmentExtension() if environmentService != nil { globals, _ := environmentService.EnvironList(repo) for _, global := range globals { diff --git a/server/pipeline/restart.go b/server/pipeline/restart.go index 936ed573ee..2482c895de 100644 --- a/server/pipeline/restart.go +++ b/server/pipeline/restart.go @@ -50,7 +50,7 @@ func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipelin } // If the config extension is active we should refetch the config in case something changed - configService := server.Config.Services.Manager.ConfigExtensionFromRepo(repo) + configService := server.Config.ExtensionsManager.ConfigExtensionFromRepo(repo) pipelineFiles, err = configService.Fetch(ctx, forge, user, repo, lastPipeline) if err != nil { return nil, &ErrBadRequest{ From 7d0e4423982cd255c2d49fe7c11f0bcea2bad32b Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Fri, 9 Feb 2024 10:25:06 +0100 Subject: [PATCH 23/28] add migration note --- docs/docs/20-usage/15-terminiology/index.md | 1 + docs/docs/91-migrations.md | 1 + 2 files changed, 2 insertions(+) diff --git a/docs/docs/20-usage/15-terminiology/index.md b/docs/docs/20-usage/15-terminiology/index.md index 4e4f614894..f031058597 100644 --- a/docs/docs/20-usage/15-terminiology/index.md +++ b/docs/docs/20-usage/15-terminiology/index.md @@ -31,6 +31,7 @@ - **YAML File**: A file format used to define and configure [workflows][Workflow]. - **Dependency**: [Workflows][Workflow] can depend on each other, and if possible, they are executed in parallel. - **Status**: Status refers to the outcome of a step or [workflow][Workflow] after it has been executed, determined by the internal command exit code. At the end of a [workflow][Workflow], its status is sent to the [forge][Forge]. +- **Extension**: Some parts of woodpecker like the secrets, registry storage or config fetcher / parser can be replaced through extensions. ## Pipeline events diff --git a/docs/docs/91-migrations.md b/docs/docs/91-migrations.md index 328d87a625..edf7ff7958 100644 --- a/docs/docs/91-migrations.md +++ b/docs/docs/91-migrations.md @@ -7,6 +7,7 @@ Some versions need some changes to the server configuration or the pipeline conf - Deprecated `steps.[name].group` in favor of `steps.[name].depends_on` (see [workflow syntax](./20-usage/20-workflow-syntax.md#depends_on) to learn how to set dependencies) - Removed `WOODPECKER_ROOT_PATH` and `WOODPECKER_ROOT_URL` config variables. Use `WOODPECKER_HOST` with a path instead - Pipelines without a config file will now be skipped instead of failing +- Config extensions list of `configs` from forge removed. Use `netrc` credentials to fetch the required configs in your extension itself. ## 2.0.0 From d5c574dd9e841bbf18172997f17ee4270bafe8a1 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Sun, 11 Feb 2024 14:44:14 +0100 Subject: [PATCH 24/28] rename back to service --- cmd/server/server.go | 10 ++--- docs/docs/20-usage/15-terminiology/index.md | 2 +- server/api/global_secret.go | 22 +++++----- server/api/org_secret.go | 22 +++++----- server/api/registry.go | 22 +++++----- server/api/repo_secret.go | 22 +++++----- server/api/signature_public_key.go | 2 +- server/config.go | 8 ++-- server/pipeline/create.go | 2 +- server/pipeline/items.go | 6 +-- server/pipeline/restart.go | 4 +- .../config/combined.go | 12 +++--- .../config/combined_test.go | 2 +- .../{extensions => services}/config/forge.go | 2 +- .../config/forge_test.go | 2 +- .../{extensions => services}/config/http.go | 4 +- .../config/service.go} | 2 +- .../encryption/aes.go | 2 +- .../encryption/aes_builder.go | 2 +- .../encryption/aes_encryption.go | 0 .../encryption/aes_state.go | 0 .../encryption/aes_test.go | 0 .../encryption/constants.go | 10 ++--- .../encryption/encryption.go | 2 +- .../encryption/encryption_builder.go | 6 +-- .../encryption/no_encryption.go | 2 +- .../encryption/tink.go | 2 +- .../encryption/tink_builder.go | 2 +- .../encryption/tink_keyset.go | 0 .../encryption/tink_keyset_watcher.go | 0 .../encryption/tink_state.go | 0 .../encryption/types/encryption.go | 4 +- .../encryption/wrapper/store/constants.go | 0 .../encryption/wrapper/store/secret_store.go | 0 .../wrapper/store/secret_store_wrapper.go | 6 +-- .../environment/extension.go | 4 +- .../environment/parse.go | 4 +- .../environment/parse_test.go | 16 ++++---- server/{extensions => services}/manager.go | 40 +++++++++---------- .../permissions/admins.go | 0 .../permissions/admins_test.go | 0 .../permissions/orgs.go | 0 .../permissions/orgs_test.go | 0 .../permissions/repo_owners.go | 0 .../permissions/repo_owners_test.go | 0 .../registry/combined.go | 6 +-- .../{extensions => services}/registry/db.go | 4 +- .../registry/filesystem.go | 2 +- .../registry/service.go} | 8 ++-- server/{extensions => services}/secret/db.go | 4 +- .../secret/db_test.go | 2 +- .../secret/service.go} | 4 +- server/{extensions => services}/setup.go | 20 +++++----- server/{extensions => services}/utils/http.go | 0 .../utils/http_test.go | 2 +- 55 files changed, 149 insertions(+), 151 deletions(-) rename server/{extensions => services}/config/combined.go (81%) rename server/{extensions => services}/config/combined_test.go (99%) rename server/{extensions => services}/config/forge.go (99%) rename server/{extensions => services}/config/forge_test.go (99%) rename server/{extensions => services}/config/http.go (96%) rename server/{extensions/config/extension.go => services/config/service.go} (97%) rename server/{extensions => services}/encryption/aes.go (96%) rename server/{extensions => services}/encryption/aes_builder.go (95%) rename server/{extensions => services}/encryption/aes_encryption.go (100%) rename server/{extensions => services}/encryption/aes_state.go (100%) rename server/{extensions => services}/encryption/aes_test.go (100%) rename server/{extensions => services}/encryption/constants.go (96%) rename server/{extensions => services}/encryption/encryption.go (95%) rename server/{extensions => services}/encryption/encryption_builder.go (90%) rename server/{extensions => services}/encryption/no_encryption.go (94%) rename server/{extensions => services}/encryption/tink.go (96%) rename server/{extensions => services}/encryption/tink_builder.go (96%) rename server/{extensions => services}/encryption/tink_keyset.go (100%) rename server/{extensions => services}/encryption/tink_keyset_watcher.go (100%) rename server/{extensions => services}/encryption/tink_state.go (100%) rename server/{extensions => services}/encryption/types/encryption.go (94%) rename server/{extensions => services}/encryption/wrapper/store/constants.go (100%) rename server/{extensions => services}/encryption/wrapper/store/secret_store.go (100%) rename server/{extensions => services}/encryption/wrapper/store/secret_store_wrapper.go (94%) rename server/{extensions => services}/environment/extension.go (88%) rename server/{extensions => services}/environment/parse.go (88%) rename server/{extensions => services}/environment/parse_test.go (72%) rename server/{extensions => services}/manager.go (57%) rename server/{extensions => services}/permissions/admins.go (100%) rename server/{extensions => services}/permissions/admins_test.go (100%) rename server/{extensions => services}/permissions/orgs.go (100%) rename server/{extensions => services}/permissions/orgs_test.go (100%) rename server/{extensions => services}/permissions/repo_owners.go (100%) rename server/{extensions => services}/permissions/repo_owners_test.go (100%) rename server/{extensions => services}/registry/combined.go (93%) rename server/{extensions => services}/registry/db.go (93%) rename server/{extensions => services}/registry/filesystem.go (98%) rename server/{extensions/registry/extension.go => services/registry/service.go} (85%) rename server/{extensions => services}/secret/db.go (97%) rename server/{extensions => services}/secret/db_test.go (97%) rename server/{extensions/secret/extension.go => services/secret/service.go} (95%) rename server/{extensions => services}/setup.go (83%) rename server/{extensions => services}/utils/http.go (100%) rename server/{extensions => services}/utils/http_test.go (96%) diff --git a/cmd/server/server.go b/cmd/server/server.go index ad8736f45b..3eb3b4bbcf 100644 --- a/cmd/server/server.go +++ b/cmd/server/server.go @@ -38,8 +38,6 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/pipeline/rpc/proto" "go.woodpecker-ci.org/woodpecker/v2/server" "go.woodpecker-ci.org/woodpecker/v2/server/cron" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/forge" woodpeckerGrpcServer "go.woodpecker-ci.org/woodpecker/v2/server/grpc" "go.woodpecker-ci.org/woodpecker/v2/server/logging" @@ -47,6 +45,8 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/pubsub" "go.woodpecker-ci.org/woodpecker/v2/server/router" "go.woodpecker-ci.org/woodpecker/v2/server/router/middleware" + "go.woodpecker-ci.org/woodpecker/v2/server/services" + "go.woodpecker-ci.org/woodpecker/v2/server/services/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/store" "go.woodpecker-ci.org/woodpecker/v2/server/web" "go.woodpecker-ci.org/woodpecker/v2/shared/constant" @@ -280,11 +280,11 @@ func setupEvilGlobals(c *cli.Context, s store.Store, f forge.Forge) error { server.Config.Services.Pubsub = pubsub.New() server.Config.Services.Membership = setupMembershipService(c, f) - extensionsMangager, err := extensions.NewManager(c, s) + serviceMangager, err := services.NewManager(c, s) if err != nil { - return fmt.Errorf("could not setup extensions manager: %w", err) + return fmt.Errorf("could not setup service manager: %w", err) } - server.Config.ExtensionsManager = extensionsMangager + server.Config.Services.Manager = serviceMangager // authentication server.Config.Pipeline.AuthenticatePublicRepos = c.Bool("authenticate-public-repos") diff --git a/docs/docs/20-usage/15-terminiology/index.md b/docs/docs/20-usage/15-terminiology/index.md index f031058597..3d8f127725 100644 --- a/docs/docs/20-usage/15-terminiology/index.md +++ b/docs/docs/20-usage/15-terminiology/index.md @@ -31,7 +31,7 @@ - **YAML File**: A file format used to define and configure [workflows][Workflow]. - **Dependency**: [Workflows][Workflow] can depend on each other, and if possible, they are executed in parallel. - **Status**: Status refers to the outcome of a step or [workflow][Workflow] after it has been executed, determined by the internal command exit code. At the end of a [workflow][Workflow], its status is sent to the [forge][Forge]. -- **Extension**: Some parts of woodpecker like the secrets, registry storage or config fetcher / parser can be replaced through extensions. +- **Service extension**: Some parts of woodpecker internal services like secrets storage or config fetcher can be replaced through service extensions. ## Pipeline events diff --git a/server/api/global_secret.go b/server/api/global_secret.go index acc29794e3..b01de72e5b 100644 --- a/server/api/global_secret.go +++ b/server/api/global_secret.go @@ -35,8 +35,8 @@ import ( // @Param page query int false "for response pagination, page offset number" default(1) // @Param perPage query int false "for response pagination, max items per page" default(50) func GetGlobalSecretList(c *gin.Context) { - secretExtension := server.Config.ExtensionsManager.SecretExtension() - list, err := secretExtension.GlobalSecretList(session.Pagination(c)) + secretService := server.Config.Services.Manager.SecretService() + list, err := secretService.GlobalSecretList(session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting global secret list. %s", err) return @@ -60,8 +60,8 @@ func GetGlobalSecretList(c *gin.Context) { // @Param secret path string true "the secret's name" func GetGlobalSecret(c *gin.Context) { name := c.Param("secret") - secretExtension := server.Config.ExtensionsManager.SecretExtension() - secret, err := secretExtension.GlobalSecretFind(name) + secretService := server.Config.Services.Manager.SecretService() + secret, err := secretService.GlobalSecretFind(name) if err != nil { handleDBError(c, err) return @@ -95,8 +95,8 @@ func PostGlobalSecret(c *gin.Context) { return } - secretExtension := server.Config.ExtensionsManager.SecretExtension() - if err := secretExtension.GlobalSecretCreate(secret); err != nil { + secretService := server.Config.Services.Manager.SecretService() + if err := secretService.GlobalSecretCreate(secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting global secret %q. %s", in.Name, err) return } @@ -123,8 +123,8 @@ func PatchGlobalSecret(c *gin.Context) { return } - secretExtension := server.Config.ExtensionsManager.SecretExtension() - secret, err := secretExtension.GlobalSecretFind(name) + secretService := server.Config.Services.Manager.SecretService() + secret, err := secretService.GlobalSecretFind(name) if err != nil { handleDBError(c, err) return @@ -144,7 +144,7 @@ func PatchGlobalSecret(c *gin.Context) { return } - if err := secretExtension.GlobalSecretUpdate(secret); err != nil { + if err := secretService.GlobalSecretUpdate(secret); err != nil { c.String(http.StatusInternalServerError, "Error updating global secret %q. %s", in.Name, err) return } @@ -162,8 +162,8 @@ func PatchGlobalSecret(c *gin.Context) { // @Param secret path string true "the secret's name" func DeleteGlobalSecret(c *gin.Context) { name := c.Param("secret") - secretExtension := server.Config.ExtensionsManager.SecretExtension() - if err := secretExtension.GlobalSecretDelete(name); err != nil { + secretService := server.Config.Services.Manager.SecretService() + if err := secretService.GlobalSecretDelete(name); err != nil { handleDBError(c, err) return } diff --git a/server/api/org_secret.go b/server/api/org_secret.go index 05c102cf32..579820bc98 100644 --- a/server/api/org_secret.go +++ b/server/api/org_secret.go @@ -44,8 +44,8 @@ func GetOrgSecret(c *gin.Context) { return } - secretExtension := server.Config.ExtensionsManager.SecretExtension() - secret, err := secretExtension.OrgSecretFind(orgID, name) + secretService := server.Config.Services.Manager.SecretService() + secret, err := secretService.OrgSecretFind(orgID, name) if err != nil { handleDBError(c, err) return @@ -71,8 +71,8 @@ func GetOrgSecretList(c *gin.Context) { return } - secretExtension := server.Config.ExtensionsManager.SecretExtension() - list, err := secretExtension.OrgSecretList(orgID, session.Pagination(c)) + secretService := server.Config.Services.Manager.SecretService() + list, err := secretService.OrgSecretList(orgID, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting secret list for %q. %s", orgID, err) return @@ -119,8 +119,8 @@ func PostOrgSecret(c *gin.Context) { return } - secretExtension := server.Config.ExtensionsManager.SecretExtension() - if err := secretExtension.OrgSecretCreate(orgID, secret); err != nil { + secretService := server.Config.Services.Manager.SecretService() + if err := secretService.OrgSecretCreate(orgID, secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting org %q secret %q. %s", orgID, in.Name, err) return } @@ -153,8 +153,8 @@ func PatchOrgSecret(c *gin.Context) { return } - secretExtension := server.Config.ExtensionsManager.SecretExtension() - secret, err := secretExtension.OrgSecretFind(orgID, name) + secretService := server.Config.Services.Manager.SecretService() + secret, err := secretService.OrgSecretFind(orgID, name) if err != nil { handleDBError(c, err) return @@ -174,7 +174,7 @@ func PatchOrgSecret(c *gin.Context) { return } - if err := secretExtension.OrgSecretUpdate(orgID, secret); err != nil { + if err := secretService.OrgSecretUpdate(orgID, secret); err != nil { c.String(http.StatusInternalServerError, "Error updating org %q secret %q. %s", orgID, in.Name, err) return } @@ -199,8 +199,8 @@ func DeleteOrgSecret(c *gin.Context) { return } - secretExtension := server.Config.ExtensionsManager.SecretExtension() - if err := secretExtension.OrgSecretDelete(orgID, name); err != nil { + secretService := server.Config.Services.Manager.SecretService() + if err := secretService.OrgSecretDelete(orgID, name); err != nil { handleDBError(c, err) return } diff --git a/server/api/registry.go b/server/api/registry.go index 0d8ce4a1db..60e6648ff8 100644 --- a/server/api/registry.go +++ b/server/api/registry.go @@ -38,8 +38,8 @@ func GetRegistry(c *gin.Context) { repo := session.Repo(c) name := c.Param("registry") - registryExtension := server.Config.ExtensionsManager.RegistryExtensionFromRepo(repo) - registry, err := registryExtension.RegistryFind(repo, name) + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + registry, err := registryService.RegistryFind(repo, name) if err != nil { handleDBError(c, err) return @@ -76,8 +76,8 @@ func PostRegistry(c *gin.Context) { return } - registryExtension := server.Config.ExtensionsManager.RegistryExtensionFromRepo(repo) - if err := registryExtension.RegistryCreate(repo, registry); err != nil { + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + if err := registryService.RegistryCreate(repo, registry); err != nil { c.String(http.StatusInternalServerError, "Error inserting registry %q. %s", in.Address, err) return } @@ -108,8 +108,8 @@ func PatchRegistry(c *gin.Context) { return } - registryExtension := server.Config.ExtensionsManager.RegistryExtensionFromRepo(repo) - registry, err := registryExtension.RegistryFind(repo, name) + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + registry, err := registryService.RegistryFind(repo, name) if err != nil { handleDBError(c, err) return @@ -125,7 +125,7 @@ func PatchRegistry(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error updating registry. %s", err) return } - if err := registryExtension.RegistryUpdate(repo, registry); err != nil { + if err := registryService.RegistryUpdate(repo, registry); err != nil { c.String(http.StatusInternalServerError, "Error updating registry %q. %s", in.Address, err) return } @@ -145,8 +145,8 @@ func PatchRegistry(c *gin.Context) { // @Param perPage query int false "for response pagination, max items per page" default(50) func GetRegistryList(c *gin.Context) { repo := session.Repo(c) - registryExtension := server.Config.ExtensionsManager.RegistryExtensionFromRepo(repo) - list, err := registryExtension.RegistryList(repo, session.Pagination(c)) + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + list, err := registryService.RegistryList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting registry list. %s", err) return @@ -173,8 +173,8 @@ func DeleteRegistry(c *gin.Context) { repo := session.Repo(c) name := c.Param("registry") - registryExtension := server.Config.ExtensionsManager.RegistryExtensionFromRepo(repo) - err := registryExtension.RegistryDelete(repo, name) + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) + err := registryService.RegistryDelete(repo, name) if err != nil { handleDBError(c, err) return diff --git a/server/api/repo_secret.go b/server/api/repo_secret.go index e41f2f5b34..2a1b963483 100644 --- a/server/api/repo_secret.go +++ b/server/api/repo_secret.go @@ -39,8 +39,8 @@ func GetSecret(c *gin.Context) { repo := session.Repo(c) name := c.Param("secret") - secretExtension := server.Config.ExtensionsManager.SecretExtensionFromRepo(repo) - secret, err := secretExtension.SecretFind(repo, name) + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + secret, err := secretService.SecretFind(repo, name) if err != nil { handleDBError(c, err) return @@ -78,8 +78,8 @@ func PostSecret(c *gin.Context) { return } - secretExtension := server.Config.ExtensionsManager.SecretExtensionFromRepo(repo) - if err := secretExtension.SecretCreate(repo, secret); err != nil { + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + if err := secretService.SecretCreate(repo, secret); err != nil { c.String(http.StatusInternalServerError, "Error inserting secret %q. %s", in.Name, err) return } @@ -110,8 +110,8 @@ func PatchSecret(c *gin.Context) { return } - secretExtension := server.Config.ExtensionsManager.SecretExtensionFromRepo(repo) - secret, err := secretExtension.SecretFind(repo, name) + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + secret, err := secretService.SecretFind(repo, name) if err != nil { handleDBError(c, err) return @@ -130,7 +130,7 @@ func PatchSecret(c *gin.Context) { c.String(http.StatusUnprocessableEntity, "Error updating secret. %s", err) return } - if err := secretExtension.SecretUpdate(repo, secret); err != nil { + if err := secretService.SecretUpdate(repo, secret); err != nil { c.String(http.StatusInternalServerError, "Error updating secret %q. %s", in.Name, err) return } @@ -150,8 +150,8 @@ func PatchSecret(c *gin.Context) { // @Param perPage query int false "for response pagination, max items per page" default(50) func GetSecretList(c *gin.Context) { repo := session.Repo(c) - secretExtension := server.Config.ExtensionsManager.SecretExtensionFromRepo(repo) - list, err := secretExtension.SecretList(repo, session.Pagination(c)) + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + list, err := secretService.SecretList(repo, session.Pagination(c)) if err != nil { c.String(http.StatusInternalServerError, "Error getting secret list. %s", err) return @@ -178,8 +178,8 @@ func DeleteSecret(c *gin.Context) { repo := session.Repo(c) name := c.Param("secret") - secretExtension := server.Config.ExtensionsManager.SecretExtensionFromRepo(repo) - if err := secretExtension.SecretDelete(repo, name); err != nil { + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) + if err := secretService.SecretDelete(repo, name); err != nil { handleDBError(c, err) return } diff --git a/server/api/signature_public_key.go b/server/api/signature_public_key.go index 1a779c50b3..b9364166b4 100644 --- a/server/api/signature_public_key.go +++ b/server/api/signature_public_key.go @@ -34,7 +34,7 @@ import ( // @Tags System // @Param Authorization header string true "Insert your personal access token" default(Bearer ) func GetSignaturePublicKey(c *gin.Context) { - b, err := x509.MarshalPKIXPublicKey(server.Config.ExtensionsManager.SignaturePublicKey()) + b, err := x509.MarshalPKIXPublicKey(server.Config.Services.Manager.SignaturePublicKey()) if err != nil { log.Error().Err(err).Msg("can't marshal public key") c.AbortWithStatus(http.StatusInternalServerError) diff --git a/server/config.go b/server/config.go index dd3bf45297..84b56c29ad 100644 --- a/server/config.go +++ b/server/config.go @@ -21,13 +21,13 @@ import ( "time" "go.woodpecker-ci.org/woodpecker/v2/server/cache" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/permissions" "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/logging" "go.woodpecker-ci.org/woodpecker/v2/server/model" "go.woodpecker-ci.org/woodpecker/v2/server/pubsub" "go.woodpecker-ci.org/woodpecker/v2/server/queue" + "go.woodpecker-ci.org/woodpecker/v2/server/services" + "go.woodpecker-ci.org/woodpecker/v2/server/services/permissions" ) var Config = struct { @@ -37,9 +37,9 @@ var Config = struct { Logs logging.Log Forge forge.Forge Membership cache.MembershipService + Manager *services.Manager } - ExtensionsManager *extensions.Manager - Server struct { + Server struct { Key string Cert string OAuthHost string diff --git a/server/pipeline/create.go b/server/pipeline/create.go index f669509d2f..4e41b6100e 100644 --- a/server/pipeline/create.go +++ b/server/pipeline/create.go @@ -69,7 +69,7 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline } // fetch the pipeline file from the forge - configService := server.Config.ExtensionsManager.ConfigExtensionFromRepo(repo) + configService := server.Config.Services.Manager.ConfigServiceFromRepo(repo) forgeYamlConfigs, configFetchErr := configService.Fetch(ctx, _forge, repoUser, repo, pipeline) if errors.Is(configFetchErr, &forge_types.ErrConfigNotFound{}) { log.Debug().Str("repo", repo.FullName).Err(configFetchErr).Msgf("cannot find config '%s' in '%s' with user: '%s'", repo.Config, pipeline.Ref, repoUser.Login) diff --git a/server/pipeline/items.go b/server/pipeline/items.go index d1955fcd4d..35206cd117 100644 --- a/server/pipeline/items.go +++ b/server/pipeline/items.go @@ -42,13 +42,13 @@ func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *mod log.Error().Err(err).Str("repo", repo.FullName).Msgf("error getting last pipeline before pipeline number '%d'", currentPipeline.Number) } - secretService := server.Config.ExtensionsManager.SecretExtensionFromRepo(repo) + secretService := server.Config.Services.Manager.SecretServiceFromRepo(repo) secs, err := secretService.SecretListPipeline(repo, currentPipeline, &model.ListOptions{All: true}) if err != nil { log.Error().Err(err).Msgf("error getting secrets for %s#%d", repo.FullName, currentPipeline.Number) } - registryService := server.Config.ExtensionsManager.RegistryExtensionFromRepo(repo) + registryService := server.Config.Services.Manager.RegistryServiceFromRepo(repo) regs, err := registryService.RegistryList(repo, &model.ListOptions{All: true}) if err != nil { log.Error().Err(err).Msgf("error getting registry credentials for %s#%d", repo.FullName, currentPipeline.Number) @@ -58,7 +58,7 @@ func parsePipeline(store store.Store, currentPipeline *model.Pipeline, user *mod envs = map[string]string{} } - environmentService := server.Config.ExtensionsManager.EnvironmentExtension() + environmentService := server.Config.Services.Manager.EnvironmentService() if environmentService != nil { globals, _ := environmentService.EnvironList(repo) for _, global := range globals { diff --git a/server/pipeline/restart.go b/server/pipeline/restart.go index 2482c895de..b434e86b27 100644 --- a/server/pipeline/restart.go +++ b/server/pipeline/restart.go @@ -49,8 +49,8 @@ func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipelin pipelineFiles = append(pipelineFiles, &forge_types.FileMeta{Data: y.Data, Name: y.Name}) } - // If the config extension is active we should refetch the config in case something changed - configService := server.Config.ExtensionsManager.ConfigExtensionFromRepo(repo) + // If the config service is active we should refetch the config in case something changed + configService := server.Config.Services.Manager.ConfigServiceFromRepo(repo) pipelineFiles, err = configService.Fetch(ctx, forge, user, repo, lastPipeline) if err != nil { return nil, &ErrBadRequest{ diff --git a/server/extensions/config/combined.go b/server/services/config/combined.go similarity index 81% rename from server/extensions/config/combined.go rename to server/services/config/combined.go index a6da3e33e7..8514d73677 100644 --- a/server/extensions/config/combined.go +++ b/server/services/config/combined.go @@ -25,19 +25,19 @@ import ( ) type combined struct { - extensions []Extension + services []Service } -func NewCombined(extensions ...Extension) Extension { - return &combined{extensions: extensions} +func NewCombined(services ...Service) Service { + return &combined{services: services} } func (c *combined) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (files []*types.FileMeta, err error) { - for _, s := range c.extensions { - // TODO(anbraten): This is a hack to get the current configs into the http extension, will be removed when removing deprecatedCurrentConfigs + for _, s := range c.services { + // TODO(anbraten): This is a hack to get the current configs into the http service, will be removed when removing deprecatedCurrentConfigs _s, ok := s.(*http) if !ok { - log.Err(err).Msg("http extension is not of type http") + log.Err(err).Msg("http service is not of type http") } else { _s.deprecatedCurrentConfigs = files } diff --git a/server/extensions/config/combined_test.go b/server/services/config/combined_test.go similarity index 99% rename from server/extensions/config/combined_test.go rename to server/services/config/combined_test.go index e93d2b9c5a..b65b3a7592 100644 --- a/server/extensions/config/combined_test.go +++ b/server/services/config/combined_test.go @@ -31,10 +31,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/config" "go.woodpecker-ci.org/woodpecker/v2/server/forge/mocks" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/config" ) func TestFetchFromConfigExtension(t *testing.T) { diff --git a/server/extensions/config/forge.go b/server/services/config/forge.go similarity index 99% rename from server/extensions/config/forge.go rename to server/services/config/forge.go index a3294e15f6..2161012e19 100644 --- a/server/extensions/config/forge.go +++ b/server/services/config/forge.go @@ -37,7 +37,7 @@ type forgeFetcher struct { timeout time.Duration } -func NewForge(timeout time.Duration) Extension { +func NewForge(timeout time.Duration) Service { return &forgeFetcher{ timeout: timeout, } diff --git a/server/extensions/config/forge_test.go b/server/services/config/forge_test.go similarity index 99% rename from server/extensions/config/forge_test.go rename to server/services/config/forge_test.go index 05ec0fa147..9826c52616 100644 --- a/server/extensions/config/forge_test.go +++ b/server/services/config/forge_test.go @@ -24,10 +24,10 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/mock" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/config" "go.woodpecker-ci.org/woodpecker/v2/server/forge/mocks" forge_types "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/config" ) func TestFetch(t *testing.T) { diff --git a/server/extensions/config/http.go b/server/services/config/http.go similarity index 96% rename from server/extensions/config/http.go rename to server/services/config/http.go index 86e3c11494..f88c08c6c5 100644 --- a/server/extensions/config/http.go +++ b/server/services/config/http.go @@ -19,10 +19,10 @@ import ( "crypto" "fmt" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/utils" "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/utils" ) type http struct { @@ -50,7 +50,7 @@ type responseStructure struct { Configs []*configData `json:"configs"` } -func NewHTTP(endpoint string, privateKey crypto.PrivateKey, deprecatedCurrentConfigs []*types.FileMeta) Extension { +func NewHTTP(endpoint string, privateKey crypto.PrivateKey, deprecatedCurrentConfigs []*types.FileMeta) Service { return &http{endpoint, privateKey, deprecatedCurrentConfigs} } diff --git a/server/extensions/config/extension.go b/server/services/config/service.go similarity index 97% rename from server/extensions/config/extension.go rename to server/services/config/service.go index fb038f73e1..89bbefc108 100644 --- a/server/extensions/config/extension.go +++ b/server/services/config/service.go @@ -22,6 +22,6 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/model" ) -type Extension interface { +type Service interface { Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (configData []*types.FileMeta, err error) } diff --git a/server/extensions/encryption/aes.go b/server/services/encryption/aes.go similarity index 96% rename from server/extensions/encryption/aes.go rename to server/services/encryption/aes.go index 192b899579..3056503a4c 100644 --- a/server/extensions/encryption/aes.go +++ b/server/services/encryption/aes.go @@ -21,7 +21,7 @@ import ( "github.com/google/tink/go/subtle/random" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) diff --git a/server/extensions/encryption/aes_builder.go b/server/services/encryption/aes_builder.go similarity index 95% rename from server/extensions/encryption/aes_builder.go rename to server/services/encryption/aes_builder.go index 3bb9f58991..e02073b46a 100644 --- a/server/extensions/encryption/aes_builder.go +++ b/server/services/encryption/aes_builder.go @@ -20,7 +20,7 @@ import ( "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) diff --git a/server/extensions/encryption/aes_encryption.go b/server/services/encryption/aes_encryption.go similarity index 100% rename from server/extensions/encryption/aes_encryption.go rename to server/services/encryption/aes_encryption.go diff --git a/server/extensions/encryption/aes_state.go b/server/services/encryption/aes_state.go similarity index 100% rename from server/extensions/encryption/aes_state.go rename to server/services/encryption/aes_state.go diff --git a/server/extensions/encryption/aes_test.go b/server/services/encryption/aes_test.go similarity index 100% rename from server/extensions/encryption/aes_test.go rename to server/services/encryption/aes_test.go diff --git a/server/extensions/encryption/constants.go b/server/services/encryption/constants.go similarity index 96% rename from server/extensions/encryption/constants.go rename to server/services/encryption/constants.go index 6d3b5a2321..f52fb4bd38 100644 --- a/server/extensions/encryption/constants.go +++ b/server/services/encryption/constants.go @@ -41,7 +41,7 @@ var ( const ( // error wrapping templates errTemplateFailedInitializingUnencrypted = "failed initializing server in unencrypted mode: %w" - errTemplateFailedInitializing = "failed initializing encryption extension: %w" + errTemplateFailedInitializing = "failed initializing encryption service: %w" errTemplateFailedEnablingEncryption = "failed enabling encryption: %w" errTemplateFailedRotatingEncryption = "failed rotating encryption: %w" errTemplateFailedDisablingEncryption = "failed disabling encryption: %w" @@ -64,9 +64,9 @@ const ( logMessageEncryptionDisabled = "encryption disabled" logMessageEncryptionKeyRegistered = "registered new encryption key" logMessageClientsInitialized = "initialized encryption on registered clients" - logMessageClientsEnabled = "enabled encryption on registered extension" - logMessageClientsRotated = "updated encryption key on registered extension" - logMessageClientsDecrypted = "disabled encryption on registered extension" + logMessageClientsEnabled = "enabled encryption on registered service" + logMessageClientsRotated = "updated encryption key on registered service" + logMessageClientsDecrypted = "disabled encryption on registered service" ) // tink @@ -84,7 +84,7 @@ const ( errMessageTinkKeysetFileWatchFailed = "failed watching encryption keyset file changes" // log message templates - logTemplateTinkKeysetFileChanged = "changes detected in encryption keyset file: '%s'. Encryption extension will be reloaded" + logTemplateTinkKeysetFileChanged = "changes detected in encryption keyset file: '%s'. Encryption service will be reloaded" logTemplateTinkLoadingKeyset = "loading encryption keyset from file: %s" logTemplateTinkFailedClosingKeysetFile = "could not close keyset file: %s" ) diff --git a/server/extensions/encryption/encryption.go b/server/services/encryption/encryption.go similarity index 95% rename from server/extensions/encryption/encryption.go rename to server/services/encryption/encryption.go index ea45bd1080..54454b123e 100644 --- a/server/extensions/encryption/encryption.go +++ b/server/services/encryption/encryption.go @@ -19,7 +19,7 @@ import ( "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) diff --git a/server/extensions/encryption/encryption_builder.go b/server/services/encryption/encryption_builder.go similarity index 90% rename from server/extensions/encryption/encryption_builder.go rename to server/services/encryption/encryption_builder.go index 9adcc85c4e..7db635c57b 100644 --- a/server/extensions/encryption/encryption_builder.go +++ b/server/services/encryption/encryption_builder.go @@ -18,7 +18,7 @@ import ( "errors" "fmt" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" storeTypes "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) @@ -27,7 +27,7 @@ func (b builder) getExtension(keyType string) (types.EncryptionExtension, error) return nil, errors.New(errMessageNoKeysProvided) } - builder, err := b.extensionBuilder(keyType) + builder, err := b.serviceBuilder(keyType) if err != nil { return nil, err } @@ -61,7 +61,7 @@ func (b builder) detectKeyType() (string, error) { return keyTypeNone, nil } -func (b builder) extensionBuilder(keyType string) (types.EncryptionExtensionBuilder, error) { +func (b builder) serviceBuilder(keyType string) (types.EncryptionExtensionBuilder, error) { switch { case keyType == keyTypeTink: return newTink(b.ctx, b.store), nil diff --git a/server/extensions/encryption/no_encryption.go b/server/services/encryption/no_encryption.go similarity index 94% rename from server/extensions/encryption/no_encryption.go rename to server/services/encryption/no_encryption.go index b381a8aa5b..fa882652b7 100644 --- a/server/extensions/encryption/no_encryption.go +++ b/server/services/encryption/no_encryption.go @@ -14,7 +14,7 @@ package encryption -import "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" +import "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" type noEncryptionBuilder struct { clients []types.EncryptionClient diff --git a/server/extensions/encryption/tink.go b/server/services/encryption/tink.go similarity index 96% rename from server/extensions/encryption/tink.go rename to server/services/encryption/tink.go index a7781a43c8..267efbd534 100644 --- a/server/extensions/encryption/tink.go +++ b/server/services/encryption/tink.go @@ -21,7 +21,7 @@ import ( "github.com/fsnotify/fsnotify" "github.com/google/tink/go/tink" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) diff --git a/server/extensions/encryption/tink_builder.go b/server/services/encryption/tink_builder.go similarity index 96% rename from server/extensions/encryption/tink_builder.go rename to server/services/encryption/tink_builder.go index 432be21fa9..eddd073c8d 100644 --- a/server/extensions/encryption/tink_builder.go +++ b/server/services/encryption/tink_builder.go @@ -20,7 +20,7 @@ import ( "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) diff --git a/server/extensions/encryption/tink_keyset.go b/server/services/encryption/tink_keyset.go similarity index 100% rename from server/extensions/encryption/tink_keyset.go rename to server/services/encryption/tink_keyset.go diff --git a/server/extensions/encryption/tink_keyset_watcher.go b/server/services/encryption/tink_keyset_watcher.go similarity index 100% rename from server/extensions/encryption/tink_keyset_watcher.go rename to server/services/encryption/tink_keyset_watcher.go diff --git a/server/extensions/encryption/tink_state.go b/server/services/encryption/tink_state.go similarity index 100% rename from server/extensions/encryption/tink_state.go rename to server/services/encryption/tink_state.go diff --git a/server/extensions/encryption/types/encryption.go b/server/services/encryption/types/encryption.go similarity index 94% rename from server/extensions/encryption/types/encryption.go rename to server/services/encryption/types/encryption.go index e81fb88554..1d015dca2f 100644 --- a/server/extensions/encryption/types/encryption.go +++ b/server/services/encryption/types/encryption.go @@ -34,8 +34,8 @@ type EncryptionExtension interface { type EncryptionClient interface { // SetEncryptionExtension should be used only by EncryptionExtensionBuilder SetEncryptionExtension(encryption EncryptionExtension) error - // EnableEncryption should encrypt all extension data + // EnableEncryption should encrypt all service data EnableEncryption() error - // MigrateEncryption should decrypt all existing data and encrypt it with new encryption extension + // MigrateEncryption should decrypt all existing data and encrypt it with new encryption service MigrateEncryption(newEncryption EncryptionExtension) error } diff --git a/server/extensions/encryption/wrapper/store/constants.go b/server/services/encryption/wrapper/store/constants.go similarity index 100% rename from server/extensions/encryption/wrapper/store/constants.go rename to server/services/encryption/wrapper/store/constants.go diff --git a/server/extensions/encryption/wrapper/store/secret_store.go b/server/services/encryption/wrapper/store/secret_store.go similarity index 100% rename from server/extensions/encryption/wrapper/store/secret_store.go rename to server/services/encryption/wrapper/store/secret_store.go diff --git a/server/extensions/encryption/wrapper/store/secret_store_wrapper.go b/server/services/encryption/wrapper/store/secret_store_wrapper.go similarity index 94% rename from server/extensions/encryption/wrapper/store/secret_store_wrapper.go rename to server/services/encryption/wrapper/store/secret_store_wrapper.go index 66c3a1cbc5..1f5b99df4a 100644 --- a/server/extensions/encryption/wrapper/store/secret_store_wrapper.go +++ b/server/services/encryption/wrapper/store/secret_store_wrapper.go @@ -21,8 +21,8 @@ import ( "github.com/rs/zerolog/log" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/encryption/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/encryption/types" ) type EncryptedSecretStore struct { @@ -38,11 +38,11 @@ func NewSecretStore(secretStore model.SecretStore) *EncryptedSecretStore { return &wrapper } -func (wrapper *EncryptedSecretStore) SetEncryptionExtension(extension types.EncryptionExtension) error { +func (wrapper *EncryptedSecretStore) SetEncryptionExtension(service types.EncryptionExtension) error { if wrapper.encryption != nil { return errors.New(errMessageInitSeveralTimes) } - wrapper.encryption = extension + wrapper.encryption = service return nil } diff --git a/server/extensions/environment/extension.go b/server/services/environment/extension.go similarity index 88% rename from server/extensions/environment/extension.go rename to server/services/environment/extension.go index 2ba7f2f705..f8ad148215 100644 --- a/server/extensions/environment/extension.go +++ b/server/services/environment/extension.go @@ -16,7 +16,7 @@ package environment import "go.woodpecker-ci.org/woodpecker/v2/server/model" -// Extension defines a extension for managing environment variables. -type Extension interface { +// Service defines a service for managing environment variables. +type Service interface { EnvironList(*model.Repo) ([]*model.Environ, error) } diff --git a/server/extensions/environment/parse.go b/server/services/environment/parse.go similarity index 88% rename from server/extensions/environment/parse.go rename to server/services/environment/parse.go index a8fa63b4ed..c1a6b8c878 100644 --- a/server/extensions/environment/parse.go +++ b/server/services/environment/parse.go @@ -26,8 +26,8 @@ type builtin struct { globals []*model.Environ } -// Parse returns a model.EnvironExtension based on a string slice where key and value are separated by a ":" delimiter. -func Parse(params []string) Extension { +// Parse returns a Service based on a string slice where key and value are separated by a ":" delimiter. +func Parse(params []string) Service { var globals []*model.Environ for _, item := range params { diff --git a/server/extensions/environment/parse_test.go b/server/services/environment/parse_test.go similarity index 72% rename from server/extensions/environment/parse_test.go rename to server/services/environment/parse_test.go index 5a96b5ec7f..97e7575698 100644 --- a/server/extensions/environment/parse_test.go +++ b/server/services/environment/parse_test.go @@ -21,25 +21,25 @@ import ( ) func TestParse(t *testing.T) { - extension := Parse([]string{}) - env, err := extension.EnvironList(nil) + service := Parse([]string{}) + env, err := service.EnvironList(nil) assert.NoError(t, err) assert.Empty(t, env) - extension = Parse([]string{"ENV:value"}) - env, err = extension.EnvironList(nil) + service = Parse([]string{"ENV:value"}) + env, err = service.EnvironList(nil) assert.NoError(t, err) assert.Len(t, env, 1) assert.Equal(t, env[0].Name, "ENV") assert.Equal(t, env[0].Value, "value") - extension = Parse([]string{"ENV:value", "ENV2:value2"}) - env, err = extension.EnvironList(nil) + service = Parse([]string{"ENV:value", "ENV2:value2"}) + env, err = service.EnvironList(nil) assert.NoError(t, err) assert.Len(t, env, 2) - extension = Parse([]string{"ENV:value", "ENV2:value2", "ENV3_WITHOUT_VALUE"}) - env, err = extension.EnvironList(nil) + service = Parse([]string{"ENV:value", "ENV2:value2", "ENV3_WITHOUT_VALUE"}) + env, err = service.EnvironList(nil) assert.NoError(t, err) assert.Len(t, env, 2) } diff --git a/server/extensions/manager.go b/server/services/manager.go similarity index 57% rename from server/extensions/manager.go rename to server/services/manager.go index 6d2d2f55d7..bd6717f762 100644 --- a/server/extensions/manager.go +++ b/server/services/manager.go @@ -12,26 +12,26 @@ // See the License for the specific language governing permissions and // limitations under the License. -package extensions +package services import ( "crypto" "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/config" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/environment" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/registry" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/secret" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/config" + "go.woodpecker-ci.org/woodpecker/v2/server/services/environment" + "go.woodpecker-ci.org/woodpecker/v2/server/services/registry" + "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" "go.woodpecker-ci.org/woodpecker/v2/server/store" ) type Manager struct { - secret secret.Extension - registry registry.Extension - config config.Extension - environment environment.Extension + secret secret.Service + registry registry.Service + config config.Service + environment environment.Service signaturePrivateKey crypto.PrivateKey signaturePublicKey crypto.PublicKey } @@ -45,7 +45,7 @@ func NewManager(c *cli.Context, store store.Store) (*Manager, error) { return &Manager{ signaturePrivateKey: signaturePrivateKey, signaturePublicKey: signaturePublicKey, - secret: setupSecretExtension(store), + secret: setupSecretService(store), registry: setupRegistryExtension(store, c.String("docker-config")), config: setupConfigExtension(c, signaturePrivateKey), environment: environment.Parse(c.StringSlice("environment")), @@ -56,29 +56,27 @@ func (e *Manager) SignaturePublicKey() crypto.PublicKey { return e.signaturePublicKey } -func (e *Manager) SecretExtensionFromRepo(_ *model.Repo) secret.Extension { - // TODO: decied based on repo property which secret extension to use - return e.secret +func (e *Manager) SecretServiceFromRepo(_ *model.Repo) secret.Service { + return e.SecretService() } -func (e *Manager) SecretExtension() secret.Extension { +func (e *Manager) SecretService() secret.Service { return e.secret } -func (e *Manager) RegistryExtensionFromRepo(_ *model.Repo) registry.Extension { - // TODO: decied based on repo property which registry extension to use - return e.registry +func (e *Manager) RegistryServiceFromRepo(_ *model.Repo) registry.Service { + return e.RegistryService() } -func (e *Manager) RegistryExtension() registry.Extension { +func (e *Manager) RegistryService() registry.Service { return e.registry } -func (e *Manager) ConfigExtensionFromRepo(_ *model.Repo) config.Extension { - // TODO: decied based on repo property which config extension to use +func (e *Manager) ConfigServiceFromRepo(_ *model.Repo) config.Service { + // TODO: decied based on repo property which config service to use return e.config } -func (e *Manager) EnvironmentExtension() environment.Extension { +func (e *Manager) EnvironmentService() environment.Service { return e.environment } diff --git a/server/extensions/permissions/admins.go b/server/services/permissions/admins.go similarity index 100% rename from server/extensions/permissions/admins.go rename to server/services/permissions/admins.go diff --git a/server/extensions/permissions/admins_test.go b/server/services/permissions/admins_test.go similarity index 100% rename from server/extensions/permissions/admins_test.go rename to server/services/permissions/admins_test.go diff --git a/server/extensions/permissions/orgs.go b/server/services/permissions/orgs.go similarity index 100% rename from server/extensions/permissions/orgs.go rename to server/services/permissions/orgs.go diff --git a/server/extensions/permissions/orgs_test.go b/server/services/permissions/orgs_test.go similarity index 100% rename from server/extensions/permissions/orgs_test.go rename to server/services/permissions/orgs_test.go diff --git a/server/extensions/permissions/repo_owners.go b/server/services/permissions/repo_owners.go similarity index 100% rename from server/extensions/permissions/repo_owners.go rename to server/services/permissions/repo_owners.go diff --git a/server/extensions/permissions/repo_owners_test.go b/server/services/permissions/repo_owners_test.go similarity index 100% rename from server/extensions/permissions/repo_owners_test.go rename to server/services/permissions/repo_owners_test.go diff --git a/server/extensions/registry/combined.go b/server/services/registry/combined.go similarity index 93% rename from server/extensions/registry/combined.go rename to server/services/registry/combined.go index 20747d5cc3..4f52bca1a5 100644 --- a/server/extensions/registry/combined.go +++ b/server/services/registry/combined.go @@ -19,11 +19,11 @@ import ( ) type combined struct { - registries []ReadOnlyExtension - dbRegistry Extension + registries []ReadOnlyService + dbRegistry Service } -func NewCombined(dbRegistry Extension, registries ...ReadOnlyExtension) Extension { +func NewCombined(dbRegistry Service, registries ...ReadOnlyService) Service { registries = append(registries, dbRegistry) return &combined{ registries: registries, diff --git a/server/extensions/registry/db.go b/server/services/registry/db.go similarity index 93% rename from server/extensions/registry/db.go rename to server/services/registry/db.go index 29be0a5a35..5e7339557f 100644 --- a/server/extensions/registry/db.go +++ b/server/services/registry/db.go @@ -22,8 +22,8 @@ type db struct { store model.RegistryStore } -// New returns a new local registry extension. -func NewDB(store model.RegistryStore) Extension { +// New returns a new local registry service. +func NewDB(store model.RegistryStore) Service { return &db{store} } diff --git a/server/extensions/registry/filesystem.go b/server/services/registry/filesystem.go similarity index 98% rename from server/extensions/registry/filesystem.go rename to server/services/registry/filesystem.go index dc224e6f08..4d98818cc0 100644 --- a/server/extensions/registry/filesystem.go +++ b/server/services/registry/filesystem.go @@ -31,7 +31,7 @@ type filesystem struct { path string } -func NewFilesystem(path string) ReadOnlyExtension { +func NewFilesystem(path string) ReadOnlyService { return &filesystem{path} } diff --git a/server/extensions/registry/extension.go b/server/services/registry/service.go similarity index 85% rename from server/extensions/registry/extension.go rename to server/services/registry/service.go index cd39b28b52..252ce9d9a9 100644 --- a/server/extensions/registry/extension.go +++ b/server/services/registry/service.go @@ -16,8 +16,8 @@ package registry import "go.woodpecker-ci.org/woodpecker/v2/server/model" -// Extension defines a extension for managing registries. -type Extension interface { +// Service defines a service for managing registries. +type Service interface { RegistryFind(*model.Repo, string) (*model.Registry, error) RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error) RegistryCreate(*model.Repo, *model.Registry) error @@ -25,8 +25,8 @@ type Extension interface { RegistryDelete(*model.Repo, string) error } -// ReadOnlyExtension defines a extension for managing registries. -type ReadOnlyExtension interface { +// ReadOnlyService defines a service for managing registries. +type ReadOnlyService interface { RegistryFind(*model.Repo, string) (*model.Registry, error) RegistryList(*model.Repo, *model.ListOptions) ([]*model.Registry, error) } diff --git a/server/extensions/secret/db.go b/server/services/secret/db.go similarity index 97% rename from server/extensions/secret/db.go rename to server/services/secret/db.go index 999a0e6acb..b3d3202618 100644 --- a/server/extensions/secret/db.go +++ b/server/services/secret/db.go @@ -22,8 +22,8 @@ type db struct { store model.SecretStore } -// NewDB returns a new local secret extension. -func NewDB(store model.SecretStore) Extension { +// NewDB returns a new local secret service. +func NewDB(store model.SecretStore) Service { return &db{store: store} } diff --git a/server/extensions/secret/db_test.go b/server/services/secret/db_test.go similarity index 97% rename from server/extensions/secret/db_test.go rename to server/services/secret/db_test.go index 09ceb0aee6..6f6099f57f 100644 --- a/server/extensions/secret/db_test.go +++ b/server/services/secret/db_test.go @@ -20,8 +20,8 @@ import ( "github.com/franela/goblin" "github.com/stretchr/testify/mock" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/secret" "go.woodpecker-ci.org/woodpecker/v2/server/model" + "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" mocks_store "go.woodpecker-ci.org/woodpecker/v2/server/store/mocks" ) diff --git a/server/extensions/secret/extension.go b/server/services/secret/service.go similarity index 95% rename from server/extensions/secret/extension.go rename to server/services/secret/service.go index 6d56feb795..24568c195b 100644 --- a/server/extensions/secret/extension.go +++ b/server/services/secret/service.go @@ -16,8 +16,8 @@ package secret import "go.woodpecker-ci.org/woodpecker/v2/server/model" -// Extension defines a extension for managing secrets. -type Extension interface { +// Service defines a service for managing secrets. +type Service interface { SecretListPipeline(*model.Repo, *model.Pipeline, *model.ListOptions) ([]*model.Secret, error) // Repository secrets SecretFind(*model.Repo, string) (*model.Secret, error) diff --git a/server/extensions/setup.go b/server/services/setup.go similarity index 83% rename from server/extensions/setup.go rename to server/services/setup.go index 040824ecea..ce6c4c0364 100644 --- a/server/extensions/setup.go +++ b/server/services/setup.go @@ -12,7 +12,7 @@ // See the License for the specific language governing permissions and // limitations under the License. -package extensions +package services import ( "crypto" @@ -25,14 +25,14 @@ import ( "github.com/rs/zerolog/log" "github.com/urfave/cli/v2" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/config" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/registry" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/secret" + "go.woodpecker-ci.org/woodpecker/v2/server/services/config" + "go.woodpecker-ci.org/woodpecker/v2/server/services/registry" + "go.woodpecker-ci.org/woodpecker/v2/server/services/secret" "go.woodpecker-ci.org/woodpecker/v2/server/store" "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) -func setupRegistryExtension(store store.Store, dockerConfig string) registry.Extension { +func setupRegistryExtension(store store.Store, dockerConfig string) registry.Service { if dockerConfig != "" { return registry.NewCombined( registry.NewDB(store), @@ -43,20 +43,20 @@ func setupRegistryExtension(store store.Store, dockerConfig string) registry.Ext return registry.NewDB(store) } -func setupSecretExtension(store store.Store) secret.Extension { +func setupSecretService(store store.Store) secret.Service { // TODO(1544): fix encrypted store // // encryption // encryptedSecretStore := encryptedStore.NewSecretStore(v) // err := encryption.Encryption(c, v).WithClient(encryptedSecretStore).Build() // if err != nil { - // log.Fatal().Err(err).Msg("could not create encryption extension") + // log.Fatal().Err(err).Msg("could not create encryption service") // } - // server.Config.Extensions.Secrets = setupSecretExtension(c, encryptedSecretStore) + // server.Config.Extensions.Secrets = setupSecretService(c, encryptedSecretStore) return secret.NewDB(store) } -func setupConfigExtension(c *cli.Context, privateSignatureKey crypto.PrivateKey) config.Extension { +func setupConfigExtension(c *cli.Context, privateSignatureKey crypto.PrivateKey) config.Service { timeout := c.Duration("forge-timeout") configFetcher := config.NewForge(timeout) @@ -68,7 +68,7 @@ func setupConfigExtension(c *cli.Context, privateSignatureKey crypto.PrivateKey) return configFetcher } -// setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for extensions) +// setupSignatureKeys generate or load key pair to sign webhooks requests (i.e. used for service extensions) func setupSignatureKeys(_store store.Store) (crypto.PrivateKey, crypto.PublicKey, error) { privKeyID := "signature-private-key" diff --git a/server/extensions/utils/http.go b/server/services/utils/http.go similarity index 100% rename from server/extensions/utils/http.go rename to server/services/utils/http.go diff --git a/server/extensions/utils/http_test.go b/server/services/utils/http_test.go similarity index 96% rename from server/extensions/utils/http_test.go rename to server/services/utils/http_test.go index 32e2a83929..272be5203d 100644 --- a/server/extensions/utils/http_test.go +++ b/server/services/utils/http_test.go @@ -25,7 +25,7 @@ import ( "github.com/go-ap/httpsig" "github.com/stretchr/testify/assert" - "go.woodpecker-ci.org/woodpecker/v2/server/extensions/utils" + "go.woodpecker-ci.org/woodpecker/v2/server/services/utils" ) func TestSign(t *testing.T) { From 5976fd4f295e8264af16884494c12ab72bfef892 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Sun, 11 Feb 2024 15:01:30 +0100 Subject: [PATCH 25/28] extension => service --- server/services/config/combined_test.go | 2 +- server/services/encryption/aes.go | 8 +++---- server/services/encryption/aes_builder.go | 8 +++---- server/services/encryption/aes_encryption.go | 6 ++--- server/services/encryption/aes_state.go | 16 +++++++------- server/services/encryption/aes_test.go | 4 ++-- server/services/encryption/constants.go | 2 +- server/services/encryption/encryption.go | 2 +- .../services/encryption/encryption_builder.go | 6 ++--- server/services/encryption/no_encryption.go | 6 ++--- server/services/encryption/tink.go | 8 +++---- server/services/encryption/tink_builder.go | 8 +++---- server/services/encryption/tink_keyset.go | 4 ++-- .../encryption/tink_keyset_watcher.go | 4 ++-- server/services/encryption/tink_state.go | 22 +++++++++---------- .../services/encryption/types/encryption.go | 14 ++++++------ .../wrapper/store/secret_store_wrapper.go | 8 +++---- server/services/setup.go | 5 ++--- 18 files changed, 66 insertions(+), 67 deletions(-) diff --git a/server/services/config/combined_test.go b/server/services/config/combined_test.go index b65b3a7592..b4389e6573 100644 --- a/server/services/config/combined_test.go +++ b/server/services/config/combined_test.go @@ -37,7 +37,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/services/config" ) -func TestFetchFromConfigExtension(t *testing.T) { +func TestFetchFromConfigService(t *testing.T) { t.Parallel() type file struct { diff --git a/server/services/encryption/aes.go b/server/services/encryption/aes.go index 3056503a4c..56a784e27a 100644 --- a/server/services/encryption/aes.go +++ b/server/services/encryption/aes.go @@ -25,14 +25,14 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/store" ) -type aesEncryptionExtension struct { +type aesEncryptionService struct { cipher cipher.AEAD keyID string store store.Store clients []types.EncryptionClient } -func (svc *aesEncryptionExtension) Encrypt(plaintext, associatedData string) (string, error) { +func (svc *aesEncryptionService) Encrypt(plaintext, associatedData string) (string, error) { msg := []byte(plaintext) aad := []byte(associatedData) @@ -46,7 +46,7 @@ func (svc *aesEncryptionExtension) Encrypt(plaintext, associatedData string) (st return base64.StdEncoding.EncodeToString(result), nil } -func (svc *aesEncryptionExtension) Decrypt(ciphertext, associatedData string) (string, error) { +func (svc *aesEncryptionService) Decrypt(ciphertext, associatedData string) (string, error) { bytes, err := base64.StdEncoding.DecodeString(ciphertext) if err != nil { return "", fmt.Errorf(errTemplateBase64DecryptionFailed, err) @@ -62,6 +62,6 @@ func (svc *aesEncryptionExtension) Decrypt(ciphertext, associatedData string) (s return string(plaintext), nil } -func (svc *aesEncryptionExtension) Disable() error { +func (svc *aesEncryptionService) Disable() error { return svc.disable() } diff --git a/server/services/encryption/aes_builder.go b/server/services/encryption/aes_builder.go index e02073b46a..1da6df8a04 100644 --- a/server/services/encryption/aes_builder.go +++ b/server/services/encryption/aes_builder.go @@ -30,18 +30,18 @@ type aesConfiguration struct { clients []types.EncryptionClient } -func newAES(ctx *cli.Context, s store.Store) types.EncryptionExtensionBuilder { +func newAES(ctx *cli.Context, s store.Store) types.EncryptionServiceBuilder { key := ctx.String(rawKeyConfigFlag) return &aesConfiguration{key, s, nil} } -func (c aesConfiguration) WithClients(clients []types.EncryptionClient) types.EncryptionExtensionBuilder { +func (c aesConfiguration) WithClients(clients []types.EncryptionClient) types.EncryptionServiceBuilder { c.clients = clients return c } -func (c aesConfiguration) Build() (types.EncryptionExtension, error) { - svc := &aesEncryptionExtension{ +func (c aesConfiguration) Build() (types.EncryptionService, error) { + svc := &aesEncryptionService{ cipher: nil, store: c.store, clients: c.clients, diff --git a/server/services/encryption/aes_encryption.go b/server/services/encryption/aes_encryption.go index 1c2447ecc7..b7b5016ffb 100644 --- a/server/services/encryption/aes_encryption.go +++ b/server/services/encryption/aes_encryption.go @@ -26,7 +26,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) -func (svc *aesEncryptionExtension) loadCipher(password string) error { +func (svc *aesEncryptionService) loadCipher(password string) error { key, err := svc.hash([]byte(password)) if err != nil { return fmt.Errorf(errTemplateAesFailedGeneratingKey, err) @@ -50,7 +50,7 @@ func (svc *aesEncryptionExtension) loadCipher(password string) error { return nil } -func (svc *aesEncryptionExtension) validateKey() error { +func (svc *aesEncryptionService) validateKey() error { ciphertextSample, err := svc.store.ServerConfigGet(ciphertextSampleConfigKey) if errors.Is(err, types.RecordNotExist) { return errEncryptionNotEnabled @@ -67,7 +67,7 @@ func (svc *aesEncryptionExtension) validateKey() error { return nil } -func (svc *aesEncryptionExtension) hash(data []byte) ([]byte, error) { +func (svc *aesEncryptionService) hash(data []byte) ([]byte, error) { result := make([]byte, 32) sha := sha3.NewShake256() diff --git a/server/services/encryption/aes_state.go b/server/services/encryption/aes_state.go index 296d8832ac..470e80f5e7 100644 --- a/server/services/encryption/aes_state.go +++ b/server/services/encryption/aes_state.go @@ -20,9 +20,9 @@ import ( "github.com/rs/zerolog/log" ) -func (svc *aesEncryptionExtension) initClients() error { +func (svc *aesEncryptionService) initClients() error { for _, client := range svc.clients { - err := client.SetEncryptionExtension(svc) + err := client.SetEncryptionService(svc) if err != nil { return fmt.Errorf(errTemplateFailedInitializingClients, err) } @@ -31,7 +31,7 @@ func (svc *aesEncryptionExtension) initClients() error { return nil } -func (svc *aesEncryptionExtension) enable() error { +func (svc *aesEncryptionService) enable() error { err := svc.callbackOnEnable() if err != nil { return fmt.Errorf(errTemplateFailedEnablingEncryption, err) @@ -44,7 +44,7 @@ func (svc *aesEncryptionExtension) enable() error { return nil } -func (svc *aesEncryptionExtension) disable() error { +func (svc *aesEncryptionService) disable() error { err := svc.callbackOnDisable() if err != nil { return fmt.Errorf(errTemplateFailedDisablingEncryption, err) @@ -57,7 +57,7 @@ func (svc *aesEncryptionExtension) disable() error { return nil } -func (svc *aesEncryptionExtension) updateCiphertextSample() error { +func (svc *aesEncryptionService) updateCiphertextSample() error { ciphertext, err := svc.Encrypt(svc.keyID, keyIDAssociatedData) if err != nil { return fmt.Errorf(errTemplateFailedUpdatingServerConfig, err) @@ -70,7 +70,7 @@ func (svc *aesEncryptionExtension) updateCiphertextSample() error { return nil } -func (svc *aesEncryptionExtension) deleteCiphertextSample() error { +func (svc *aesEncryptionService) deleteCiphertextSample() error { err := svc.store.ServerConfigDelete(ciphertextSampleConfigKey) if err != nil { err = fmt.Errorf(errTemplateFailedUpdatingServerConfig, err) @@ -78,7 +78,7 @@ func (svc *aesEncryptionExtension) deleteCiphertextSample() error { return err } -func (svc *aesEncryptionExtension) callbackOnEnable() error { +func (svc *aesEncryptionService) callbackOnEnable() error { for _, client := range svc.clients { err := client.EnableEncryption() if err != nil { @@ -89,7 +89,7 @@ func (svc *aesEncryptionExtension) callbackOnEnable() error { return nil } -func (svc *aesEncryptionExtension) callbackOnDisable() error { +func (svc *aesEncryptionService) callbackOnDisable() error { for _, client := range svc.clients { err := client.MigrateEncryption(&noEncryption{}) if err != nil { diff --git a/server/services/encryption/aes_test.go b/server/services/encryption/aes_test.go index 18524c3fcd..1a2bc5abb1 100644 --- a/server/services/encryption/aes_test.go +++ b/server/services/encryption/aes_test.go @@ -22,7 +22,7 @@ import ( ) func TestShortMessageLongKey(t *testing.T) { - aes := &aesEncryptionExtension{} + aes := &aesEncryptionService{} err := aes.loadCipher(string(random.GetRandomBytes(32))) assert.NoError(t, err) @@ -36,7 +36,7 @@ func TestShortMessageLongKey(t *testing.T) { } func TestLongMessageShortKey(t *testing.T) { - aes := &aesEncryptionExtension{} + aes := &aesEncryptionService{} err := aes.loadCipher(string(random.GetRandomBytes(12))) assert.NoError(t, err) diff --git a/server/services/encryption/constants.go b/server/services/encryption/constants.go index f52fb4bd38..90feabd324 100644 --- a/server/services/encryption/constants.go +++ b/server/services/encryption/constants.go @@ -55,7 +55,7 @@ const ( // error messages errMessageTemplateUnsupportedKeyType = "unsupported encryption key type: %s" - errMessageCantUseBothExtensions = "cannot use raw encryption key and tink keyset at the same time" + errMessageCantUseBothServices = "cannot use raw encryption key and tink keyset at the same time" errMessageNoKeysProvided = "encryption enabled but no keys provided" errMessageFailedRotatingEncryption = "failed rotating encryption" diff --git a/server/services/encryption/encryption.go b/server/services/encryption/encryption.go index 54454b123e..4548aa8a3d 100644 --- a/server/services/encryption/encryption.go +++ b/server/services/encryption/encryption.go @@ -57,7 +57,7 @@ func (b builder) Build() error { return fmt.Errorf(errTemplateFailedInitializingUnencrypted, err) } } - svc, err := b.getExtension(keyType) + svc, err := b.getService(keyType) if err != nil { return fmt.Errorf(errTemplateFailedInitializing, err) } diff --git a/server/services/encryption/encryption_builder.go b/server/services/encryption/encryption_builder.go index 7db635c57b..700fbdb77e 100644 --- a/server/services/encryption/encryption_builder.go +++ b/server/services/encryption/encryption_builder.go @@ -22,7 +22,7 @@ import ( storeTypes "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) -func (b builder) getExtension(keyType string) (types.EncryptionExtension, error) { +func (b builder) getService(keyType string) (types.EncryptionService, error) { if keyType == keyTypeNone { return nil, errors.New(errMessageNoKeysProvided) } @@ -52,7 +52,7 @@ func (b builder) detectKeyType() (string, error) { tinkKeysetPresent := b.ctx.IsSet(tinkKeysetFilepathConfigFlag) switch { case rawKeyPresent && tinkKeysetPresent: - return "", errors.New(errMessageCantUseBothExtensions) + return "", errors.New(errMessageCantUseBothServices) case rawKeyPresent: return keyTypeRaw, nil case tinkKeysetPresent: @@ -61,7 +61,7 @@ func (b builder) detectKeyType() (string, error) { return keyTypeNone, nil } -func (b builder) serviceBuilder(keyType string) (types.EncryptionExtensionBuilder, error) { +func (b builder) serviceBuilder(keyType string) (types.EncryptionServiceBuilder, error) { switch { case keyType == keyTypeTink: return newTink(b.ctx, b.store), nil diff --git a/server/services/encryption/no_encryption.go b/server/services/encryption/no_encryption.go index fa882652b7..6bc9230d01 100644 --- a/server/services/encryption/no_encryption.go +++ b/server/services/encryption/no_encryption.go @@ -20,15 +20,15 @@ type noEncryptionBuilder struct { clients []types.EncryptionClient } -func (b noEncryptionBuilder) WithClients(clients []types.EncryptionClient) types.EncryptionExtensionBuilder { +func (b noEncryptionBuilder) WithClients(clients []types.EncryptionClient) types.EncryptionServiceBuilder { b.clients = clients return b } -func (b noEncryptionBuilder) Build() (types.EncryptionExtension, error) { +func (b noEncryptionBuilder) Build() (types.EncryptionService, error) { svc := &noEncryption{} for _, client := range b.clients { - err := client.SetEncryptionExtension(svc) + err := client.SetEncryptionService(svc) if err != nil { return nil, err } diff --git a/server/services/encryption/tink.go b/server/services/encryption/tink.go index 267efbd534..3c210ac7a7 100644 --- a/server/services/encryption/tink.go +++ b/server/services/encryption/tink.go @@ -25,7 +25,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/store" ) -type tinkEncryptionExtension struct { +type tinkEncryptionService struct { keysetFilePath string primaryKeyID string encryption tink.AEAD @@ -34,7 +34,7 @@ type tinkEncryptionExtension struct { clients []types.EncryptionClient } -func (svc *tinkEncryptionExtension) Encrypt(plaintext, associatedData string) (string, error) { +func (svc *tinkEncryptionService) Encrypt(plaintext, associatedData string) (string, error) { msg := []byte(plaintext) aad := []byte(associatedData) ciphertext, err := svc.encryption.Encrypt(msg, aad) @@ -44,7 +44,7 @@ func (svc *tinkEncryptionExtension) Encrypt(plaintext, associatedData string) (s return base64.StdEncoding.EncodeToString(ciphertext), nil } -func (svc *tinkEncryptionExtension) Decrypt(ciphertext, associatedData string) (string, error) { +func (svc *tinkEncryptionService) Decrypt(ciphertext, associatedData string) (string, error) { ct, err := base64.StdEncoding.DecodeString(ciphertext) if err != nil { return "", fmt.Errorf(errTemplateBase64DecryptionFailed, err) @@ -57,6 +57,6 @@ func (svc *tinkEncryptionExtension) Decrypt(ciphertext, associatedData string) ( return string(plaintext), nil } -func (svc *tinkEncryptionExtension) Disable() error { +func (svc *tinkEncryptionService) Disable() error { return svc.disable() } diff --git a/server/services/encryption/tink_builder.go b/server/services/encryption/tink_builder.go index eddd073c8d..9dd1ddd799 100644 --- a/server/services/encryption/tink_builder.go +++ b/server/services/encryption/tink_builder.go @@ -30,18 +30,18 @@ type tinkConfiguration struct { clients []types.EncryptionClient } -func newTink(ctx *cli.Context, s store.Store) types.EncryptionExtensionBuilder { +func newTink(ctx *cli.Context, s store.Store) types.EncryptionServiceBuilder { filepath := ctx.String(tinkKeysetFilepathConfigFlag) return &tinkConfiguration{filepath, s, nil} } -func (c tinkConfiguration) WithClients(clients []types.EncryptionClient) types.EncryptionExtensionBuilder { +func (c tinkConfiguration) WithClients(clients []types.EncryptionClient) types.EncryptionServiceBuilder { c.clients = clients return c } -func (c tinkConfiguration) Build() (types.EncryptionExtension, error) { - svc := &tinkEncryptionExtension{ +func (c tinkConfiguration) Build() (types.EncryptionService, error) { + svc := &tinkEncryptionService{ keysetFilePath: c.keysetFilePath, primaryKeyID: "", encryption: nil, diff --git a/server/services/encryption/tink_keyset.go b/server/services/encryption/tink_keyset.go index 90762b1f54..482b178140 100644 --- a/server/services/encryption/tink_keyset.go +++ b/server/services/encryption/tink_keyset.go @@ -28,7 +28,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) -func (svc *tinkEncryptionExtension) loadKeyset() error { +func (svc *tinkEncryptionService) loadKeyset() error { log.Warn().Msgf(logTemplateTinkLoadingKeyset, svc.keysetFilePath) file, err := os.Open(svc.keysetFilePath) if err != nil { @@ -56,7 +56,7 @@ func (svc *tinkEncryptionExtension) loadKeyset() error { return nil } -func (svc *tinkEncryptionExtension) validateKeyset() error { +func (svc *tinkEncryptionService) validateKeyset() error { ciphertextSample, err := svc.store.ServerConfigGet(ciphertextSampleConfigKey) if errors.Is(err, types.RecordNotExist) { return errEncryptionNotEnabled diff --git a/server/services/encryption/tink_keyset_watcher.go b/server/services/encryption/tink_keyset_watcher.go index 0ef9a94d7f..149278d559 100644 --- a/server/services/encryption/tink_keyset_watcher.go +++ b/server/services/encryption/tink_keyset_watcher.go @@ -22,7 +22,7 @@ import ( ) // Watch keyset file events to detect key rotations and hot reload keys -func (svc *tinkEncryptionExtension) initFileWatcher() error { +func (svc *tinkEncryptionService) initFileWatcher() error { watcher, err := fsnotify.NewWatcher() if err != nil { return fmt.Errorf(errTemplateTinkFailedSubscribeKeysetFileChanges, err) @@ -37,7 +37,7 @@ func (svc *tinkEncryptionExtension) initFileWatcher() error { return nil } -func (svc *tinkEncryptionExtension) handleFileEvents() { +func (svc *tinkEncryptionService) handleFileEvents() { for { select { case event, ok := <-svc.keysetFileWatcher.Events: diff --git a/server/services/encryption/tink_state.go b/server/services/encryption/tink_state.go index 5b928a452a..f15bf180a0 100644 --- a/server/services/encryption/tink_state.go +++ b/server/services/encryption/tink_state.go @@ -21,7 +21,7 @@ import ( "github.com/rs/zerolog/log" ) -func (svc *tinkEncryptionExtension) enable() error { +func (svc *tinkEncryptionService) enable() error { if err := svc.callbackOnEnable(); err != nil { return fmt.Errorf(errTemplateFailedEnablingEncryption, err) } @@ -34,7 +34,7 @@ func (svc *tinkEncryptionExtension) enable() error { return nil } -func (svc *tinkEncryptionExtension) disable() error { +func (svc *tinkEncryptionService) disable() error { if err := svc.callbackOnDisable(); err != nil { return fmt.Errorf(errTemplateFailedDisablingEncryption, err) } @@ -47,8 +47,8 @@ func (svc *tinkEncryptionExtension) disable() error { return nil } -func (svc *tinkEncryptionExtension) rotate() error { - newSvc := &tinkEncryptionExtension{ +func (svc *tinkEncryptionService) rotate() error { + newSvc := &tinkEncryptionService{ keysetFilePath: svc.keysetFilePath, primaryKeyID: "", encryption: nil, @@ -79,7 +79,7 @@ func (svc *tinkEncryptionExtension) rotate() error { return nil } -func (svc *tinkEncryptionExtension) updateCiphertextSample() error { +func (svc *tinkEncryptionService) updateCiphertextSample() error { ciphertext, err := svc.Encrypt(svc.primaryKeyID, keyIDAssociatedData) if err != nil { return fmt.Errorf(errTemplateFailedUpdatingServerConfig, err) @@ -93,16 +93,16 @@ func (svc *tinkEncryptionExtension) updateCiphertextSample() error { return nil } -func (svc *tinkEncryptionExtension) deleteCiphertextSample() error { +func (svc *tinkEncryptionService) deleteCiphertextSample() error { if err := svc.store.ServerConfigDelete(ciphertextSampleConfigKey); err != nil { return fmt.Errorf(errTemplateFailedUpdatingServerConfig, err) } return nil } -func (svc *tinkEncryptionExtension) initClients() error { +func (svc *tinkEncryptionService) initClients() error { for _, client := range svc.clients { - if err := client.SetEncryptionExtension(svc); err != nil { + if err := client.SetEncryptionService(svc); err != nil { return err } } @@ -110,7 +110,7 @@ func (svc *tinkEncryptionExtension) initClients() error { return nil } -func (svc *tinkEncryptionExtension) callbackOnEnable() error { +func (svc *tinkEncryptionService) callbackOnEnable() error { for _, client := range svc.clients { if err := client.EnableEncryption(); err != nil { return err @@ -120,7 +120,7 @@ func (svc *tinkEncryptionExtension) callbackOnEnable() error { return nil } -func (svc *tinkEncryptionExtension) callbackOnRotation() error { +func (svc *tinkEncryptionService) callbackOnRotation() error { for _, client := range svc.clients { if err := client.MigrateEncryption(svc); err != nil { return err @@ -130,7 +130,7 @@ func (svc *tinkEncryptionExtension) callbackOnRotation() error { return nil } -func (svc *tinkEncryptionExtension) callbackOnDisable() error { +func (svc *tinkEncryptionService) callbackOnDisable() error { for _, client := range svc.clients { if err := client.MigrateEncryption(&noEncryption{}); err != nil { return err diff --git a/server/services/encryption/types/encryption.go b/server/services/encryption/types/encryption.go index 1d015dca2f..2d6b136fb8 100644 --- a/server/services/encryption/types/encryption.go +++ b/server/services/encryption/types/encryption.go @@ -20,22 +20,22 @@ type EncryptionBuilder interface { Build() error } -type EncryptionExtensionBuilder interface { - WithClients(clients []EncryptionClient) EncryptionExtensionBuilder - Build() (EncryptionExtension, error) +type EncryptionServiceBuilder interface { + WithClients(clients []EncryptionClient) EncryptionServiceBuilder + Build() (EncryptionService, error) } -type EncryptionExtension interface { +type EncryptionService interface { Encrypt(plaintext, associatedData string) (string, error) Decrypt(ciphertext, associatedData string) (string, error) Disable() error } type EncryptionClient interface { - // SetEncryptionExtension should be used only by EncryptionExtensionBuilder - SetEncryptionExtension(encryption EncryptionExtension) error + // SetEncryptionService should be used only by EncryptionServiceBuilder + SetEncryptionService(encryption EncryptionService) error // EnableEncryption should encrypt all service data EnableEncryption() error // MigrateEncryption should decrypt all existing data and encrypt it with new encryption service - MigrateEncryption(newEncryption EncryptionExtension) error + MigrateEncryption(newEncryption EncryptionService) error } diff --git a/server/services/encryption/wrapper/store/secret_store_wrapper.go b/server/services/encryption/wrapper/store/secret_store_wrapper.go index 1f5b99df4a..c776bcfa16 100644 --- a/server/services/encryption/wrapper/store/secret_store_wrapper.go +++ b/server/services/encryption/wrapper/store/secret_store_wrapper.go @@ -27,7 +27,7 @@ import ( type EncryptedSecretStore struct { store model.SecretStore - encryption types.EncryptionExtension + encryption types.EncryptionService } // ensure wrapper match interface @@ -38,7 +38,7 @@ func NewSecretStore(secretStore model.SecretStore) *EncryptedSecretStore { return &wrapper } -func (wrapper *EncryptedSecretStore) SetEncryptionExtension(service types.EncryptionExtension) error { +func (wrapper *EncryptedSecretStore) SetEncryptionService(service types.EncryptionService) error { if wrapper.encryption != nil { return errors.New(errMessageInitSeveralTimes) } @@ -64,7 +64,7 @@ func (wrapper *EncryptedSecretStore) EnableEncryption() error { return nil } -func (wrapper *EncryptedSecretStore) MigrateEncryption(newEncryptionExtension types.EncryptionExtension) error { +func (wrapper *EncryptedSecretStore) MigrateEncryption(newEncryptionService types.EncryptionService) error { log.Warn().Msg(logMessageMigratingSecretsEncryption) secrets, err := wrapper.store.SecretListAll() if err != nil { @@ -73,7 +73,7 @@ func (wrapper *EncryptedSecretStore) MigrateEncryption(newEncryptionExtension ty if err := wrapper.decryptList(secrets); err != nil { return err } - wrapper.encryption = newEncryptionExtension + wrapper.encryption = newEncryptionService for _, secret := range secrets { if err := wrapper.encrypt(secret); err != nil { return err diff --git a/server/services/setup.go b/server/services/setup.go index ce6c4c0364..7603809a22 100644 --- a/server/services/setup.go +++ b/server/services/setup.go @@ -32,7 +32,7 @@ import ( "go.woodpecker-ci.org/woodpecker/v2/server/store/types" ) -func setupRegistryExtension(store store.Store, dockerConfig string) registry.Service { +func setupRegistryService(store store.Store, dockerConfig string) registry.Service { if dockerConfig != "" { return registry.NewCombined( registry.NewDB(store), @@ -51,12 +51,11 @@ func setupSecretService(store store.Store) secret.Service { // if err != nil { // log.Fatal().Err(err).Msg("could not create encryption service") // } - // server.Config.Extensions.Secrets = setupSecretService(c, encryptedSecretStore) return secret.NewDB(store) } -func setupConfigExtension(c *cli.Context, privateSignatureKey crypto.PrivateKey) config.Service { +func setupConfigService(c *cli.Context, privateSignatureKey crypto.PrivateKey) config.Service { timeout := c.Duration("forge-timeout") configFetcher := config.NewForge(timeout) From 6e0f1aae413cf0edd5c3c923a0000681badccf1b Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Sun, 11 Feb 2024 15:06:48 +0100 Subject: [PATCH 26/28] update --- server/services/manager.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/services/manager.go b/server/services/manager.go index bd6717f762..d5669df2b6 100644 --- a/server/services/manager.go +++ b/server/services/manager.go @@ -46,8 +46,8 @@ func NewManager(c *cli.Context, store store.Store) (*Manager, error) { signaturePrivateKey: signaturePrivateKey, signaturePublicKey: signaturePublicKey, secret: setupSecretService(store), - registry: setupRegistryExtension(store, c.String("docker-config")), - config: setupConfigExtension(c, signaturePrivateKey), + registry: setupRegistryService(store, c.String("docker-config")), + config: setupConfigService(c, signaturePrivateKey), environment: environment.Parse(c.StringSlice("environment")), }, nil } From 828580134ac2c6b7241c02f4fa9d5363da909f23 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Sun, 11 Feb 2024 15:18:27 +0100 Subject: [PATCH 27/28] cleanup --- server/api/pipeline.go | 8 +------- server/pipeline/restart.go | 2 +- 2 files changed, 2 insertions(+), 8 deletions(-) diff --git a/server/api/pipeline.go b/server/api/pipeline.go index 5fbfaa09b3..c151e1f264 100644 --- a/server/api/pipeline.go +++ b/server/api/pipeline.go @@ -434,13 +434,7 @@ func PostPipeline(c *gin.Context) { } } - netrc, err := server.Config.Services.Forge.Netrc(user, repo) - if err != nil { - handlePipelineErr(c, err) - return - } - - newpipeline, err := pipeline.Restart(c, _store, pl, user, repo, envs, netrc) + newpipeline, err := pipeline.Restart(c, _store, pl, user, repo, envs) if err != nil { handlePipelineErr(c, err) } else { diff --git a/server/pipeline/restart.go b/server/pipeline/restart.go index b434e86b27..87027ae9c9 100644 --- a/server/pipeline/restart.go +++ b/server/pipeline/restart.go @@ -28,7 +28,7 @@ import ( ) // Restart a pipeline by creating a new one out of the old and start it -func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipeline, user *model.User, repo *model.Repo, envs map[string]string, netrc *model.Netrc) (*model.Pipeline, error) { +func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipeline, user *model.User, repo *model.Repo, envs map[string]string) (*model.Pipeline, error) { forge := server.Config.Services.Forge switch lastPipeline.Status { case model.StatusDeclined, From f1133e085af6f43ca17784b8ded245687a33b080 Mon Sep 17 00:00:00 2001 From: Anbraten <6918444+anbraten@users.noreply.github.com> Date: Sun, 11 Feb 2024 16:09:43 +0100 Subject: [PATCH 28/28] fix config fetching --- server/pipeline/create.go | 2 +- server/pipeline/restart.go | 5 ++-- server/services/config/combined.go | 15 +++-------- server/services/config/combined_test.go | 4 ++- server/services/config/forge.go | 7 ++++- server/services/config/forge_test.go | 2 ++ server/services/config/http.go | 35 +++++++++++-------------- server/services/config/service.go | 2 +- server/services/setup.go | 2 +- 9 files changed, 35 insertions(+), 39 deletions(-) diff --git a/server/pipeline/create.go b/server/pipeline/create.go index 4e41b6100e..3d25744474 100644 --- a/server/pipeline/create.go +++ b/server/pipeline/create.go @@ -70,7 +70,7 @@ func Create(ctx context.Context, _store store.Store, repo *model.Repo, pipeline // fetch the pipeline file from the forge configService := server.Config.Services.Manager.ConfigServiceFromRepo(repo) - forgeYamlConfigs, configFetchErr := configService.Fetch(ctx, _forge, repoUser, repo, pipeline) + forgeYamlConfigs, configFetchErr := configService.Fetch(ctx, _forge, repoUser, repo, pipeline, nil, false) if errors.Is(configFetchErr, &forge_types.ErrConfigNotFound{}) { log.Debug().Str("repo", repo.FullName).Err(configFetchErr).Msgf("cannot find config '%s' in '%s' with user: '%s'", repo.Config, pipeline.Ref, repoUser.Login) if err := _store.DeletePipeline(pipeline); err != nil { diff --git a/server/pipeline/restart.go b/server/pipeline/restart.go index 87027ae9c9..4cab9ae928 100644 --- a/server/pipeline/restart.go +++ b/server/pipeline/restart.go @@ -36,8 +36,6 @@ func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipelin return nil, &ErrBadRequest{Msg: fmt.Sprintf("cannot restart a pipeline with status %s", lastPipeline.Status)} } - var pipelineFiles []*forge_types.FileMeta - // fetch the old pipeline config from the database configs, err := store.ConfigsForPipeline(lastPipeline.ID) if err != nil { @@ -45,13 +43,14 @@ func Restart(ctx context.Context, store store.Store, lastPipeline *model.Pipelin return nil, &ErrNotFound{Msg: fmt.Sprintf("failure to get pipeline config for %s. %s", repo.FullName, err)} } + var pipelineFiles []*forge_types.FileMeta for _, y := range configs { pipelineFiles = append(pipelineFiles, &forge_types.FileMeta{Data: y.Data, Name: y.Name}) } // If the config service is active we should refetch the config in case something changed configService := server.Config.Services.Manager.ConfigServiceFromRepo(repo) - pipelineFiles, err = configService.Fetch(ctx, forge, user, repo, lastPipeline) + pipelineFiles, err = configService.Fetch(ctx, forge, user, repo, lastPipeline, pipelineFiles, true) if err != nil { return nil, &ErrBadRequest{ Msg: fmt.Sprintf("On fetching external pipeline config: %s", err), diff --git a/server/services/config/combined.go b/server/services/config/combined.go index 8514d73677..5daebb2812 100644 --- a/server/services/config/combined.go +++ b/server/services/config/combined.go @@ -17,8 +17,6 @@ package config import ( "context" - "github.com/rs/zerolog/log" - "go.woodpecker-ci.org/woodpecker/v2/server/forge" "go.woodpecker-ci.org/woodpecker/v2/server/forge/types" "go.woodpecker-ci.org/woodpecker/v2/server/model" @@ -32,17 +30,10 @@ func NewCombined(services ...Service) Service { return &combined{services: services} } -func (c *combined) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (files []*types.FileMeta, err error) { +func (c *combined) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline, oldConfigData []*types.FileMeta, restart bool) (files []*types.FileMeta, err error) { + files = oldConfigData for _, s := range c.services { - // TODO(anbraten): This is a hack to get the current configs into the http service, will be removed when removing deprecatedCurrentConfigs - _s, ok := s.(*http) - if !ok { - log.Err(err).Msg("http service is not of type http") - } else { - _s.deprecatedCurrentConfigs = files - } - - files, err = s.Fetch(ctx, forge, user, repo, pipeline) + files, err = s.Fetch(ctx, forge, user, repo, pipeline, files, restart) } return files, err diff --git a/server/services/config/combined_test.go b/server/services/config/combined_test.go index b4389e6573..4bccd6d507 100644 --- a/server/services/config/combined_test.go +++ b/server/services/config/combined_test.go @@ -192,7 +192,7 @@ func TestFetchFromConfigService(t *testing.T) { ts := httptest.NewServer(http.HandlerFunc(fixtureHandler)) defer ts.Close() - httpFetcher := config.NewHTTP(ts.URL, privEd25519Key, nil) + httpFetcher := config.NewHTTP(ts.URL, privEd25519Key) for _, tt := range testTable { t.Run(tt.name, func(t *testing.T) { @@ -229,6 +229,8 @@ func TestFetchFromConfigService(t *testing.T) { &model.User{Token: "xxx"}, repo, &model.Pipeline{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"}, + []*forge_types.FileMeta{}, + false, ) if tt.expectedError && err == nil { t.Fatal("expected an error") diff --git a/server/services/config/forge.go b/server/services/config/forge.go index 2161012e19..fc794bad8a 100644 --- a/server/services/config/forge.go +++ b/server/services/config/forge.go @@ -43,7 +43,12 @@ func NewForge(timeout time.Duration) Service { } } -func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (files []*types.FileMeta, err error) { +func (f *forgeFetcher) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline, oldConfigData []*types.FileMeta, restart bool) (files []*types.FileMeta, err error) { + // skip fetching if we are restarting and have the old config + if restart && len(oldConfigData) > 0 { + return + } + ffc := &forgeFetcherContext{ forge: forge, user: user, diff --git a/server/services/config/forge_test.go b/server/services/config/forge_test.go index 9826c52616..c442e78fd2 100644 --- a/server/services/config/forge_test.go +++ b/server/services/config/forge_test.go @@ -314,6 +314,8 @@ func TestFetch(t *testing.T) { &model.User{Token: "xxx"}, repo, &model.Pipeline{Commit: "89ab7b2d6bfb347144ac7c557e638ab402848fee"}, + nil, + false, ) if tt.expectedError && err == nil { t.Fatal("expected an error") diff --git a/server/services/config/http.go b/server/services/config/http.go index f88c08c6c5..dc51a47349 100644 --- a/server/services/config/http.go +++ b/server/services/config/http.go @@ -26,9 +26,8 @@ import ( ) type http struct { - endpoint string - privateKey crypto.PrivateKey - deprecatedCurrentConfigs []*types.FileMeta + endpoint string + privateKey crypto.PrivateKey } // configData same as forge.FileMeta but with json tags and string data @@ -38,25 +37,23 @@ type configData struct { } type requestStructure struct { - Repo *model.Repo `json:"repo"` - Pipeline *model.Pipeline `json:"pipeline"` - Netrc *model.Netrc `json:"netrc"` - - // @deprecated use netrc data to fetch the config by yourself instead - DeprecatedConfiguration []*configData `json:"configs"` // TODO: remove in next major release + Repo *model.Repo `json:"repo"` + Pipeline *model.Pipeline `json:"pipeline"` + Netrc *model.Netrc `json:"netrc"` + Configuration []*configData `json:"configs"` // TODO: deprecate in favor of netrc and remove in next major release } type responseStructure struct { Configs []*configData `json:"configs"` } -func NewHTTP(endpoint string, privateKey crypto.PrivateKey, deprecatedCurrentConfigs []*types.FileMeta) Service { - return &http{endpoint, privateKey, deprecatedCurrentConfigs} +func NewHTTP(endpoint string, privateKey crypto.PrivateKey) Service { + return &http{endpoint, privateKey} } -func (h *http) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) ([]*types.FileMeta, error) { - currentConfigs := make([]*configData, len(h.deprecatedCurrentConfigs)) - for i, pipe := range h.deprecatedCurrentConfigs { +func (h *http) Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline, oldConfigData []*types.FileMeta, _ bool) ([]*types.FileMeta, error) { + currentConfigs := make([]*configData, len(oldConfigData)) + for i, pipe := range oldConfigData { currentConfigs[i] = &configData{Name: pipe.Name, Data: string(pipe.Data)} } @@ -67,10 +64,10 @@ func (h *http) Fetch(ctx context.Context, forge forge.Forge, user *model.User, r response := new(responseStructure) body := requestStructure{ - Repo: repo, - Pipeline: pipeline, - DeprecatedConfiguration: currentConfigs, - Netrc: netrc, + Repo: repo, + Pipeline: pipeline, + Configuration: currentConfigs, + Netrc: netrc, } status, err := utils.Send(ctx, "POST", h.endpoint, h.privateKey, body, response) @@ -79,7 +76,7 @@ func (h *http) Fetch(ctx context.Context, forge forge.Forge, user *model.User, r } if status != 200 { - return []*types.FileMeta{}, nil + return oldConfigData, nil } fileMetas := make([]*types.FileMeta, len(response.Configs)) diff --git a/server/services/config/service.go b/server/services/config/service.go index 89bbefc108..12ba46f21b 100644 --- a/server/services/config/service.go +++ b/server/services/config/service.go @@ -23,5 +23,5 @@ import ( ) type Service interface { - Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline) (configData []*types.FileMeta, err error) + Fetch(ctx context.Context, forge forge.Forge, user *model.User, repo *model.Repo, pipeline *model.Pipeline, oldConfigData []*types.FileMeta, restart bool) (configData []*types.FileMeta, err error) } diff --git a/server/services/setup.go b/server/services/setup.go index 7603809a22..b6ca4506fa 100644 --- a/server/services/setup.go +++ b/server/services/setup.go @@ -60,7 +60,7 @@ func setupConfigService(c *cli.Context, privateSignatureKey crypto.PrivateKey) c configFetcher := config.NewForge(timeout) if endpoint := c.String("config-extension-endpoint"); endpoint != "" { - httpFetcher := config.NewHTTP(endpoint, privateSignatureKey, nil) + httpFetcher := config.NewHTTP(endpoint, privateSignatureKey) return config.NewCombined(configFetcher, httpFetcher) }