From 8b98979697aa37dd3678f22056f9737f7de5c2bb Mon Sep 17 00:00:00 2001 From: Adam Bozanich Date: Sat, 2 Mar 2024 00:43:43 +0000 Subject: [PATCH] feat(private-containers): private registries refs akash-network/support#54 Signed-off-by: Adam Bozanich --- cluster/kube/apply.go | 23 ++++ cluster/kube/builder/service_credentials.go | 111 ++++++++++++++++++++ cluster/kube/builder/workload.go | 18 +++- cluster/kube/client.go | 13 +++ go.mod | 2 +- go.sum | 4 +- 6 files changed, 166 insertions(+), 5 deletions(-) create mode 100644 cluster/kube/builder/service_credentials.go diff --git a/cluster/kube/apply.go b/cluster/kube/apply.go index 57ab0d2c..e957af29 100644 --- a/cluster/kube/apply.go +++ b/cluster/kube/apply.go @@ -86,6 +86,29 @@ func applyNetPolicies(ctx context.Context, kc kubernetes.Interface, b builder.Ne // return err // } +func applyServiceCredentials(ctx context.Context, kc kubernetes.Interface, b builder.ServiceCredentials) error { + obj, err := kc.CoreV1().Secrets(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{}) + metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "secrets-get", err, errors.IsNotFound) + + switch { + case err == nil: + obj, err = b.Update(obj) + if err == nil { + _, err = kc.CoreV1().Secrets(b.NS()).Update(ctx, obj, metav1.UpdateOptions{}) + metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "secrets-get", err) + + } + case errors.IsNotFound(err): + obj, err = b.Create() + if err == nil { + _, err = kc.CoreV1().Secrets(b.NS()).Create(ctx, obj, metav1.CreateOptions{}) + metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "secrets-create", err) + } + } + return err + +} + func applyDeployment(ctx context.Context, kc kubernetes.Interface, b builder.Deployment) error { obj, err := kc.AppsV1().Deployments(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{}) metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "deployments-get", err, errors.IsNotFound) diff --git a/cluster/kube/builder/service_credentials.go b/cluster/kube/builder/service_credentials.go new file mode 100644 index 00000000..bad65f9f --- /dev/null +++ b/cluster/kube/builder/service_credentials.go @@ -0,0 +1,111 @@ +package builder + +import ( + "encoding/base64" + "encoding/json" + "fmt" + "strings" + + mani "github.com/akash-network/akash-api/go/manifest/v2beta2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type ServiceCredentials interface { + NS() string + Name() string + Create() (*corev1.Secret, error) + Update(obj *corev1.Secret) (*corev1.Secret, error) +} + +type serviceCredentials struct { + ns string + serviceName string + credentials *mani.ServiceImageCredentials + // TODO: labels for deleting + // labels map[string]string +} + +func NewServiceCredentials(ns string, serviceName string, credentials *mani.ServiceImageCredentials) ServiceCredentials { + return serviceCredentials{ + ns: ns, + serviceName: serviceName, + credentials: credentials, + } +} + +func (b serviceCredentials) NS() string { + return b.ns +} + +func (b serviceCredentials) Name() string { + return fmt.Sprintf("docker-creds-%v", b.serviceName) +} + +func (b serviceCredentials) Create() (*corev1.Secret, error) { + // see https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/create/create_secret_docker.go#L280-L298 + + data, err := b.encodeSecret() + if err != nil { + return nil, err + } + + obj := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.ns, + Name: b.Name(), + }, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: data, + }, + Type: corev1.SecretTypeDockerConfigJson, + } + + return obj, nil +} + +func (b serviceCredentials) Update(obj *corev1.Secret) (*corev1.Secret, error) { + // see https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/create/create_secret_docker.go#L280-L298 + + data, err := b.encodeSecret() + if err != nil { + return nil, err + } + + obj.Data = map[string][]byte{ + corev1.DockerConfigJsonKey: data, + } + obj.Type = corev1.SecretTypeDockerConfigJson + return obj, nil +} + +type dockerCredentialsEntry struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Email string `json:"email,omitempty"` + Auth string `json:"auth,omitempty"` +} + +type dockerCredentials struct { + Auths map[string]dockerCredentialsEntry `json:"auths"` +} + +func (b serviceCredentials) encodeSecret() ([]byte, error) { + entry := dockerCredentialsEntry{ + Username: strings.TrimSpace(b.credentials.Username), + Password: strings.TrimSpace(b.credentials.Password), + Email: strings.TrimSpace(b.credentials.Email), + Auth: encodeAuth(strings.TrimSpace(b.credentials.Username), strings.TrimSpace(b.credentials.Password)), + } + creds := dockerCredentials{ + Auths: map[string]dockerCredentialsEntry{ + b.credentials.Host: entry, + }, + } + return json.Marshal(creds) +} + +func encodeAuth(username, password string) string { + value := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(value)) +} diff --git a/cluster/kube/builder/workload.go b/cluster/kube/builder/workload.go index bd7c751b..d8f00b3b 100644 --- a/cluster/kube/builder/workload.go +++ b/cluster/kube/builder/workload.go @@ -26,6 +26,7 @@ const ( type workloadBase interface { builderBase Name() string + NS() string } type Workload struct { @@ -54,6 +55,10 @@ func (b *Workload) Name() string { return b.deployment.ManifestGroup().Services[b.serviceIdx].Name } +func (b *Workload) NS() string { + return LidNS(b.deployment.LeaseID()) +} + func (b *Workload) container() corev1.Container { falseValue := false @@ -362,11 +367,20 @@ func (b *Workload) labels() map[string]string { } func (b *Workload) imagePullSecrets() []corev1.LocalObjectReference { - if b.settings.DockerImagePullSecretsName == "" { + + sname := b.settings.DockerImagePullSecretsName + + // TODO: fix akash-api proto-gen + service := &b.deployment.ManifestGroup().Services[b.serviceIdx] + if service.Credentials != nil { + sname = NewServiceCredentials(b.NS(), b.Name(), service.Credentials).Name() + } + + if sname == "" { return nil } - return []corev1.LocalObjectReference{{Name: b.settings.DockerImagePullSecretsName}} + return []corev1.LocalObjectReference{{Name: sname}} } func (b *Workload) addEnvVarsForDeployment(envVarsAlreadyAdded map[string]int, env []corev1.EnvVar) []corev1.EnvVar { diff --git a/cluster/kube/client.go b/cluster/kube/client.go index a53ec4c3..f2eca84c 100644 --- a/cluster/kube/client.go +++ b/cluster/kube/client.go @@ -190,6 +190,7 @@ type deploymentService struct { statefulSet builder.StatefulSet localService builder.Service globalService builder.Service + credentials builder.ServiceCredentials } type deploymentApplies struct { @@ -268,6 +269,10 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err svc := &deploymentService{} + if service.Credentials != nil { + svc.credentials = builder.NewServiceCredentials(workload.NS(), workload.Name(), service.Credentials) + } + persistent := false for i := range service.Resources.Storage { attrVal := service.Resources.Storage[i].Attributes.Find(sdl.StorageAttributePersistent) @@ -291,6 +296,7 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err svc.localService = builder.BuildService(workload, false) svc.globalService = builder.BuildService(workload, true) + } if err := applyNS(ctx, c.kc, applies.ns); err != nil { @@ -320,6 +326,13 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err applyObjs := applies.services[svcIdx] service := &group.Services[svcIdx] + if applyObjs.credentials != nil { + if err = applyServiceCredentials(ctx, c.kc, applyObjs.credentials); err != nil { + c.log.Error("applying credentials", "err", err, "lease", lid, "service", service.Name) + return err + } + } + if applyObjs.statefulSet != nil { if err = applyStatefulSet(ctx, c.kc, applyObjs.statefulSet); err != nil { c.log.Error("applying statefulSet", "err", err, "lease", lid, "service", service.Name) diff --git a/go.mod b/go.mod index 409bdb21..5b787ca8 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/akash-network/provider go 1.21 require ( - github.com/akash-network/akash-api v0.0.61 + github.com/akash-network/akash-api v0.0.62 github.com/akash-network/node v0.32.3 github.com/avast/retry-go/v4 v4.5.0 github.com/blang/semver/v4 v4.0.0 diff --git a/go.sum b/go.sum index 0af2fda9..fee66b31 100644 --- a/go.sum +++ b/go.sum @@ -197,8 +197,8 @@ github.com/afex/hystrix-go v0.0.0-20180502004556-fa1af6a1f4f5/go.mod h1:SkGFH1ia github.com/agnivade/levenshtein v1.0.1/go.mod h1:CURSv5d9Uaml+FovSIICkLbAUZ9S4RqaHDIsdSBg7lM= github.com/ajg/form v1.5.1/go.mod h1:uL1WgH+h2mgNtvBq0339dVnzXdBETtL2LeUXaIv25UY= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= -github.com/akash-network/akash-api v0.0.61 h1:Hj/IBr9cFMsFs4VjymLZCoX/5dNfZSFd4iFwUSqhBtQ= -github.com/akash-network/akash-api v0.0.61/go.mod h1:pNr61L4+0sheol7ZK0HjgK3rxpIAbYBGq1w1oH4B0+M= +github.com/akash-network/akash-api v0.0.62 h1:o/V/iwlrel++5xo1UTlGO8aASKWTXm3to5+Z3IyYZUw= +github.com/akash-network/akash-api v0.0.62/go.mod h1:pNr61L4+0sheol7ZK0HjgK3rxpIAbYBGq1w1oH4B0+M= github.com/akash-network/cometbft v0.34.27-akash h1:V1dApDOr8Ee7BJzYyQ7Z9VBtrAul4+baMeA6C49dje0= github.com/akash-network/cometbft v0.34.27-akash/go.mod h1:BcCbhKv7ieM0KEddnYXvQZR+pZykTKReJJYf7YC7qhw= github.com/akash-network/ledger-go v0.14.3 h1:LCEFkTfgGA2xFMN2CtiKvXKE7dh0QSM77PJHCpSkaAo=