diff --git a/cluster/kube/apply.go b/cluster/kube/apply.go index 57ab0d2c..e957af29 100644 --- a/cluster/kube/apply.go +++ b/cluster/kube/apply.go @@ -86,6 +86,29 @@ func applyNetPolicies(ctx context.Context, kc kubernetes.Interface, b builder.Ne // return err // } +func applyServiceCredentials(ctx context.Context, kc kubernetes.Interface, b builder.ServiceCredentials) error { + obj, err := kc.CoreV1().Secrets(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{}) + metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "secrets-get", err, errors.IsNotFound) + + switch { + case err == nil: + obj, err = b.Update(obj) + if err == nil { + _, err = kc.CoreV1().Secrets(b.NS()).Update(ctx, obj, metav1.UpdateOptions{}) + metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "secrets-get", err) + + } + case errors.IsNotFound(err): + obj, err = b.Create() + if err == nil { + _, err = kc.CoreV1().Secrets(b.NS()).Create(ctx, obj, metav1.CreateOptions{}) + metricsutils.IncCounterVecWithLabelValues(kubeCallsCounter, "secrets-create", err) + } + } + return err + +} + func applyDeployment(ctx context.Context, kc kubernetes.Interface, b builder.Deployment) error { obj, err := kc.AppsV1().Deployments(b.NS()).Get(ctx, b.Name(), metav1.GetOptions{}) metricsutils.IncCounterVecWithLabelValuesFiltered(kubeCallsCounter, "deployments-get", err, errors.IsNotFound) diff --git a/cluster/kube/builder/service_credentials.go b/cluster/kube/builder/service_credentials.go new file mode 100644 index 00000000..c517beb0 --- /dev/null +++ b/cluster/kube/builder/service_credentials.go @@ -0,0 +1,110 @@ +package builder + +import ( + "encoding/base64" + "encoding/json" + "fmt" + + mani "github.com/akash-network/akash-api/go/manifest/v2beta2" + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +type ServiceCredentials interface { + NS() string + Name() string + Create() (*corev1.Secret, error) + Update(obj *corev1.Secret) (*corev1.Secret, error) +} + +type serviceCredentials struct { + ns string + serviceName string + credentials *mani.ServiceImageCredentials + // TODO: labels for deleting + // labels map[string]string +} + +func NewServiceCredentials(ns string, serviceName string, credentials *mani.ServiceImageCredentials) ServiceCredentials { + return serviceCredentials{ + ns: ns, + serviceName: serviceName, + credentials: credentials, + } +} + +func (b serviceCredentials) NS() string { + return b.ns +} + +func (b serviceCredentials) Name() string { + return fmt.Sprintf("docker-creds-%v", b.serviceName) +} + +func (b serviceCredentials) Create() (*corev1.Secret, error) { + // see https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/create/create_secret_docker.go#L280-L298 + + data, err := b.encodeSecret() + if err != nil { + return nil, err + } + + obj := &corev1.Secret{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: b.ns, + Name: b.Name(), + }, + Data: map[string][]byte{ + corev1.DockerConfigJsonKey: data, + }, + Type: corev1.SecretTypeDockerConfigJson, + } + + return obj, nil +} + +func (b serviceCredentials) Update(obj *corev1.Secret) (*corev1.Secret, error) { + // see https://github.com/kubernetes/kubectl/blob/master/pkg/cmd/create/create_secret_docker.go#L280-L298 + + data, err := b.encodeSecret() + if err != nil { + return nil, err + } + + obj.Data = map[string][]byte{ + corev1.DockerConfigJsonKey: data, + } + obj.Type = corev1.SecretTypeDockerConfigJson + return obj, nil +} + +type dockerCredentialsEntry struct { + Username string `json:"username,omitempty"` + Password string `json:"password,omitempty"` + Email string `json:"email,omitempty"` + Auth string `json:"auth,omitempty"` +} + +type dockerCredentials struct { + Auths map[string]dockerCredentialsEntry `json:"auths"` +} + +func (b serviceCredentials) encodeSecret() ([]byte, error) { + entry := dockerCredentialsEntry{ + Username: b.credentials.Username, + Password: b.credentials.Password, + Email: b.credentials.Email, + Auth: encodeAuth(b.credentials.Username, b.credentials.Password), + } + creds := dockerCredentials{ + Auths: map[string]dockerCredentialsEntry{ + b.credentials.Host: entry, + }, + } + return json.Marshal(creds) +} + +func encodeAuth(username, password string) string { + value := username + ":" + password + return base64.StdEncoding.EncodeToString([]byte(value)) +} diff --git a/cluster/kube/builder/workload.go b/cluster/kube/builder/workload.go index 48683f28..b7049c16 100644 --- a/cluster/kube/builder/workload.go +++ b/cluster/kube/builder/workload.go @@ -26,6 +26,7 @@ const ( type workloadBase interface { builderBase Name() string + NS() string } type Workload struct { @@ -54,6 +55,10 @@ func (b *Workload) Name() string { return b.deployment.ManifestGroup().Services[b.serviceIdx].Name } +func (b *Workload) NS() string { + return LidNS(b.deployment.LeaseID()) +} + func (b *Workload) container() corev1.Container { falseValue := false @@ -317,11 +322,20 @@ func (b *Workload) labels() map[string]string { } func (b *Workload) imagePullSecrets() []corev1.LocalObjectReference { - if b.settings.DockerImagePullSecretsName == "" { + + sname := b.settings.DockerImagePullSecretsName + + // TODO: fix akash-api proto-gen + service := &b.deployment.ManifestGroup().Services[b.serviceIdx] + if service.Credentials != nil { + sname = NewServiceCredentials(b.NS(), b.Name(), service.Credentials).Name() + } + + if sname == "" { return nil } - return []corev1.LocalObjectReference{{Name: b.settings.DockerImagePullSecretsName}} + return []corev1.LocalObjectReference{{Name: sname}} } func (b *Workload) addEnvVarsForDeployment(envVarsAlreadyAdded map[string]int, env []corev1.EnvVar) []corev1.EnvVar { diff --git a/cluster/kube/client.go b/cluster/kube/client.go index a53ec4c3..f2eca84c 100644 --- a/cluster/kube/client.go +++ b/cluster/kube/client.go @@ -190,6 +190,7 @@ type deploymentService struct { statefulSet builder.StatefulSet localService builder.Service globalService builder.Service + credentials builder.ServiceCredentials } type deploymentApplies struct { @@ -268,6 +269,10 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err svc := &deploymentService{} + if service.Credentials != nil { + svc.credentials = builder.NewServiceCredentials(workload.NS(), workload.Name(), service.Credentials) + } + persistent := false for i := range service.Resources.Storage { attrVal := service.Resources.Storage[i].Attributes.Find(sdl.StorageAttributePersistent) @@ -291,6 +296,7 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err svc.localService = builder.BuildService(workload, false) svc.globalService = builder.BuildService(workload, true) + } if err := applyNS(ctx, c.kc, applies.ns); err != nil { @@ -320,6 +326,13 @@ func (c *client) Deploy(ctx context.Context, deployment ctypes.IDeployment) (err applyObjs := applies.services[svcIdx] service := &group.Services[svcIdx] + if applyObjs.credentials != nil { + if err = applyServiceCredentials(ctx, c.kc, applyObjs.credentials); err != nil { + c.log.Error("applying credentials", "err", err, "lease", lid, "service", service.Name) + return err + } + } + if applyObjs.statefulSet != nil { if err = applyStatefulSet(ctx, c.kc, applyObjs.statefulSet); err != nil { c.log.Error("applying statefulSet", "err", err, "lease", lid, "service", service.Name)