WIP feat: allow cert-manager annotations on ingress based on environment variables PT.2

_run/common-helm.mk

@@ -21,7 +21,7 @@ kind-install-helm-chart-loki:
 	helm upgrade --install promtail grafana/promtail \
 		--version $(PROMTAIL_VERSION) \
 		--namespace loki-stack \
-		-f ../promtail-values.yaml
+		-f ../promtail-values.yamlk
 	helm upgrade --install grafana grafana/grafana \
 		--version $(GRAFANA_VERSION) \
 		--namespace loki-stack \

cluster/client.go

@@ -82,7 +82,7 @@ type Client interface {
 		tsq remotecommand.TerminalSizeQueue) (ctypes.ExecResult, error)
 
 	// ConnectHostnameToDeployment Connect a given hostname to a deployment
-	ConnectHostnameToDeployment(ctx context.Context, directive ctypes.ConnectHostnameToDeploymentDirective) error
+	ConnectHostnameToDeployment(ctx context.Context, directive ctypes.ConnectHostnameToDeploymentDirective, sslEnabled bool) error
 	// RemoveHostnameFromDeployment Remove a given hostname from a deployment
 	RemoveHostnameFromDeployment(ctx context.Context, hostname string, leaseID mtypes.LeaseID, allowMissing bool) error
 
@@ -415,7 +415,7 @@ func (c *nullClient) GetHostnameDeploymentConnections(_ context.Context) ([]ctyp
 	return nil, errNotImplemented
 }
 
-func (c *nullClient) ConnectHostnameToDeployment(_ context.Context, _ ctypes.ConnectHostnameToDeploymentDirective) error {
+func (c *nullClient) ConnectHostnameToDeployment(_ context.Context, _ ctypes.ConnectHostnameToDeploymentDirective, _ bool) error {
 	return errNotImplemented
 }
 

cluster/kube/client.go

@@ -56,6 +56,7 @@ type client struct {
 	ns                string
 	log               log.Logger
 	kubeContentConfig *restclient.Config
+	cfg               ClientConfig
 }
 
 func (c *client) String() string {
@@ -64,7 +65,7 @@ func (c *client) String() string {
 
 // NewClient returns new Kubernetes Client instance with provided logger, host and ns. Returns error in-case of failure
 // configPath may be the empty string
-func NewClient(ctx context.Context, log log.Logger, ns string, configPath string) (Client, error) {
+func NewClient(ctx context.Context, log log.Logger, ns string, configPath string, ccfg ClientConfig) (Client, error) {
 	config, err := clientcommon.OpenKubeConfig(configPath, log)
 	if err != nil {
 		return nil, errors.Wrap(err, "kube: error building config flags")
@@ -98,6 +99,7 @@ func NewClient(ctx context.Context, log log.Logger, ns string, configPath string
 		ns:                ns,
 		log:               log.With("client", "kube"),
 		kubeContentConfig: config,
+		cfg:               ccfg,
 	}, nil
 }
 

cluster/kube/client_ingress.go

@@ -24,12 +24,13 @@ import (
 
 const (
 	akashIngressClassName = "akash-ingress-class"
+	root                  = "nginx.ingress.kubernetes.io"
+	certManager           = "cert-manager.io"
 )
 
-func kubeNginxIngressAnnotations(directive ctypes.ConnectHostnameToDeploymentDirective) map[string]string {
+func (c *client) kubeNginxIngressAnnotations(directive ctypes.ConnectHostnameToDeploymentDirective) map[string]string {
 	// For kubernetes/ingress-nginx
 	// https://github.com/kubernetes/ingress-nginx
-	const root = "nginx.ingress.kubernetes.io"
 
 	readTimeout := math.Ceil(float64(directive.ReadTimeout) / 1000.0)
 	sendTimeout := math.Ceil(float64(directive.SendTimeout) / 1000.0)
@@ -66,11 +67,20 @@ func kubeNginxIngressAnnotations(directive ctypes.ConnectHostnameToDeploymentDir
 		}
 	}
 
+	switch c.cfg.Ssl.IssuerType {
+	case clusterIssuer:
+		result[fmt.Sprintf("%s/cluster-issuer", certManager)] = c.cfg.Ssl.IssuerName
+		break
+	case issuer:
+		result[fmt.Sprintf("%s/issuer", certManager)] = c.cfg.Ssl.IssuerName
+		break
+	}
+
 	result[fmt.Sprintf("%s/proxy-next-upstream", root)] = strBuilder.String()
 	return result
 }
 
-func (c *client) ConnectHostnameToDeployment(ctx context.Context, directive ctypes.ConnectHostnameToDeploymentDirective) error {
+func (c *client) ConnectHostnameToDeployment(ctx context.Context, directive ctypes.ConnectHostnameToDeploymentDirective, tlsEnabled bool) error {
 	ingressName := directive.Hostname
 	ns := builder.LidNS(directive.LeaseID)
 	rules := ingressRules(directive.Hostname, directive.ServiceName, directive.ServicePort)
@@ -82,16 +92,27 @@ func (c *client) ConnectHostnameToDeployment(ctx context.Context, directive ctyp
 	labels[builder.AkashManagedLabelName] = "true"
 	builder.AppendLeaseLabels(directive.LeaseID, labels)
 
+	var tls []netv1.IngressTLS
+	if tlsEnabled {
+		tls = []netv1.IngressTLS{
+			{
+				Hosts:      []string{directive.Hostname},
+				SecretName: fmt.Sprintf("%s-tls", ingressName),
+			},
+		}
+	}
+
 	ingressClassName := akashIngressClassName
 	obj := &netv1.Ingress{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:        ingressName,
 			Labels:      labels,
-			Annotations: kubeNginxIngressAnnotations(directive),
+			Annotations: c.kubeNginxIngressAnnotations(directive),
 		},
 		Spec: netv1.IngressSpec{
 			IngressClassName: &ingressClassName,
 			Rules:            rules,
+			TLS:              tls,
 		},
 	}
 

cluster/kube/config.go

@@ -0,0 +1,15 @@
+package kube
+
+const (
+	issuer        = "issuer"
+	clusterIssuer = "cluster-issuer"
+)
+
+type ClientConfig struct {
+	Ssl Ssl
+}
+
+type Ssl struct {
+	IssuerType string
+	IssuerName string
+}

cluster/kube/deploy_test.go

@@ -44,7 +44,7 @@ func TestDeploy(t *testing.T) {
 	require.NoError(t, err)
 
 	log := testutil.Logger(t)
-	client, err := NewClient(ctx, log, "lease", "")
+	client, err := NewClient(ctx, log, "lease", "", ClientConfig{})
 	require.NoError(t, err)
 
 	ctx = context.WithValue(ctx, builder.SettingsKey, builder.NewDefaultSettings())

cluster/kube/k8s_integration_test.go

@@ -38,7 +38,7 @@ func TestNewClientNSNotFound(t *testing.T) {
 
 	ctx := context.WithValue(context.Background(), builder.SettingsKey, settings)
 
-	ac, err := NewClient(ctx, atestutil.Logger(t), ns, providerflags.KubeConfigDefaultPath)
+	ac, err := NewClient(ctx, atestutil.Logger(t), ns, providerflags.KubeConfigDefaultPath, ClientConfig{})
 	require.True(t, kubeErrors.IsNotFound(err))
 	require.Nil(t, ac)
 }
@@ -76,7 +76,7 @@ func TestNewClient(t *testing.T) {
 	}, metav1.CreateOptions{})
 	require.NoError(t, err)
 
-	ac, err := NewClient(ctx, atestutil.Logger(t), ns, providerflags.KubeConfigDefaultPath)
+	ac, err := NewClient(ctx, atestutil.Logger(t), ns, providerflags.KubeConfigDefaultPath, ClientConfig{})
 
 	require.NoError(t, err)
 

cmd/provider-services/cmd/flags/flags.go

@@ -9,5 +9,8 @@ const (
 	FlagWebRefreshInterval = "web-refresh-interval"
 	FlagRetryDelay         = "retry-delay"
 
-	FlagKubeConfig = "kubeconfig"
+	FlagKubeConfig    = "kubeconfig"
+	FlagSslEnabled    = "ssl"
+	FlagSslIssuerType = "ssl-issuer-type"
+	FlagSslIssuerName = "ssl-issuer-name"
 )

cmd/provider-services/cmd/run.go

@@ -350,6 +350,11 @@ func RunCmd() *cobra.Command {
 		return nil
 	}
 
+	cmd.Flags().Bool(providerflags.FlagSslEnabled, false, "enable issuing of SSL certificates on the provider's ingress controller. defaults to false")
+	if err := viper.BindPFlag(providerflags.FlagSslEnabled, cmd.Flags().Lookup(providerflags.FlagSslEnabled)); err != nil {
+		return nil
+	}
+
 	return cmd
 }
 
@@ -760,7 +765,16 @@ func createClusterClient(ctx context.Context, log log.Logger, _ *cobra.Command,
 	if ns == "" {
 		return nil, fmt.Errorf("%w: --%s required", errInvalidConfig, providerflags.FlagK8sManifestNS)
 	}
-	return kube.NewClient(ctx, log, ns, configPath)
+
+	var sslCfg kube.Ssl
+	if viper.GetBool(providerflags.FlagSslEnabled) {
+		sslCfg = kube.Ssl{
+			IssuerName: viper.GetString(providerflags.FlagSslIssuerName),
+			IssuerType: viper.GetString(providerflags.FlagSslIssuerType),
+		}
+	}
+	ccfg := kube.ClientConfig{Ssl: sslCfg}
+	return kube.NewClient(ctx, log, ns, configPath, ccfg)
 }
 
 func showErrorToUser(err error) error {

operator/hostnameoperator/hostname_operator.go

@@ -389,7 +389,7 @@ func (op *hostnameOperator) applyAddOrUpdateEvent(ctx context.Context, ev ctypes
 		if shouldConnect {
 			op.log.Debug("Updating ingress")
 			// Update or create the existing ingress
-			err = op.client.ConnectHostnameToDeployment(ctx, directive)
+			err = op.client.ConnectHostnameToDeployment(ctx, directive, op.isSslEnabled())
 		}
 	} else {
 		op.log.Debug("Swapping ingress to new deployment")
@@ -398,7 +398,7 @@ func (op *hostnameOperator) applyAddOrUpdateEvent(ctx context.Context, ev ctypes
 		if err == nil {
 			// Remove the current entry, if the next action succeeds then it gets inserted below
 			delete(op.hostnames, ev.GetHostname())
-			err = op.client.ConnectHostnameToDeployment(ctx, directive)
+			err = op.client.ConnectHostnameToDeployment(ctx, directive, op.isSslEnabled())
 		}
 	}
 
@@ -445,7 +445,7 @@ func doHostnameOperator(cmd *cobra.Command) error {
 	logger := operatorcommon.OpenLogger().With("op", "hostname")
 	logger.Info("HTTP listening", "address", listenAddr)
 
-	client, err := clusterClient.NewClient(cmd.Context(), logger, ns, configPath)
+	client, err := clusterClient.NewClient(cmd.Context(), logger, ns, configPath, config.ClientConfig)
 	if err != nil {
 		return err
 	}
@@ -502,3 +502,7 @@ func Cmd() *cobra.Command {
 
 	return cmd
 }
+
+func (op *hostnameOperator) isSslEnabled() bool {
+	return op.cfg.ClientConfig.Ssl != clusterClient.Ssl{}
+}

operator/hostnameoperator/hostname_operator_test.go

@@ -424,7 +424,7 @@ func TestHostnameOperatorApplyAdd(t *testing.T) {
 	}
 	s.client.On("GetManifestGroup", mock.Anything, leaseID).Return(true, mg, nil)
 	directive := buildDirective(ev, serviceExpose) // result tested in other unit tests
-	s.client.On("ConnectHostnameToDeployment", mock.Anything, directive).Return(nil)
+	s.client.On("ConnectHostnameToDeployment", mock.Anything, directive, mock.Anything).Return(nil)
 
 	managed := grabManagedHostnames(t, s.op.server.GetRouter().ServeHTTP)
 	require.Empty(t, managed)
@@ -511,7 +511,7 @@ func TestHostnameOperatorApplyAddMultipleServices(t *testing.T) {
 	}
 	s.client.On("GetManifestGroup", mock.Anything, leaseID).Return(true, mg, nil)
 	directive := buildDirective(ev, serviceExpose) // result tested in other unit tests
-	s.client.On("ConnectHostnameToDeployment", mock.Anything, directive).Return(nil)
+	s.client.On("ConnectHostnameToDeployment", mock.Anything, directive, mock.Anything).Return(nil)
 
 	err := s.op.applyEvent(s.ctx, ev)
 	require.NoError(t, err)
@@ -596,9 +596,9 @@ func TestHostnameOperatorApplyUpdate(t *testing.T) {
 	s.client.On("GetManifestGroup", mock.Anything, secondLeaseID).Return(true, mg2, nil)
 
 	directive := buildDirective(ev, serviceExpose) // result tested in other unit tests
-	s.client.On("ConnectHostnameToDeployment", mock.Anything, directive).Return(nil)
+	s.client.On("ConnectHostnameToDeployment", mock.Anything, directive, mock.Anything).Return(nil)
 	secondDirective := buildDirective(secondEv, secondServiceExpose) // result tested in other unit tests
-	s.client.On("ConnectHostnameToDeployment", mock.Anything, secondDirective).Return(nil)
+	s.client.On("ConnectHostnameToDeployment", mock.Anything, secondDirective, mock.Anything).Return(nil)
 
 	s.client.On("RemoveHostnameFromDeployment", mock.Anything, hostname, leaseID, false).Return(nil)
 

operator/ipoperator/ip_operator.go

@@ -579,13 +579,13 @@ func doIPOperator(cmd *cobra.Command) error {
 	poolName := viper.GetString(flagMetalLbPoolName)
 	logger := operatorcommon.OpenLogger().With("operator", "ip")
 
-	opcfg := operatorcommon.GetOperatorConfigFromViper()
-	_, err := sdk.AccAddressFromBech32(opcfg.ProviderAddress)
+	config := operatorcommon.GetOperatorConfigFromViper()
+	_, err := sdk.AccAddressFromBech32(config.ProviderAddress)
 	if err != nil {
 		return fmt.Errorf("%w: provider address must valid bech32", err)
 	}
 
-	client, err := clusterClient.NewClient(cmd.Context(), logger, ns, configPath)
+	client, err := clusterClient.NewClient(cmd.Context(), logger, ns, configPath, config.ClientConfig)
 	if err != nil {
 		return err
 	}
@@ -603,7 +603,7 @@ func doIPOperator(cmd *cobra.Command) error {
 	logger.Info("clients", "kube", client, "metallb", mllbc)
 	logger.Info("HTTP listening", "address", listenAddr)
 
-	op, err := newIPOperator(logger, client, opcfg, operatorcommon.IgnoreListConfigFromViper(), mllbc)
+	op, err := newIPOperator(logger, client, config, operatorcommon.IgnoreListConfigFromViper(), mllbc)
 	if err != nil {
 		return err
 	}

operator/operatorcommon/operator_config.go

@@ -1,6 +1,7 @@
 package operatorcommon
 
 import (
+	"github.com/akash-network/provider/cluster/kube"
 	"time"
 
 	"github.com/spf13/viper"
@@ -13,13 +14,24 @@ type OperatorConfig struct {
 	WebRefreshInterval time.Duration
 	RetryDelay         time.Duration
 	ProviderAddress    string
+	ClientConfig       kube.ClientConfig
 }
 
 func GetOperatorConfigFromViper() OperatorConfig {
+	var sslCfg kube.Ssl
+	if viper.GetBool(providerflags.FlagSslEnabled) {
+		sslCfg = kube.Ssl{
+			IssuerName: viper.GetString(providerflags.FlagSslIssuerName),
+			IssuerType: viper.GetString(providerflags.FlagSslIssuerType),
+		}
+	}
+	ccfg := kube.ClientConfig{Ssl: sslCfg}
+
 	return OperatorConfig{
 		PruneInterval:      viper.GetDuration(providerflags.FlagPruneInterval),
 		WebRefreshInterval: viper.GetDuration(providerflags.FlagWebRefreshInterval),
 		RetryDelay:         viper.GetDuration(providerflags.FlagRetryDelay),
 		ProviderAddress:    viper.GetString(flagProviderAddress),
+		ClientConfig:       ccfg,
 	}
 }