feat: support specifying env vars for control plane pods

Fixes #5055

Signed-off-by: Andrey Smirnov <[email protected]>

internal/app/machined/pkg/controllers/config/k8s_control_plane.go

@@ -151,6 +151,7 @@ func (ctrl *K8sControlPlaneController) manageAPIServerConfig(ctx context.Context
 			ServiceCIDRs:             cfgProvider.Cluster().Network().ServiceCIDRs(),
 			ExtraArgs:                cfgProvider.Cluster().APIServer().ExtraArgs(),
 			ExtraVolumes:             convertVolumes(cfgProvider.Cluster().APIServer().ExtraVolumes()),
+			EnvironmentVariables:     cfgProvider.Cluster().APIServer().Env(),
 			PodSecurityPolicyEnabled: !cfgProvider.Cluster().APIServer().DisablePodSecurityPolicy(),
 		})
 
@@ -185,13 +186,14 @@ func (ctrl *K8sControlPlaneController) manageControllerManagerConfig(ctx context
 
 	return r.Modify(ctx, config.NewK8sControlPlaneControllerManager(), func(r resource.Resource) error {
 		r.(*config.K8sControlPlane).SetControllerManager(config.K8sControlPlaneControllerManagerSpec{
-			Enabled:       !cfgProvider.Machine().Controlplane().ControllerManager().Disabled(),
-			Image:         cfgProvider.Cluster().ControllerManager().Image(),
-			CloudProvider: cloudProvider,
-			PodCIDRs:      cfgProvider.Cluster().Network().PodCIDRs(),
-			ServiceCIDRs:  cfgProvider.Cluster().Network().ServiceCIDRs(),
-			ExtraArgs:     cfgProvider.Cluster().ControllerManager().ExtraArgs(),
-			ExtraVolumes:  convertVolumes(cfgProvider.Cluster().ControllerManager().ExtraVolumes()),
+			Enabled:              !cfgProvider.Machine().Controlplane().ControllerManager().Disabled(),
+			Image:                cfgProvider.Cluster().ControllerManager().Image(),
+			CloudProvider:        cloudProvider,
+			PodCIDRs:             cfgProvider.Cluster().Network().PodCIDRs(),
+			ServiceCIDRs:         cfgProvider.Cluster().Network().ServiceCIDRs(),
+			ExtraArgs:            cfgProvider.Cluster().ControllerManager().ExtraArgs(),
+			ExtraVolumes:         convertVolumes(cfgProvider.Cluster().ControllerManager().ExtraVolumes()),
+			EnvironmentVariables: cfgProvider.Cluster().ControllerManager().Env(),
 		})
 
 		return nil
@@ -201,10 +203,11 @@ func (ctrl *K8sControlPlaneController) manageControllerManagerConfig(ctx context
 func (ctrl *K8sControlPlaneController) manageSchedulerConfig(ctx context.Context, r controller.Runtime, logger *zap.Logger, cfgProvider talosconfig.Provider) error {
 	return r.Modify(ctx, config.NewK8sControlPlaneScheduler(), func(r resource.Resource) error {
 		r.(*config.K8sControlPlane).SetScheduler(config.K8sControlPlaneSchedulerSpec{
-			Enabled:      !cfgProvider.Machine().Controlplane().Scheduler().Disabled(),
-			Image:        cfgProvider.Cluster().Scheduler().Image(),
-			ExtraArgs:    cfgProvider.Cluster().Scheduler().ExtraArgs(),
-			ExtraVolumes: convertVolumes(cfgProvider.Cluster().Scheduler().ExtraVolumes()),
+			Enabled:              !cfgProvider.Machine().Controlplane().Scheduler().Disabled(),
+			Image:                cfgProvider.Cluster().Scheduler().Image(),
+			ExtraArgs:            cfgProvider.Cluster().Scheduler().ExtraArgs(),
+			ExtraVolumes:         convertVolumes(cfgProvider.Cluster().Scheduler().ExtraVolumes()),
+			EnvironmentVariables: cfgProvider.Cluster().Scheduler().Env(),
 		})
 
 		return nil

internal/app/machined/pkg/controllers/config/k8s_control_plane_test.go

@@ -180,6 +180,33 @@ func (suite *K8sControlPlaneSuite) TestReconcileExtraVolumes() {
 	}, apiServerCfg.ExtraVolumes)
 }
 
+func (suite *K8sControlPlaneSuite) TestReconcileEnvironment() {
+	u, err := url.Parse("https://foo:6443")
+	suite.Require().NoError(err)
+
+	cfg := config.NewMachineConfig(&v1alpha1.Config{
+		ConfigVersion: "v1alpha1",
+		MachineConfig: &v1alpha1.MachineConfig{},
+		ClusterConfig: &v1alpha1.ClusterConfig{
+			ControlPlane: &v1alpha1.ControlPlaneConfig{
+				Endpoint: &v1alpha1.Endpoint{
+					URL: u,
+				},
+			},
+			APIServerConfig: &v1alpha1.APIServerConfig{
+				EnvConfig: v1alpha1.Env{
+					"HTTP_PROXY": "foo",
+				},
+			},
+		},
+	})
+
+	apiServerCfg := suite.setupMachine(cfg)
+	suite.Assert().Equal(map[string]string{
+		"HTTP_PROXY": "foo",
+	}, apiServerCfg.EnvironmentVariables)
+}
+
 func (suite *K8sControlPlaneSuite) TestReconcileExternalCloudProvider() {
 	u, err := url.Parse("https://foo:6443")
 	suite.Require().NoError(err)

internal/app/machined/pkg/controllers/k8s/control_plane_static_pod.go

@@ -8,6 +8,7 @@ import (
 	"context"
 	"fmt"
 	"path/filepath"
+	"sort"
 	"strconv"
 	"strings"
 
@@ -248,6 +249,32 @@ func volumes(volumes []config.K8sExtraVolume) []v1.Volume {
 	return result
 }
 
+func envVars(environment map[string]string) []v1.EnvVar {
+	if len(environment) == 0 {
+		return nil
+	}
+
+	keys := make([]string, 0, len(environment))
+
+	for k := range environment {
+		keys = append(keys, k)
+	}
+
+	sort.Strings(keys)
+
+	result := make([]v1.EnvVar, 0, len(environment))
+
+	for _, k := range keys {
+		// Kubernetes supports variable references in variable values, so escape '$' to prevent that.
+		result = append(result, v1.EnvVar{
+			Name:  k,
+			Value: strings.ReplaceAll(environment[k], "$", "$$"),
+		})
+	}
+
+	return result
+}
+
 func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context, r controller.Runtime, logger *zap.Logger,
 	configResource *config.K8sControlPlane, secretsVersion, configVersion string) (string, error) {
 	cfg := configResource.APIServer()
@@ -364,16 +391,18 @@ func (ctrl *ControlPlaneStaticPodController) manageAPIServer(ctx context.Context
 						Name:    "kube-apiserver",
 						Image:   cfg.Image,
 						Command: args,
-						Env: []v1.EnvVar{
-							{
-								Name: "POD_IP",
-								ValueFrom: &v1.EnvVarSource{
-									FieldRef: &v1.ObjectFieldSelector{
-										FieldPath: "status.podIP",
+						Env: append(
+							[]v1.EnvVar{
+								{
+									Name: "POD_IP",
+									ValueFrom: &v1.EnvVarSource{
+										FieldRef: &v1.ObjectFieldSelector{
+											FieldPath: "status.podIP",
+										},
 									},
 								},
 							},
-						},
+							envVars(cfg.EnvironmentVariables)...),
 						VolumeMounts: append([]v1.VolumeMount{
 							{
 								Name:      "secrets",
@@ -516,6 +545,7 @@ func (ctrl *ControlPlaneStaticPodController) manageControllerManager(ctx context
 						Name:    "kube-controller-manager",
 						Image:   cfg.Image,
 						Command: args,
+						Env:     envVars(cfg.EnvironmentVariables),
 						VolumeMounts: append([]v1.VolumeMount{
 							{
 								Name:      "secrets",
@@ -625,6 +655,7 @@ func (ctrl *ControlPlaneStaticPodController) manageScheduler(ctx context.Context
 						Name:    "kube-scheduler",
 						Image:   cfg.Image,
 						Command: args,
+						Env:     envVars(cfg.EnvironmentVariables),
 						VolumeMounts: append([]v1.VolumeMount{
 							{
 								Name:      "secrets",

internal/app/machined/pkg/controllers/k8s/control_plane_static_pod_test.go

@@ -349,7 +349,104 @@ func (suite *ControlPlaneStaticPodSuite) TestReconcileExtraArgs() {
 	}
 }
 
-func (suite *ControlPlaneStaticPodSuite) TestControlPlaneStaticPodsExeptScheduler() {
+func (suite *ControlPlaneStaticPodSuite) TestReconcileEnvironmentVariables() {
+	configStatus := k8s.NewConfigStatus(k8s.ControlPlaneNamespaceName, k8s.ConfigStatusStaticPodID)
+	secretStatus := k8s.NewSecretsStatus(k8s.ControlPlaneNamespaceName, k8s.StaticPodSecretsStaticPodID)
+
+	suite.Require().NoError(suite.state.Create(suite.ctx, configStatus))
+	suite.Require().NoError(suite.state.Create(suite.ctx, secretStatus))
+
+	tests := []struct {
+		env      map[string]string
+		expected []v1.EnvVar
+	}{
+		{
+			env: nil,
+			expected: []v1.EnvVar{
+				{
+					Name: "POD_IP",
+					ValueFrom: &v1.EnvVarSource{
+						FieldRef: &v1.ObjectFieldSelector{
+							FieldPath: "status.podIP",
+						},
+					},
+				},
+			},
+		},
+		{
+			env: map[string]string{
+				"foo": "bar",
+				"baz": "$(foo)",
+			},
+			expected: []v1.EnvVar{
+				{
+					Name: "POD_IP",
+					ValueFrom: &v1.EnvVarSource{
+						FieldRef: &v1.ObjectFieldSelector{
+							FieldPath: "status.podIP",
+						},
+					},
+				},
+				{
+					Name:  "baz",
+					Value: "$$(foo)",
+				},
+				{
+					Name:  "foo",
+					Value: "bar",
+				},
+			},
+		},
+	}
+	for _, test := range tests {
+		configAPIServer := config.NewK8sControlPlaneAPIServer()
+
+		configAPIServer.SetAPIServer(config.K8sControlPlaneAPIServerSpec{
+			EnvironmentVariables: test.env,
+		})
+
+		suite.Require().NoError(suite.state.Create(suite.ctx, configAPIServer))
+
+		suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
+			func() error {
+				return suite.assertControlPlaneStaticPods(
+					[]string{
+						"kube-apiserver",
+					},
+				)
+			},
+		))
+
+		r, err := suite.state.Get(suite.ctx, resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "kube-apiserver", resource.VersionUndefined))
+		suite.Require().NoError(err)
+
+		apiServerPod, err := k8sadapter.StaticPod(r.(*k8s.StaticPod)).Pod()
+		suite.Require().NoError(err)
+
+		suite.Require().NotEmpty(apiServerPod.Spec.Containers)
+
+		suite.Assert().Equal(test.expected, apiServerPod.Spec.Containers[0].Env)
+
+		suite.Require().NoError(suite.state.Destroy(suite.ctx, configAPIServer.Metadata()))
+
+		suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry(
+			func() error {
+				list, err := suite.state.List(suite.ctx, resource.NewMetadata(k8s.NamespaceName, k8s.StaticPodType, "", resource.VersionUndefined))
+				if err != nil {
+					return err
+				}
+
+				if len(list.Items) > 0 {
+					return retry.ExpectedErrorf("expected no pods, got %d", len(list.Items))
+				}
+
+				return nil
+			},
+		))
+	}
+}
+
+func (suite *ControlPlaneStaticPodSuite) TestControlPlaneStaticPodsExceptScheduler() {
 	configStatus := k8s.NewConfigStatus(k8s.ControlPlaneNamespaceName, k8s.ConfigStatusStaticPodID)
 	secretStatus := k8s.NewSecretsStatus(k8s.ControlPlaneNamespaceName, k8s.StaticPodSecretsStaticPodID)
 	configAPIServer := config.NewK8sControlPlaneAPIServer()

pkg/machinery/config/provider.go

@@ -379,6 +379,7 @@ type APIServer interface {
 	Image() string
 	ExtraArgs() map[string]string
 	ExtraVolumes() []VolumeMount
+	Env() Env
 	DisablePodSecurityPolicy() bool
 	AdmissionControl() []AdmissionPlugin
 }
@@ -395,6 +396,7 @@ type ControllerManager interface {
 	Image() string
 	ExtraArgs() map[string]string
 	ExtraVolumes() []VolumeMount
+	Env() Env
 }
 
 // Proxy defines the requirements for a config that pertains to the kube-proxy
@@ -417,6 +419,7 @@ type Scheduler interface {
 	Image() string
 	ExtraArgs() map[string]string
 	ExtraVolumes() []VolumeMount
+	Env() Env
 }
 
 // Etcd defines the requirements for a config that pertains to etcd related

pkg/machinery/config/types/v1alpha1/v1alpha1_apiserverconfig.go

@@ -38,6 +38,11 @@ func (a *APIServerConfig) ExtraVolumes() []config.VolumeMount {
 	return volumes
 }
 
+// Env implements the config.APIServer interface.
+func (a *APIServerConfig) Env() Env {
+	return a.EnvConfig
+}
+
 // DisablePodSecurityPolicy implements the config.APIServer interface.
 func (a *APIServerConfig) DisablePodSecurityPolicy() bool {
 	return a.DisablePodSecurityPolicyConfig

pkg/machinery/config/types/v1alpha1/v1alpha1_controllermanagerconfig.go

@@ -37,3 +37,8 @@ func (c *ControllerManagerConfig) ExtraVolumes() []config.VolumeMount {
 
 	return volumes
 }
+
+// Env implements the config.ControllerManager interface.
+func (c *ControllerManagerConfig) Env() Env {
+	return c.EnvConfig
+}

pkg/machinery/config/types/v1alpha1/v1alpha1_schedulerconfig.go

@@ -37,3 +37,8 @@ func (s *SchedulerConfig) ExtraVolumes() []config.VolumeMount {
 
 	return volumes
 }
+
+// Env implements the config.Scheduler interface.
+func (s *SchedulerConfig) Env() Env {
+	return s.EnvConfig
+}

pkg/machinery/config/types/v1alpha1/v1alpha1_types.go

@@ -1397,6 +1397,9 @@ type APIServerConfig struct {
 	//     Extra volumes to mount to the API server static pod.
 	ExtraVolumesConfig []VolumeMountConfig `yaml:"extraVolumes,omitempty"`
 	//   description: |
+	//     The `env` field allows for the addition of environment variables for the control plane component.
+	EnvConfig Env `yaml:"env,omitempty"`
+	//   description: |
 	//     Extra certificate subject alternative names for the API server's certificate.
 	CertSANs []string `yaml:"certSANs,omitempty"`
 	//   description: |
@@ -1434,6 +1437,9 @@ type ControllerManagerConfig struct {
 	//   description: |
 	//     Extra volumes to mount to the controller manager static pod.
 	ExtraVolumesConfig []VolumeMountConfig `yaml:"extraVolumes,omitempty"`
+	//   description: |
+	//     The `env` field allows for the addition of environment variables for the control plane component.
+	EnvConfig Env `yaml:"env,omitempty"`
 }
 
 // ProxyConfig represents the kube proxy configuration options.
@@ -1470,6 +1476,9 @@ type SchedulerConfig struct {
 	//   description: |
 	//     Extra volumes to mount to the scheduler static pod.
 	ExtraVolumesConfig []VolumeMountConfig `yaml:"extraVolumes,omitempty"`
+	//   description: |
+	//     The `env` field allows for the addition of environment variables for the control plane component.
+	EnvConfig Env `yaml:"env,omitempty"`
 }
 
 // EtcdConfig represents the etcd configuration options.