feat: add a way to override kubelet configuration via machine config

Fixes #4629 Note: some fields are enforced by Talos and are not overridable. Signed-off-by: Andrey Smirnov <[email protected]>
siderolabs · Feb 25, 2022 · eb40b92 · eb40b92
1 parent dc23715
commit eb40b92
Show file tree

Hide file tree

Showing 15 changed files with 422 additions and 60 deletions.
diff --git a/hack/release.toml b/hack/release.toml
@@ -16,7 +16,7 @@ preface = """\
 [notes]
 
     [notes.equinixMetal]
-        title = "Rename packer to equinixMetal in talos.platform"
+        title = "Equinix Metal Platform"
         description="""\
 `talos.platform` for Equinix Metal is renamed from `packet` to `equinixMetal`, the older name is still supported for backwards compatibility.
 """
@@ -35,8 +35,12 @@ with a single `--mode` flag that can take the following values:
 """
 
     [notes.kubelet]
-        title = "Kubelet conformance tweaks"
+        title = "Kubelet"
         description="""\
+Kubelet configuration can now be overridden with the `.machine.kubelet.extraConfig` machine configuration field.
+As most of the kubelet command line arguments are being depreacted, it is recommended to migrate to `extraConfig`
+instead of using `extraArgs`.
+
 A number of conformance tweaks have been made to the `kubelet` to allow it to run without
 `protectKernelDefaults`.
 This includes both kubelet configuration options and sysctls.
@@ -45,7 +49,7 @@ If your kubelet fails to start after the upgrade, please check the `kubelet` log
 """
 
     [notes.auditlog]
-        title = "API Server audit logs"
+        title = "API Server Audit Logs"
         description="""\
 `kube-apiserver` is now configured to store its audit logs separately from the `kube-apiserver` standard logs and directly to file.
 The `kube-apiserver` will maintain the rotation and retirement of these logs, which are stored in `/var/log/audit/`.

diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_config.go b/internal/app/machined/pkg/controllers/k8s/kubelet_config.go
@@ -97,6 +97,7 @@ func (ctrl *KubeletConfigController) Run(ctx context.Context, r controller.Runti
 				kubeletConfig.ClusterDomain = cfgProvider.Cluster().Network().DNSDomain()
 				kubeletConfig.ExtraArgs = cfgProvider.Machine().Kubelet().ExtraArgs()
 				kubeletConfig.ExtraMounts = cfgProvider.Machine().Kubelet().ExtraMounts()
+				kubeletConfig.ExtraConfig = cfgProvider.Machine().Kubelet().ExtraConfig()
 				kubeletConfig.CloudProviderExternal = cfgProvider.Cluster().ExternalCloudProvider().Enabled()
 
 				return nil

diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_config_test.go b/internal/app/machined/pkg/controllers/k8s/kubelet_config_test.go
@@ -89,6 +89,11 @@ func (suite *KubeletConfigSuite) TestReconcile() {
 						},
 					},
 				},
+				KubeletExtraConfig: v1alpha1.Unstructured{
+					Object: map[string]interface{}{
+						"serverTLSBootstrap": true,
+					},
+				},
 			},
 		},
 		ClusterConfig: &v1alpha1.ClusterConfig{
@@ -138,6 +143,12 @@ func (suite *KubeletConfigSuite) TestReconcile() {
 					},
 				},
 				spec.ExtraMounts)
+			suite.Assert().Equal(
+				map[string]interface{}{
+					"serverTLSBootstrap": true,
+				},
+				spec.ExtraConfig,
+			)
 			suite.Assert().True(spec.CloudProviderExternal)
 
 			return nil

diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go b/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go
@@ -14,15 +14,16 @@ import (
 	"github.com/cosi-project/runtime/pkg/controller"
 	"github.com/cosi-project/runtime/pkg/resource"
 	"github.com/cosi-project/runtime/pkg/state"
+	"github.com/hashicorp/go-multierror"
 	"go.uber.org/zap"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
-	"k8s.io/component-base/config/v1alpha1"
 	kubeletconfig "k8s.io/kubelet/config/v1beta1"
 
 	v1alpha1runtime "github.com/talos-systems/talos/internal/app/machined/pkg/runtime"
 	"github.com/talos-systems/talos/pkg/argsbuilder"
 	"github.com/talos-systems/talos/pkg/machinery/constants"
+	"github.com/talos-systems/talos/pkg/machinery/kubelet"
 	"github.com/talos-systems/talos/pkg/machinery/resources/k8s"
 )
 
@@ -158,7 +159,10 @@ func (ctrl *KubeletSpecController) Run(ctx context.Context, r controller.Runtime
 			return fmt.Errorf("error merging arguments: %w", err)
 		}
 
-		kubeletConfig := newKubeletConfiguration(cfgSpec.ClusterDNS, cfgSpec.ClusterDomain)
+		kubeletConfig, err := NewKubeletConfiguration(cfgSpec.ClusterDNS, cfgSpec.ClusterDomain, cfgSpec.ExtraConfig)
+		if err != nil {
+			return fmt.Errorf("error creating kubelet configuration: %w", err)
+		}
 
 		// If our platform is container, we cannot rely on the ability to change kernel parameters.
 		// Therefore, we need to NOT attempt to enforce the kernel parameter checking done by the kubelet
@@ -191,49 +195,112 @@ func (ctrl *KubeletSpecController) Run(ctx context.Context, r controller.Runtime
 	}
 }
 
-func newKubeletConfiguration(clusterDNS []string, dnsDomain string) *kubeletconfig.KubeletConfiguration {
-	return &kubeletconfig.KubeletConfiguration{
-		TypeMeta: metav1.TypeMeta{
-			APIVersion: "kubelet.config.k8s.io/v1beta1",
-			Kind:       "KubeletConfiguration",
+func prepareExtraConfig(extraConfig map[string]interface{}) (*kubeletconfig.KubeletConfiguration, error) {
+	// check for fields that can't be overridden via extraConfig
+	var multiErr *multierror.Error
+
+	for _, field := range kubelet.ProtectedConfigurationFields {
+		if _, exists := extraConfig[field]; exists {
+			multiErr = multierror.Append(multiErr, fmt.Errorf("field %q can't be overridden", field))
+		}
+	}
+
+	if err := multiErr.ErrorOrNil(); err != nil {
+		return nil, err
+	}
+
+	var config kubeletconfig.KubeletConfiguration
+
+	// unmarshal extra config into the config structure
+	// as unmarshalling zeroes the missing fields, we can't do that after setting the defaults
+	if err := runtime.DefaultUnstructuredConverter.FromUnstructuredWithValidation(extraConfig, &config, true); err != nil {
+		return nil, fmt.Errorf("error unmarshalling extra kubelet configuration: %w", err)
+	}
+
+	return &config, nil
+}
+
+// NewKubeletConfiguration builds kubelet configuration with defaults and overrides from extraConfig.
+//
+//nolint:gocyclo
+func NewKubeletConfiguration(clusterDNS []string, dnsDomain string, extraConfig map[string]interface{}) (*kubeletconfig.KubeletConfiguration, error) {
+	config, err := prepareExtraConfig(extraConfig)
+	if err != nil {
+		return nil, err
+	}
+
+	// required fields (always set)
+	config.TypeMeta = metav1.TypeMeta{
+		APIVersion: kubeletconfig.SchemeGroupVersion.String(),
+		Kind:       "KubeletConfiguration",
+	}
+	config.StaticPodPath = constants.ManifestsDirectory
+	config.Port = constants.KubeletPort
+	config.Authentication = kubeletconfig.KubeletAuthentication{
+		X509: kubeletconfig.KubeletX509Authentication{
+			ClientCAFile: constants.KubernetesCACert,
 		},
-		StaticPodPath:      constants.ManifestsDirectory,
-		Address:            "0.0.0.0",
-		Port:               constants.KubeletPort,
-		OOMScoreAdj:        pointer.ToInt32(constants.KubeletOOMScoreAdj),
-		RotateCertificates: true,
-		Authentication: kubeletconfig.KubeletAuthentication{
-			X509: kubeletconfig.KubeletX509Authentication{
-				ClientCAFile: constants.KubernetesCACert,
-			},
-			Webhook: kubeletconfig.KubeletWebhookAuthentication{
-				Enabled: pointer.ToBool(true),
-			},
-			Anonymous: kubeletconfig.KubeletAnonymousAuthentication{
-				Enabled: pointer.ToBool(false),
-			},
+		Webhook: kubeletconfig.KubeletWebhookAuthentication{
+			Enabled: pointer.ToBool(true),
 		},
-		Authorization: kubeletconfig.KubeletAuthorization{
-			Mode: kubeletconfig.KubeletAuthorizationModeWebhook,
+		Anonymous: kubeletconfig.KubeletAnonymousAuthentication{
+			Enabled: pointer.ToBool(false),
 		},
-		ClusterDomain:       dnsDomain,
-		ClusterDNS:          clusterDNS,
-		SerializeImagePulls: pointer.ToBool(false),
-		FailSwapOn:          pointer.ToBool(false),
-		CgroupRoot:          "/",
-		SystemCgroups:       constants.CgroupSystem,
-		SystemReserved: map[string]string{
+	}
+	config.Authorization = kubeletconfig.KubeletAuthorization{
+		Mode: kubeletconfig.KubeletAuthorizationModeWebhook,
+	}
+	config.CgroupRoot = "/"
+	config.SystemCgroups = constants.CgroupSystem
+	config.KubeletCgroups = constants.CgroupKubelet
+	config.RotateCertificates = true
+	config.ProtectKernelDefaults = true
+
+	// fields which can be overridden
+	if config.Address == "" {
+		config.Address = "0.0.0.0"
+	}
+
+	if config.OOMScoreAdj == nil {
+		config.OOMScoreAdj = pointer.ToInt32(constants.KubeletOOMScoreAdj)
+	}
+
+	if config.ClusterDomain == "" {
+		config.ClusterDomain = dnsDomain
+	}
+
+	if len(config.ClusterDNS) == 0 {
+		config.ClusterDNS = clusterDNS
+	}
+
+	if config.SerializeImagePulls == nil {
+		config.SerializeImagePulls = pointer.ToBool(false)
+	}
+
+	if config.FailSwapOn == nil {
+		config.FailSwapOn = pointer.ToBool(false)
+	}
+
+	if len(config.SystemReserved) == 0 {
+		config.SystemReserved = map[string]string{
 			"cpu":               constants.KubeletSystemReservedCPU,
 			"memory":            constants.KubeletSystemReservedMemory,
 			"pid":               constants.KubeletSystemReservedPid,
 			"ephemeral-storage": constants.KubeletSystemReservedEphemeralStorage,
-		},
-		KubeletCgroups: constants.CgroupKubelet,
-		Logging: v1alpha1.LoggingConfiguration{
-			Format: "json",
-		},
-		ProtectKernelDefaults:          true,
-		StreamingConnectionIdleTimeout: metav1.Duration{Duration: 5 * time.Minute},
-		TLSMinVersion:                  "VersionTLS13",
+		}
+	}
+
+	if config.Logging.Format == "" {
+		config.Logging.Format = "json"
 	}
+
+	if config.StreamingConnectionIdleTimeout.Duration == 0 {
+		config.StreamingConnectionIdleTimeout = metav1.Duration{Duration: 5 * time.Minute}
+	}
+
+	if config.TLSMinVersion == "" {
+		config.TLSMinVersion = "VersionTLS13"
+	}
+
+	return config, nil
 }