From eb40b9254f4cce4db86bc49c5dfa67f3feffbdaa Mon Sep 17 00:00:00 2001 From: Andrey Smirnov Date: Fri, 25 Feb 2022 00:23:28 +0300 Subject: [PATCH] feat: add a way to override kubelet configuration via machine config Fixes #4629 Note: some fields are enforced by Talos and are not overridable. Signed-off-by: Andrey Smirnov --- hack/release.toml | 10 +- .../pkg/controllers/k8s/kubelet_config.go | 1 + .../controllers/k8s/kubelet_config_test.go | 11 ++ .../pkg/controllers/k8s/kubelet_spec.go | 145 +++++++++++----- .../pkg/controllers/k8s/kubelet_spec_test.go | 156 ++++++++++++++++++ pkg/machinery/config/provider.go | 1 + .../types/v1alpha1/v1alpha1_provider.go | 5 + .../config/types/v1alpha1/v1alpha1_types.go | 14 ++ .../types/v1alpha1/v1alpha1_types_doc.go | 31 ++-- .../types/v1alpha1/v1alpha1_validation.go | 7 + .../v1alpha1/v1alpha1_validation_test.go | 24 +++ .../types/v1alpha1/zz_generated.deepcopy.go | 1 + pkg/machinery/kubelet/kubelet.go | 23 +++ pkg/machinery/resources/k8s/kubelet_config.go | 19 ++- .../docs/v0.15/Reference/configuration.md | 34 ++++ 15 files changed, 422 insertions(+), 60 deletions(-) create mode 100644 pkg/machinery/kubelet/kubelet.go diff --git a/hack/release.toml b/hack/release.toml index d855972505..4ca1ab9680 100644 --- a/hack/release.toml +++ b/hack/release.toml @@ -16,7 +16,7 @@ preface = """\ [notes] [notes.equinixMetal] - title = "Rename packer to equinixMetal in talos.platform" + title = "Equinix Metal Platform" description="""\ `talos.platform` for Equinix Metal is renamed from `packet` to `equinixMetal`, the older name is still supported for backwards compatibility. """ @@ -35,8 +35,12 @@ with a single `--mode` flag that can take the following values: """ [notes.kubelet] - title = "Kubelet conformance tweaks" + title = "Kubelet" description="""\ +Kubelet configuration can now be overridden with the `.machine.kubelet.extraConfig` machine configuration field. +As most of the kubelet command line arguments are being depreacted, it is recommended to migrate to `extraConfig` +instead of using `extraArgs`. + A number of conformance tweaks have been made to the `kubelet` to allow it to run without `protectKernelDefaults`. This includes both kubelet configuration options and sysctls. @@ -45,7 +49,7 @@ If your kubelet fails to start after the upgrade, please check the `kubelet` log """ [notes.auditlog] - title = "API Server audit logs" + title = "API Server Audit Logs" description="""\ `kube-apiserver` is now configured to store its audit logs separately from the `kube-apiserver` standard logs and directly to file. The `kube-apiserver` will maintain the rotation and retirement of these logs, which are stored in `/var/log/audit/`. diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_config.go b/internal/app/machined/pkg/controllers/k8s/kubelet_config.go index 6a64da78bf..652b213c6a 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_config.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_config.go @@ -97,6 +97,7 @@ func (ctrl *KubeletConfigController) Run(ctx context.Context, r controller.Runti kubeletConfig.ClusterDomain = cfgProvider.Cluster().Network().DNSDomain() kubeletConfig.ExtraArgs = cfgProvider.Machine().Kubelet().ExtraArgs() kubeletConfig.ExtraMounts = cfgProvider.Machine().Kubelet().ExtraMounts() + kubeletConfig.ExtraConfig = cfgProvider.Machine().Kubelet().ExtraConfig() kubeletConfig.CloudProviderExternal = cfgProvider.Cluster().ExternalCloudProvider().Enabled() return nil diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_config_test.go b/internal/app/machined/pkg/controllers/k8s/kubelet_config_test.go index bee817d18b..1bc8227f13 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_config_test.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_config_test.go @@ -89,6 +89,11 @@ func (suite *KubeletConfigSuite) TestReconcile() { }, }, }, + KubeletExtraConfig: v1alpha1.Unstructured{ + Object: map[string]interface{}{ + "serverTLSBootstrap": true, + }, + }, }, }, ClusterConfig: &v1alpha1.ClusterConfig{ @@ -138,6 +143,12 @@ func (suite *KubeletConfigSuite) TestReconcile() { }, }, spec.ExtraMounts) + suite.Assert().Equal( + map[string]interface{}{ + "serverTLSBootstrap": true, + }, + spec.ExtraConfig, + ) suite.Assert().True(spec.CloudProviderExternal) return nil diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go b/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go index ea0b98bdcc..53a3dcbf22 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_spec.go @@ -14,15 +14,16 @@ import ( "github.com/cosi-project/runtime/pkg/controller" "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/state" + "github.com/hashicorp/go-multierror" "go.uber.org/zap" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" - "k8s.io/component-base/config/v1alpha1" kubeletconfig "k8s.io/kubelet/config/v1beta1" v1alpha1runtime "github.com/talos-systems/talos/internal/app/machined/pkg/runtime" "github.com/talos-systems/talos/pkg/argsbuilder" "github.com/talos-systems/talos/pkg/machinery/constants" + "github.com/talos-systems/talos/pkg/machinery/kubelet" "github.com/talos-systems/talos/pkg/machinery/resources/k8s" ) @@ -158,7 +159,10 @@ func (ctrl *KubeletSpecController) Run(ctx context.Context, r controller.Runtime return fmt.Errorf("error merging arguments: %w", err) } - kubeletConfig := newKubeletConfiguration(cfgSpec.ClusterDNS, cfgSpec.ClusterDomain) + kubeletConfig, err := NewKubeletConfiguration(cfgSpec.ClusterDNS, cfgSpec.ClusterDomain, cfgSpec.ExtraConfig) + if err != nil { + return fmt.Errorf("error creating kubelet configuration: %w", err) + } // If our platform is container, we cannot rely on the ability to change kernel parameters. // Therefore, we need to NOT attempt to enforce the kernel parameter checking done by the kubelet @@ -191,49 +195,112 @@ func (ctrl *KubeletSpecController) Run(ctx context.Context, r controller.Runtime } } -func newKubeletConfiguration(clusterDNS []string, dnsDomain string) *kubeletconfig.KubeletConfiguration { - return &kubeletconfig.KubeletConfiguration{ - TypeMeta: metav1.TypeMeta{ - APIVersion: "kubelet.config.k8s.io/v1beta1", - Kind: "KubeletConfiguration", +func prepareExtraConfig(extraConfig map[string]interface{}) (*kubeletconfig.KubeletConfiguration, error) { + // check for fields that can't be overridden via extraConfig + var multiErr *multierror.Error + + for _, field := range kubelet.ProtectedConfigurationFields { + if _, exists := extraConfig[field]; exists { + multiErr = multierror.Append(multiErr, fmt.Errorf("field %q can't be overridden", field)) + } + } + + if err := multiErr.ErrorOrNil(); err != nil { + return nil, err + } + + var config kubeletconfig.KubeletConfiguration + + // unmarshal extra config into the config structure + // as unmarshalling zeroes the missing fields, we can't do that after setting the defaults + if err := runtime.DefaultUnstructuredConverter.FromUnstructuredWithValidation(extraConfig, &config, true); err != nil { + return nil, fmt.Errorf("error unmarshalling extra kubelet configuration: %w", err) + } + + return &config, nil +} + +// NewKubeletConfiguration builds kubelet configuration with defaults and overrides from extraConfig. +// +//nolint:gocyclo +func NewKubeletConfiguration(clusterDNS []string, dnsDomain string, extraConfig map[string]interface{}) (*kubeletconfig.KubeletConfiguration, error) { + config, err := prepareExtraConfig(extraConfig) + if err != nil { + return nil, err + } + + // required fields (always set) + config.TypeMeta = metav1.TypeMeta{ + APIVersion: kubeletconfig.SchemeGroupVersion.String(), + Kind: "KubeletConfiguration", + } + config.StaticPodPath = constants.ManifestsDirectory + config.Port = constants.KubeletPort + config.Authentication = kubeletconfig.KubeletAuthentication{ + X509: kubeletconfig.KubeletX509Authentication{ + ClientCAFile: constants.KubernetesCACert, }, - StaticPodPath: constants.ManifestsDirectory, - Address: "0.0.0.0", - Port: constants.KubeletPort, - OOMScoreAdj: pointer.ToInt32(constants.KubeletOOMScoreAdj), - RotateCertificates: true, - Authentication: kubeletconfig.KubeletAuthentication{ - X509: kubeletconfig.KubeletX509Authentication{ - ClientCAFile: constants.KubernetesCACert, - }, - Webhook: kubeletconfig.KubeletWebhookAuthentication{ - Enabled: pointer.ToBool(true), - }, - Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ - Enabled: pointer.ToBool(false), - }, + Webhook: kubeletconfig.KubeletWebhookAuthentication{ + Enabled: pointer.ToBool(true), }, - Authorization: kubeletconfig.KubeletAuthorization{ - Mode: kubeletconfig.KubeletAuthorizationModeWebhook, + Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ + Enabled: pointer.ToBool(false), }, - ClusterDomain: dnsDomain, - ClusterDNS: clusterDNS, - SerializeImagePulls: pointer.ToBool(false), - FailSwapOn: pointer.ToBool(false), - CgroupRoot: "/", - SystemCgroups: constants.CgroupSystem, - SystemReserved: map[string]string{ + } + config.Authorization = kubeletconfig.KubeletAuthorization{ + Mode: kubeletconfig.KubeletAuthorizationModeWebhook, + } + config.CgroupRoot = "/" + config.SystemCgroups = constants.CgroupSystem + config.KubeletCgroups = constants.CgroupKubelet + config.RotateCertificates = true + config.ProtectKernelDefaults = true + + // fields which can be overridden + if config.Address == "" { + config.Address = "0.0.0.0" + } + + if config.OOMScoreAdj == nil { + config.OOMScoreAdj = pointer.ToInt32(constants.KubeletOOMScoreAdj) + } + + if config.ClusterDomain == "" { + config.ClusterDomain = dnsDomain + } + + if len(config.ClusterDNS) == 0 { + config.ClusterDNS = clusterDNS + } + + if config.SerializeImagePulls == nil { + config.SerializeImagePulls = pointer.ToBool(false) + } + + if config.FailSwapOn == nil { + config.FailSwapOn = pointer.ToBool(false) + } + + if len(config.SystemReserved) == 0 { + config.SystemReserved = map[string]string{ "cpu": constants.KubeletSystemReservedCPU, "memory": constants.KubeletSystemReservedMemory, "pid": constants.KubeletSystemReservedPid, "ephemeral-storage": constants.KubeletSystemReservedEphemeralStorage, - }, - KubeletCgroups: constants.CgroupKubelet, - Logging: v1alpha1.LoggingConfiguration{ - Format: "json", - }, - ProtectKernelDefaults: true, - StreamingConnectionIdleTimeout: metav1.Duration{Duration: 5 * time.Minute}, - TLSMinVersion: "VersionTLS13", + } + } + + if config.Logging.Format == "" { + config.Logging.Format = "json" } + + if config.StreamingConnectionIdleTimeout.Duration == 0 { + config.StreamingConnectionIdleTimeout = metav1.Duration{Duration: 5 * time.Minute} + } + + if config.TLSMinVersion == "" { + config.TLSMinVersion = "VersionTLS13" + } + + return config, nil } diff --git a/internal/app/machined/pkg/controllers/k8s/kubelet_spec_test.go b/internal/app/machined/pkg/controllers/k8s/kubelet_spec_test.go index df1a0a5094..3776c40860 100644 --- a/internal/app/machined/pkg/controllers/k8s/kubelet_spec_test.go +++ b/internal/app/machined/pkg/controllers/k8s/kubelet_spec_test.go @@ -12,18 +12,26 @@ import ( "testing" "time" + "github.com/AlekSi/pointer" "github.com/cosi-project/runtime/pkg/controller/runtime" "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/state" "github.com/cosi-project/runtime/pkg/state/impl/inmem" "github.com/cosi-project/runtime/pkg/state/impl/namespaced" "github.com/opencontainers/runtime-spec/specs-go" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" "github.com/stretchr/testify/suite" "github.com/talos-systems/go-retry/retry" "inet.af/netaddr" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + k8sruntime "k8s.io/apimachinery/pkg/runtime" + "k8s.io/component-base/config/v1alpha1" + kubeletconfig "k8s.io/kubelet/config/v1beta1" k8sctrl "github.com/talos-systems/talos/internal/app/machined/pkg/controllers/k8s" "github.com/talos-systems/talos/pkg/logging" + "github.com/talos-systems/talos/pkg/machinery/constants" "github.com/talos-systems/talos/pkg/machinery/resources/k8s" ) @@ -173,6 +181,55 @@ func (suite *KubeletSpecSuite) TestReconcileWithExplicitNodeIP() { )) } +func (suite *KubeletSpecSuite) TestReconcileWithExtraConfig() { + cfg := k8s.NewKubeletConfig(k8s.NamespaceName, k8s.KubeletID) + cfg.TypedSpec().Image = "kubelet:v2.0.0" + cfg.TypedSpec().ClusterDNS = []string{"10.96.0.11"} + cfg.TypedSpec().ClusterDomain = "some.local" + cfg.TypedSpec().ExtraConfig = map[string]interface{}{ + "serverTLSBootstrap": true, + } + + suite.Require().NoError(suite.state.Create(suite.ctx, cfg)) + + nodename := k8s.NewNodename(k8s.NamespaceName, k8s.NodenameID) + nodename.TypedSpec().Nodename = "foo.com" + + suite.Require().NoError(suite.state.Create(suite.ctx, nodename)) + + nodeIP := k8s.NewNodeIP(k8s.NamespaceName, k8s.KubeletID) + nodeIP.TypedSpec().Addresses = []netaddr.IP{netaddr.MustParseIP("172.20.0.3")} + + suite.Require().NoError(suite.state.Create(suite.ctx, nodeIP)) + + suite.Assert().NoError(retry.Constant(10*time.Second, retry.WithUnits(100*time.Millisecond)).Retry( + func() error { + kubeletSpec, err := suite.state.Get(suite.ctx, resource.NewMetadata(k8s.NamespaceName, k8s.KubeletSpecType, k8s.KubeletID, resource.VersionUndefined)) + if err != nil { + if state.IsNotFoundError(err) { + return retry.ExpectedError(err) + } + + return err + } + + spec := kubeletSpec.(*k8s.KubeletSpec).TypedSpec() + + var kubeletConfiguration kubeletconfig.KubeletConfiguration + + if err := k8sruntime.DefaultUnstructuredConverter.FromUnstructured(spec.Config, &kubeletConfiguration); err != nil { + return err + } + + suite.Assert().Equal("/", kubeletConfiguration.CgroupRoot) + suite.Assert().Equal(cfg.TypedSpec().ClusterDomain, kubeletConfiguration.ClusterDomain) + suite.Assert().True(kubeletConfiguration.ServerTLSBootstrap) + + return nil + }, + )) +} + func (suite *KubeletSpecSuite) TearDownTest() { suite.T().Log("tear down") @@ -184,3 +241,102 @@ func (suite *KubeletSpecSuite) TearDownTest() { func TestKubeletSpecSuite(t *testing.T) { suite.Run(t, new(KubeletSpecSuite)) } + +func TestNewKubeletConfigurationFail(t *testing.T) { + for _, tt := range []struct { + name string + extraConfig map[string]interface{} + expectedErr string + }{ + { + name: "wrong fields", + extraConfig: map[string]interface{}{ + "API": "v1", + "foo": "bar", + "Port": "xyz", + }, + expectedErr: "error unmarshalling extra kubelet configuration: strict decoding error: unknown field \"API\", unknown field \"Port\", unknown field \"foo\"", + }, + { + name: "wrong field type", + extraConfig: map[string]interface{}{ + "oomScoreAdj": "v1", + }, + expectedErr: "error unmarshalling extra kubelet configuration: unrecognized type: int32", + }, + { + name: "not overridable", + extraConfig: map[string]interface{}{ + "oomScoreAdj": -300, + "port": 81, + "authentication": nil, + }, + expectedErr: "2 errors occurred:\n\t* field \"authentication\" can't be overridden\n\t* field \"port\" can't be overridden\n\n", + }, + } { + tt := tt + + t.Run(tt.name, func(t *testing.T) { + _, err := k8sctrl.NewKubeletConfiguration([]string{"10.96.0.10"}, "cluster.svc", tt.extraConfig) + require.Error(t, err) + + assert.EqualError(t, err, tt.expectedErr) + }) + } +} + +func TestNewKubeletConfigurationSuccess(t *testing.T) { + config, err := k8sctrl.NewKubeletConfiguration([]string{"10.0.0.5"}, "cluster.local", map[string]interface{}{ + "oomScoreAdj": -300, + "enableDebuggingHandlers": true, + }) + require.NoError(t, err) + + assert.Equal(t, &kubeletconfig.KubeletConfiguration{ + TypeMeta: metav1.TypeMeta{ + APIVersion: kubeletconfig.SchemeGroupVersion.String(), + Kind: "KubeletConfiguration", + }, + StaticPodPath: constants.ManifestsDirectory, + Port: constants.KubeletPort, + Authentication: kubeletconfig.KubeletAuthentication{ + X509: kubeletconfig.KubeletX509Authentication{ + ClientCAFile: constants.KubernetesCACert, + }, + Webhook: kubeletconfig.KubeletWebhookAuthentication{ + Enabled: pointer.ToBool(true), + }, + Anonymous: kubeletconfig.KubeletAnonymousAuthentication{ + Enabled: pointer.ToBool(false), + }, + }, + Authorization: kubeletconfig.KubeletAuthorization{ + Mode: kubeletconfig.KubeletAuthorizationModeWebhook, + }, + CgroupRoot: "/", + SystemCgroups: constants.CgroupSystem, + KubeletCgroups: constants.CgroupKubelet, + RotateCertificates: true, + ProtectKernelDefaults: true, + Address: "0.0.0.0", + OOMScoreAdj: pointer.ToInt32(-300), + ClusterDomain: "cluster.local", + ClusterDNS: []string{"10.0.0.5"}, + SerializeImagePulls: pointer.ToBool(false), + FailSwapOn: pointer.ToBool(false), + SystemReserved: map[string]string{ + "cpu": constants.KubeletSystemReservedCPU, + "memory": constants.KubeletSystemReservedMemory, + "pid": constants.KubeletSystemReservedPid, + "ephemeral-storage": constants.KubeletSystemReservedEphemeralStorage, + }, + Logging: v1alpha1.LoggingConfiguration{ + Format: "json", + }, + + StreamingConnectionIdleTimeout: metav1.Duration{Duration: 5 * time.Minute}, + TLSMinVersion: "VersionTLS13", + EnableDebuggingHandlers: pointer.ToBool(true), + }, + config) +} diff --git a/pkg/machinery/config/provider.go b/pkg/machinery/config/provider.go index 354fca4ca9..8f2ad717f1 100644 --- a/pkg/machinery/config/provider.go +++ b/pkg/machinery/config/provider.go @@ -275,6 +275,7 @@ type Kubelet interface { ClusterDNS() []string ExtraArgs() map[string]string ExtraMounts() []specs.Mount + ExtraConfig() map[string]interface{} RegisterWithFQDN() bool NodeIP() KubeletNodeIP } diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go index b3904e1d6f..8fdd5e9fca 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_provider.go @@ -335,6 +335,11 @@ func (k *KubeletConfig) ExtraMounts() []specs.Mount { return out } +// ExtraConfig implements the config.Provider interface. +func (k *KubeletConfig) ExtraConfig() map[string]interface{} { + return k.KubeletExtraConfig.Object +} + // RegisterWithFQDN implements the config.Provider interface. func (k *KubeletConfig) RegisterWithFQDN() bool { return k.KubeletRegisterWithFQDN diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go index 8a245361f4..e103f1b4c1 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_types.go @@ -480,6 +480,12 @@ metadata: }, } + kubeletExtraConfigExample = Unstructured{ + Object: map[string]interface{}{ + "serverTLSBootstrap": true, + }, + } + loggingEndpointExample1 = &Endpoint{ mustParseURL("udp://127.0.0.1:12345"), } @@ -979,6 +985,14 @@ type KubeletConfig struct { // - value: kubeletExtraMountsExample KubeletExtraMounts []ExtraMount `yaml:"extraMounts,omitempty"` // description: | + // The `extraConfig` field is used to provide kubelet configuration overrides. + // + // Some fields are not allowed to be overridden: authentication and authorization, cgroups + // configuration, ports, etc. + // examples: + // - value: kubeletExtraConfigExample + KubeletExtraConfig Unstructured `yaml:"extraConfig,omitempty"` + // description: | // The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration. // This is required in clouds like AWS. // values: diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go index 862dc6231a..55f5a2232d 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_types_doc.go @@ -551,7 +551,7 @@ func init() { FieldName: "kubelet", }, } - KubeletConfigDoc.Fields = make([]encoder.Doc, 6) + KubeletConfigDoc.Fields = make([]encoder.Doc, 7) KubeletConfigDoc.Fields[0].Name = "image" KubeletConfigDoc.Fields[0].Type = "string" KubeletConfigDoc.Fields[0].Note = "" @@ -582,24 +582,31 @@ func init() { KubeletConfigDoc.Fields[3].Comments[encoder.LineComment] = "The `extraMounts` field is used to add additional mounts to the kubelet container." KubeletConfigDoc.Fields[3].AddExample("", kubeletExtraMountsExample) - KubeletConfigDoc.Fields[4].Name = "registerWithFQDN" - KubeletConfigDoc.Fields[4].Type = "bool" + KubeletConfigDoc.Fields[4].Name = "extraConfig" + KubeletConfigDoc.Fields[4].Type = "Unstructured" KubeletConfigDoc.Fields[4].Note = "" - KubeletConfigDoc.Fields[4].Description = "The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration.\nThis is required in clouds like AWS." - KubeletConfigDoc.Fields[4].Comments[encoder.LineComment] = "The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration." - KubeletConfigDoc.Fields[4].Values = []string{ + KubeletConfigDoc.Fields[4].Description = "The `extraConfig` field is used to provide kubelet configuration overrides.\n\nSome fields are not allowed to be overridden: authentication and authorization, cgroups\nconfiguration, ports, etc." + KubeletConfigDoc.Fields[4].Comments[encoder.LineComment] = "The `extraConfig` field is used to provide kubelet configuration overrides." + + KubeletConfigDoc.Fields[4].AddExample("", kubeletExtraConfigExample) + KubeletConfigDoc.Fields[5].Name = "registerWithFQDN" + KubeletConfigDoc.Fields[5].Type = "bool" + KubeletConfigDoc.Fields[5].Note = "" + KubeletConfigDoc.Fields[5].Description = "The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration.\nThis is required in clouds like AWS." + KubeletConfigDoc.Fields[5].Comments[encoder.LineComment] = "The `registerWithFQDN` field is used to force kubelet to use the node FQDN for registration." + KubeletConfigDoc.Fields[5].Values = []string{ "true", "yes", "false", "no", } - KubeletConfigDoc.Fields[5].Name = "nodeIP" - KubeletConfigDoc.Fields[5].Type = "KubeletNodeIPConfig" - KubeletConfigDoc.Fields[5].Note = "" - KubeletConfigDoc.Fields[5].Description = "The `nodeIP` field is used to configure `--node-ip` flag for the kubelet.\nThis is used when a node has multiple addresses to choose from." - KubeletConfigDoc.Fields[5].Comments[encoder.LineComment] = "The `nodeIP` field is used to configure `--node-ip` flag for the kubelet." + KubeletConfigDoc.Fields[6].Name = "nodeIP" + KubeletConfigDoc.Fields[6].Type = "KubeletNodeIPConfig" + KubeletConfigDoc.Fields[6].Note = "" + KubeletConfigDoc.Fields[6].Description = "The `nodeIP` field is used to configure `--node-ip` flag for the kubelet.\nThis is used when a node has multiple addresses to choose from." + KubeletConfigDoc.Fields[6].Comments[encoder.LineComment] = "The `nodeIP` field is used to configure `--node-ip` flag for the kubelet." - KubeletConfigDoc.Fields[5].AddExample("", kubeletNodeIPExample) + KubeletConfigDoc.Fields[6].AddExample("", kubeletNodeIPExample) KubeletNodeIPConfigDoc.Type = "KubeletNodeIPConfig" KubeletNodeIPConfigDoc.Comments[encoder.LineComment] = "KubeletNodeIPConfig represents the kubelet node IP configuration." diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_validation.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_validation.go index abd4e32cd1..2ef96baa6d 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_validation.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_validation.go @@ -22,6 +22,7 @@ import ( "github.com/talos-systems/talos/pkg/machinery/config" "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1/machine" "github.com/talos-systems/talos/pkg/machinery/constants" + "github.com/talos-systems/talos/pkg/machinery/kubelet" "github.com/talos-systems/talos/pkg/machinery/nethelpers" ) @@ -733,5 +734,11 @@ func (k *KubeletConfig) Validate() ([]string, error) { } } + for _, field := range kubelet.ProtectedConfigurationFields { + if _, exists := k.KubeletExtraConfig.Object[field]; exists { + result = multierror.Append(result, fmt.Errorf("kubelet configuration field %q can't be overridden", field)) + } + } + return nil, result.ErrorOrNil() } diff --git a/pkg/machinery/config/types/v1alpha1/v1alpha1_validation_test.go b/pkg/machinery/config/types/v1alpha1/v1alpha1_validation_test.go index 59db5375eb..98d3c0ee26 100644 --- a/pkg/machinery/config/types/v1alpha1/v1alpha1_validation_test.go +++ b/pkg/machinery/config/types/v1alpha1/v1alpha1_validation_test.go @@ -926,6 +926,30 @@ func TestValidate(t *testing.T) { "\t* kubelet nodeIP subnet is not valid: \"[fd00::169:254:2:53]:344\"\n" + "\n", }, + { + name: "BadKubeletExtraConfig", + config: &v1alpha1.Config{ + ConfigVersion: "v1alpha1", + MachineConfig: &v1alpha1.MachineConfig{ + MachineType: "worker", + MachineKubelet: &v1alpha1.KubeletConfig{ + KubeletExtraConfig: v1alpha1.Unstructured{ + Object: map[string]interface{}{ + "port": 345, + }, + }, + }, + }, + ClusterConfig: &v1alpha1.ClusterConfig{ + ControlPlane: &v1alpha1.ControlPlaneConfig{ + Endpoint: &v1alpha1.Endpoint{ + endpointURL, + }, + }, + }, + }, + expectedError: "1 error occurred:\n\t* kubelet configuration field \"port\" can't be overridden\n\n", + }, } { test := test diff --git a/pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go b/pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go index fd67b158a2..84052927e0 100644 --- a/pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go +++ b/pkg/machinery/config/types/v1alpha1/zz_generated.deepcopy.go @@ -947,6 +947,7 @@ func (in *KubeletConfig) DeepCopyInto(out *KubeletConfig) { (*in)[i].DeepCopyInto(&(*out)[i]) } } + in.KubeletExtraConfig.DeepCopyInto(&out.KubeletExtraConfig) in.KubeletNodeIP.DeepCopyInto(&out.KubeletNodeIP) return } diff --git a/pkg/machinery/kubelet/kubelet.go b/pkg/machinery/kubelet/kubelet.go new file mode 100644 index 0000000000..65ddbace90 --- /dev/null +++ b/pkg/machinery/kubelet/kubelet.go @@ -0,0 +1,23 @@ +// This Source Code Form is subject to the terms of the Mozilla Public +// License, v. 2.0. If a copy of the MPL was not distributed with this +// file, You can obtain one at http://mozilla.org/MPL/2.0/. + +// Package kubelet defines Talos interface for the kubelet. +package kubelet + +// ProtectedConfigurationFields is a list of kubelet config fields that can't be overridden +// with the machine configuration. +var ProtectedConfigurationFields = []string{ + "apiVersion", + "authentication", + "authorization", + "cgroupRoot", + "kind", + "kubeletCgroups", + "port", + "protectKernelDefaults", + "resolvConf", + "rotateCertificates", + "systemCgroups", + "staticPodPath", +} diff --git a/pkg/machinery/resources/k8s/kubelet_config.go b/pkg/machinery/resources/k8s/kubelet_config.go index 19594719f3..37cdbe737b 100644 --- a/pkg/machinery/resources/k8s/kubelet_config.go +++ b/pkg/machinery/resources/k8s/kubelet_config.go @@ -10,6 +10,8 @@ import ( "github.com/cosi-project/runtime/pkg/resource" "github.com/cosi-project/runtime/pkg/resource/meta" "github.com/opencontainers/runtime-spec/specs-go" + + "github.com/talos-systems/talos/pkg/machinery/config/types/v1alpha1" ) // KubeletConfigType is type of KubeletConfig resource. @@ -26,12 +28,13 @@ type KubeletConfig struct { // KubeletConfigSpec holds the source of kubelet configuration. type KubeletConfigSpec struct { - Image string `yaml:"image"` - ClusterDNS []string `yaml:"clusterDNS"` - ClusterDomain string `yaml:"clusterDomain"` - ExtraArgs map[string]string `yaml:"extraArgs,omitempty"` - ExtraMounts []specs.Mount `yaml:"extraMounts,omitempty"` - CloudProviderExternal bool `yaml:"cloudProviderExternal"` + Image string `yaml:"image"` + ClusterDNS []string `yaml:"clusterDNS"` + ClusterDomain string `yaml:"clusterDomain"` + ExtraArgs map[string]string `yaml:"extraArgs,omitempty"` + ExtraMounts []specs.Mount `yaml:"extraMounts,omitempty"` + ExtraConfig map[string]interface{} `yaml:"extraConfig,omitempty"` + CloudProviderExternal bool `yaml:"cloudProviderExternal"` } // NewKubeletConfig initializes an empty KubeletConfig resource. @@ -68,6 +71,9 @@ func (r *KubeletConfig) DeepCopy() resource.Resource { extraArgs[k] = v } + extraConfig := &v1alpha1.Unstructured{Object: r.spec.ExtraConfig} + extraConfig = extraConfig.DeepCopy() + return &KubeletConfig{ md: r.md, spec: &KubeletConfigSpec{ @@ -76,6 +82,7 @@ func (r *KubeletConfig) DeepCopy() resource.Resource { ClusterDomain: r.spec.ClusterDomain, ExtraArgs: extraArgs, ExtraMounts: append([]specs.Mount(nil), r.spec.ExtraMounts...), + ExtraConfig: extraConfig.Object, CloudProviderExternal: r.spec.CloudProviderExternal, }, } diff --git a/website/content/docs/v0.15/Reference/configuration.md b/website/content/docs/v0.15/Reference/configuration.md index fd16006725..e8db7b27c0 100644 --- a/website/content/docs/v0.15/Reference/configuration.md +++ b/website/content/docs/v0.15/Reference/configuration.md @@ -336,6 +336,10 @@ kubelet: # - rshared # - rw + # # The `extraConfig` field is used to provide kubelet configuration overrides. + # extraConfig: + # serverTLSBootstrap: true + # # The `nodeIP` field is used to configure `--node-ip` flag for the kubelet. # nodeIP: # # The `validSubnets` field configures the networks to pick kubelet node IP from. @@ -1649,6 +1653,10 @@ extraArgs: # - rshared # - rw +# # The `extraConfig` field is used to provide kubelet configuration overrides. +# extraConfig: +# serverTLSBootstrap: true + # # The `nodeIP` field is used to configure `--node-ip` flag for the kubelet. # nodeIP: # # The `validSubnets` field configures the networks to pick kubelet node IP from. @@ -1756,6 +1764,32 @@ extraMounts: ``` + + +
+
+ +extraConfig Unstructured + +
+
+ +The `extraConfig` field is used to provide kubelet configuration overrides. + +Some fields are not allowed to be overridden: authentication and authorization, cgroups +configuration, ports, etc. + + + +Examples: + + +``` yaml +extraConfig: + serverTLSBootstrap: true +``` + +