fix: bind HostDNS to 169.254.x link-local address

This is an attempt to fix many issues related with trying to use Service IP for host DNS. Fixes #9196 Signed-off-by: Andrey Smirnov <[email protected]>
siderolabs · Aug 19, 2024 · ee4290f · ee4290f
1 parent c312a46
commit ee4290f
Show file tree

Hide file tree

Showing 14 changed files with 349 additions and 476 deletions.
diff --git a/api/api.descriptors b/api/api.descriptors
diff --git a/api/resource/definitions/k8s/k8s.proto b/api/resource/definitions/k8s/k8s.proto
@@ -59,7 +59,6 @@ message BootstrapManifestsConfigSpec {
   repeated string flannel_extra_args = 16;
   string flannel_kube_service_host = 17;
   string flannel_kube_service_port = 18;
-  string service_host_dns_address = 19;
 }
 
 // ConfigStatusSpec describes status of rendered secrets.

diff --git a/hack/release.toml b/hack/release.toml
@@ -77,6 +77,10 @@ machine:
 ```
 
 Please note that on running cluster you will have to kill CoreDNS pods for this change to apply.
+
+The IP address used to forward DNS queries has changed to the fixed `169.254.116.108` address.
+For those upgrading from Talos 1.7 with `forwardKubeDNSToHost` enabled, the old Kubernetes service
+can be cleaned up with `kubectl delete -n kube-system service host-dns`.
 """
 
     [notes.lspci]

diff --git a/internal/app/machined/pkg/controllers/k8s/control_plane.go b/internal/app/machined/pkg/controllers/k8s/control_plane.go
@@ -14,10 +14,7 @@ import (
 	"github.com/cosi-project/runtime/pkg/controller"
 	"github.com/cosi-project/runtime/pkg/controller/generic"
 	"github.com/cosi-project/runtime/pkg/controller/generic/transform"
-	"github.com/cosi-project/runtime/pkg/safe"
-	"github.com/cosi-project/runtime/pkg/state"
 	"github.com/siderolabs/gen/optional"
-	"github.com/siderolabs/gen/value"
 	"github.com/siderolabs/gen/xslices"
 	"go.uber.org/zap"
 	v1 "k8s.io/api/core/v1"
@@ -215,8 +212,6 @@ func NewControlPlaneSchedulerController() *ControlPlaneSchedulerController {
 type ControlPlaneBootstrapManifestsController = transform.Controller[*config.MachineConfig, *k8s.BootstrapManifestsConfig]
 
 // NewControlPlaneBootstrapManifestsController instanciates the controller.
-//
-//nolint:gocyclo
 func NewControlPlaneBootstrapManifestsController() *ControlPlaneBootstrapManifestsController {
 	return transform.NewController(
 		transform.Settings[*config.MachineConfig, *k8s.BootstrapManifestsConfig]{
@@ -263,17 +258,6 @@ func NewControlPlaneBootstrapManifestsController() *ControlPlaneBootstrapManifes
 					server = cfgProvider.Cluster().Endpoint().String()
 				}
 
-				hostDNSCfg, err := safe.ReaderGetByID[*network.HostDNSConfig](ctx, r, network.HostDNSConfigID)
-				if err != nil && !state.IsNotFoundError(err) {
-					return fmt.Errorf("error getting host DNS config: %w", err)
-				}
-
-				var serviceHostDNSAddress string
-
-				if hostDNSCfg != nil && !value.IsZero(hostDNSCfg.TypedSpec().ServiceHostDNSAddress) {
-					serviceHostDNSAddress = hostDNSCfg.TypedSpec().ServiceHostDNSAddress.String()
-				}
-
 				*res.TypedSpec() = k8s.BootstrapManifestsConfigSpec{
 					Server:        server,
 					ClusterDomain: cfgProvider.Cluster().Network().DNSDomain(),
@@ -299,8 +283,6 @@ func NewControlPlaneBootstrapManifestsController() *ControlPlaneBootstrapManifes
 					PodSecurityPolicyEnabled: !cfgProvider.Cluster().APIServer().DisablePodSecurityPolicy(),
 
 					TalosAPIServiceEnabled: cfgProvider.Machine().Features().KubernetesTalosAPIAccess().Enabled(),
-
-					ServiceHostDNSAddress: serviceHostDNSAddress,
 				}
 
 				return nil

diff --git a/internal/app/machined/pkg/controllers/k8s/manifest.go b/internal/app/machined/pkg/controllers/k8s/manifest.go
@@ -169,8 +169,6 @@ func (ctrl *ManifestController) render(cfg k8s.BootstrapManifestsConfigSpec, scr
 		ApidPort int
 
 		TalosServiceAccount TalosServiceAccount
-
-		HostDNSAddr string
 	}{
 		BootstrapManifestsConfigSpec: cfg,
 		Secrets:                      scrt,
@@ -237,12 +235,6 @@ func (ctrl *ManifestController) render(cfg k8s.BootstrapManifestsConfigSpec, scr
 		)
 	}
 
-	if cfg.ServiceHostDNSAddress != "" {
-		defaultManifests = append(defaultManifests,
-			manifestDesc{"15-host-dns-service", talosHostDNSSvcTemplate},
-		)
-	}
-
 	manifests := make([]renderedManifest, len(defaultManifests))
 
 	for i := range defaultManifests {

diff --git a/internal/app/machined/pkg/controllers/k8s/templates.go b/internal/app/machined/pkg/controllers/k8s/templates.go
@@ -84,6 +84,3 @@ var flannelTemplate = string(flannel.Template)
 //
 //go:embed templates/talos-service-account-crd-template.yaml
 var talosServiceAccountCRDTemplate string
-
-//go:embed templates/talos-host-dns-svc-template.yaml
-var talosHostDNSSvcTemplate string
diff --git a/internal/app/machined/pkg/controllers/k8s/templates/talos-host-dns-svc-template.yaml b/internal/app/machined/pkg/controllers/k8s/templates/talos-host-dns-svc-template.yaml
diff --git a/internal/app/machined/pkg/controllers/network/hostdns_config.go b/internal/app/machined/pkg/controllers/network/hostdns_config.go
@@ -19,6 +19,7 @@ import (
 	"go.uber.org/zap"
 
 	talosconfig "github.com/siderolabs/talos/pkg/machinery/config"
+	"github.com/siderolabs/talos/pkg/machinery/constants"
 	"github.com/siderolabs/talos/pkg/machinery/nethelpers"
 	"github.com/siderolabs/talos/pkg/machinery/resources/config"
 	"github.com/siderolabs/talos/pkg/machinery/resources/network"
@@ -101,17 +102,7 @@ func (ctrl *HostDNSConfigController) Run(ctx context.Context, r controller.Runti
 			res.TypedSpec().ResolveMemberNames = cfgProvider.Machine().Features().HostDNS().ResolveMemberNames()
 
 			if cfgProvider.Machine().Features().HostDNS().ForwardKubeDNSToHost() {
-				serviceCIDRStr := cfgProvider.Cluster().Network().ServiceCIDRs()[0]
-
-				serviceCIDR, err := netip.ParsePrefix(serviceCIDRStr)
-				if err != nil {
-					return fmt.Errorf("error parsing service CIDR: %w", err)
-				}
-
-				newServiceAddr = serviceCIDR.Addr()
-				for range 9 {
-					newServiceAddr = newServiceAddr.Next()
-				}
+				newServiceAddr = netip.MustParseAddr(constants.HostDNSAddress)
 
 				res.TypedSpec().ListenAddresses = append(res.TypedSpec().ListenAddresses, netip.AddrPortFrom(newServiceAddr, 53))
 				res.TypedSpec().ServiceHostDNSAddress = newServiceAddr
@@ -131,7 +122,7 @@ func (ctrl *HostDNSConfigController) Run(ctx context.Context, r controller.Runti
 			}
 		}
 
-		if err = ctrl.cleanupLinkSpecs(
+		if err = ctrl.cleanupAddressSpecs(
 			ctx,
 			r,
 			func(id resource.ID) bool {
@@ -150,28 +141,28 @@ func (ctrl *HostDNSConfigController) Run(ctx context.Context, r controller.Runti
 	}
 }
 
-func (ctrl *HostDNSConfigController) cleanupLinkSpecs(ctx context.Context, r controller.Runtime, checkResource func(id resource.ID) bool, logger *zap.Logger) error {
+func (ctrl *HostDNSConfigController) cleanupAddressSpecs(ctx context.Context, r controller.Runtime, checkResource func(id resource.ID) bool, logger *zap.Logger) error {
 	list, err := safe.ReaderList[*network.AddressSpec](ctx, r, network.NewAddressSpec(network.ConfigNamespaceName, "").Metadata())
 	if err != nil {
 		return err
 	}
 
 	for iter := list.Iterator(); iter.Next(); {
-		link := iter.Value()
+		address := iter.Value()
 
-		if link.Metadata().Owner() != ctrl.Name() {
+		if address.Metadata().Owner() != ctrl.Name() {
 			continue
 		}
 
-		if checkResource(link.Metadata().ID()) {
+		if checkResource(address.Metadata().ID()) {
 			continue
 		}
 
-		if err = r.Destroy(ctx, link.Metadata()); err != nil && !state.IsNotFoundError(err) {
+		if err = r.Destroy(ctx, address.Metadata()); err != nil && !state.IsNotFoundError(err) {
 			return err
 		}
 
-		logger.Info("destroyed link spec", zap.String("link_id", link.Metadata().ID()))
+		logger.Info("destroyed address spec", zap.String("address_id", address.Metadata().ID()))
 	}
 
 	return nil