From 311efb667c5df10acf80f3b57b20559b1ec35751 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Tue, 21 May 2024 20:48:35 +0000 Subject: [PATCH 1/3] Add alternative ingress controller support ADR Signed-off-by: Brad Davidson --- docs/adrs/008-traefik-ingress.md | 29 +++++++++++++++++++++++++++++ 1 file changed, 29 insertions(+) create mode 100644 docs/adrs/008-traefik-ingress.md diff --git a/docs/adrs/008-traefik-ingress.md b/docs/adrs/008-traefik-ingress.md new file mode 100644 index 0000000000..2f128b02c5 --- /dev/null +++ b/docs/adrs/008-traefik-ingress.md @@ -0,0 +1,29 @@ +# Support for Alternative Ingress Controllers + +Date: 2024-05-21 + +## Status + +Accepted + +## Context + +RKE2 currently supports only a single ingress controller, ingress-nginx. +It has been requested RKE2 support alternative ingress controllers, similar to how RKE2 supports multiple CNIs. + +## Decision + +* A new --ingress-controller flag will be added; the default will be only `ingress-nginx` to preserve current behavior. +* All selected ingress controllers will be deployed to the cluster. +* The first selected ingress controller will be set as the default, via the `ingressclass.kubernetes.io/is-default-class` annotation + on the IngressClass resource. +* Any packaged ingress controllers not listed in the flag value will be disabled, similar to how inactive packaged CNIs are handled. +* RKE2 will package Traefik's HelmChart as a supported ingress controller, deploying as a Daemonset + ClusterIP Service + for parity with the `ingress-nginx` default configuration due to RKE2's lack of a default LoadBalancer controller. +* RKE2 will use mirrored upstream Traefik images; custom-rebuilt hardened-traefik images will not be provided or supported. + +## Consequences + +* We will add an additional packaged component and CLI flag for ingress controller selection. +* We will need to track updates to Traefik and the Traefik chart. +* QA will need additional resources to test the new ingress controllers. From 56281dc86cf664784ec2eab0e2f7c388e00f9f69 Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 22 May 2024 00:48:19 +0000 Subject: [PATCH 2/3] Add CLI flag for ingress controllers * Add new --ingress-controller CLI flag * Refactor --ingress-controller and --cni flags to use common helper for disabling all unused charts * Wire first ingress controller name into global.systemDefaultIngressClass chart variable Signed-off-by: Brad Davidson --- pkg/bootstrap/bootstrap.go | 7 +- pkg/cli/cmds/server.go | 125 ++++++++++++++++++++----------- pkg/pebinaryexecutor/pebinary.go | 15 ++-- pkg/podexecutor/staticpod.go | 26 +++---- pkg/rke2/rke2.go | 27 ++++++- pkg/rke2/rke2_linux.go | 6 ++ pkg/rke2/rke2_windows.go | 28 ++++--- 7 files changed, 154 insertions(+), 80 deletions(-) diff --git a/pkg/bootstrap/bootstrap.go b/pkg/bootstrap/bootstrap.go index c0e3bce280..dfc4db3d24 100644 --- a/pkg/bootstrap/bootstrap.go +++ b/pkg/bootstrap/bootstrap.go @@ -162,7 +162,7 @@ func Stage(resolver *images.Resolver, nodeConfig *daemonconfig.Node, cfg cmds.Ag // UpdateManifests copies the staged manifests into the server's manifests dir, and applies // cluster configuration values to any HelmChart manifests found in the manifests directory. -func UpdateManifests(resolver *images.Resolver, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { +func UpdateManifests(resolver *images.Resolver, ingressController string, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { ref, err := resolver.GetReference(images.Runtime) if err != nil { return err @@ -189,7 +189,7 @@ func UpdateManifests(resolver *images.Resolver, nodeConfig *daemonconfig.Node, c // Fix up HelmCharts to pass through configured values. // This needs to be done every time in order to sync values from the CLI - if err := setChartValues(manifestsDir, nodeConfig, cfg); err != nil { + if err := setChartValues(manifestsDir, ingressController, nodeConfig, cfg); err != nil { return errors.Wrap(err, "failed to rewrite HelmChart manifests to pass through CLI values") } return nil @@ -309,7 +309,7 @@ func copyFile(target, source string) error { // pass through settings to both the Helm job and the chart values. // NOTE: This will probably fail if any manifest contains multiple documents. This should // not matter for any of our packaged components, but may prevent this from working on user manifests. -func setChartValues(manifestsDir string, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { +func setChartValues(manifestsDir, ingressController string, nodeConfig *daemonconfig.Node, cfg cmds.Agent) error { chartValues := map[string]string{ "global.clusterCIDR": util.JoinIPNets(nodeConfig.AgentConfig.ClusterCIDRs), "global.clusterCIDRv4": util.JoinIP4Nets(nodeConfig.AgentConfig.ClusterCIDRs), @@ -318,6 +318,7 @@ func setChartValues(manifestsDir string, nodeConfig *daemonconfig.Node, cfg cmds "global.clusterDomain": nodeConfig.AgentConfig.ClusterDomain, "global.rke2DataDir": cfg.DataDir, "global.serviceCIDR": util.JoinIPNets(nodeConfig.AgentConfig.ServiceCIDRs), + "global.systemDefaultIngressClass": ingressController, "global.systemDefaultRegistry": nodeConfig.AgentConfig.SystemDefaultRegistry, "global.cattle.systemDefaultRegistry": nodeConfig.AgentConfig.SystemDefaultRegistry, } diff --git a/pkg/cli/cmds/server.go b/pkg/cli/cmds/server.go index c63a40ff8c..e184e46c67 100644 --- a/pkg/cli/cmds/server.go +++ b/pkg/cli/cmds/server.go @@ -1,6 +1,7 @@ package cmds import ( + "errors" "strings" "github.com/k3s-io/k3s/pkg/cli/cmds" @@ -9,6 +10,7 @@ import ( "github.com/rancher/wrangler/v3/pkg/slice" "github.com/sirupsen/logrus" "github.com/urfave/cli" + "k8s.io/apimachinery/pkg/util/sets" ) const ( @@ -16,22 +18,12 @@ const ( ) var ( - DisableItems = []string{"rke2-coredns", "rke2-ingress-nginx", "rke2-metrics-server"} - CNIItems = []string{"calico", "canal", "cilium", "flannel"} - config = rke2.Config{} serverFlag = []cli.Flag{ - &cli.StringSliceFlag{ - Name: "cni", - Usage: "(networking) CNI Plugins to deploy, one of none, " + strings.Join(CNIItems, ", ") + "; optionally with multus as the first value to enable the multus meta-plugin (default: canal)", - EnvVar: "RKE2_CNI", - }, - &cli.BoolFlag{ - Name: "enable-servicelb", - Usage: "(components) Enable rke2 default cloud controller manager's service controller", - EnvVar: "RKE2_ENABLE_SERVICELB", - }, + rke2.CNIFlag, + rke2.IngressControllerFlag, + rke2.ServiceLBFlag, } k3sServerBase = mustCmdFromK3S(cmds.NewServerCommand(ServerRun), K3SFlagSet{ @@ -81,7 +73,7 @@ var ( "kine-tls": dropFlag, "default-local-storage-path": dropFlag, "disable": { - Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: " + strings.Join(DisableItems, ", ") + ")", + Usage: "(components) Do not deploy packaged components and delete any deployed components (valid items: " + strings.Join(rke2.DisableItems, ", ") + ")", }, "disable-scheduler": copyFlag, "disable-cloud-controller": copyFlag, @@ -166,47 +158,90 @@ func ServerRun(clx *cli.Context) error { validateCloudProviderName(clx, Server) validateProfile(clx, Server) validateCNI(clx) + validateIngress(clx) return rke2.Server(clx, config) } +// validateCNI validates the CNI selection, and disables any un-selected CNI charts func validateCNI(clx *cli.Context) { - cnis := []string{} - for _, cni := range clx.StringSlice("cni") { - for _, v := range strings.Split(cni, ",") { - cnis = append(cnis, v) + disableExceptSelected(clx, rke2.CNIItems, rke2.CNIFlag, func(values cli.StringSlice) (cli.StringSlice, error) { + switch len(values) { + case 0: + values = append(values, "canal") + fallthrough + case 1: + if values[0] == "multus" { + return nil, errors.New("multus must be used alongside another primary cni selection") + } + clx.Set("disable", "rke2-multus") + case 2: + if values[0] == "multus" { + values = values[1:] + } else { + return nil, errors.New("may only provide multiple values if multus is the first value") + } + default: + return nil, errors.New("must specify 1 or 2 values") } - } + return values, nil + }) +} - switch len(cnis) { - case 0: - cnis = append(cnis, "canal") - fallthrough - case 1: - if cnis[0] == "multus" { - logrus.Fatal("invalid value provided for --cni flag: multus must be used alongside another primary cni selection") +// validateCNI validates the ingress controller selection, and disables any un-selected ingress controller charts +func validateIngress(clx *cli.Context) { + disableExceptSelected(clx, rke2.IngressItems, rke2.IngressControllerFlag, func(values cli.StringSlice) (cli.StringSlice, error) { + if len(values) == 0 { + values = append(values, "ingress-nginx") } - clx.Set("disable", "rke2-multus") - case 2: - if cnis[0] == "multus" { - cnis = cnis[1:] - } else { - logrus.Fatal("invalid values provided for --cni flag: may only provide multiple values if multus is the first value") + return values, nil + }) +} + +// disableExceptSelected takes a list of valid flag values, and a CLI StringSlice flag that holds the user's selected values. +// Selected values are split to support comma-separated lists, in addition to repeated use of the same flag. +// Once the list has been split, a validation function is called to allow for custom validation or defaulting of selected values. +// Finally, charts for any valid items not selected are added to the --disable list. +// A value of 'none' will cause all valid items to be disabled. +// Errors from the validation function, or selection of a value not in the valid list, will cause a fatal error to be logged. +func disableExceptSelected(clx *cli.Context, valid []string, flag *cli.StringSliceFlag, validateFunc func(cli.StringSlice) (cli.StringSlice, error)) { + // split comma-separated values + values := cli.StringSlice{} + if flag.Value != nil { + for _, value := range *flag.Value { + for _, v := range strings.Split(value, ",") { + values = append(values, v) + } } - default: - logrus.Fatal("invalid values provided for --cni flag: may not provide more than two values") } - switch { - case cnis[0] == "none": - fallthrough - case slice.ContainsString(CNIItems, cnis[0]): - for _, d := range CNIItems { - if cnis[0] != d { - clx.Set("disable", "rke2-"+d) - clx.Set("disable", "rke2-"+d+"-crd") - } + // validate the flag after splitting values + if v, err := validateFunc(values); err != nil { + logrus.Fatalf("Failed to validate --%s flag: %v", flag.Name, err) + } else { + flag.Value = &v + } + + // prepare a list of items to disable, based on all valid components. + // we have to use an intermediate set because the flag interface + // doesn't allow us to remove flag values once added. + disabledCharts := sets.Set[string]{} + for _, d := range valid { + disabledCharts.Insert("rke2-"+d, "rke2-"+d+"-crd") + } + + // re-enable components for any selected flag values + for _, d := range *flag.Value { + switch { + case d == "none": + break + case slice.ContainsString(valid, d): + disabledCharts.Delete("rke2-"+d, "rke2-"+d+"-crd") + default: + logrus.Fatalf("Invalid value %s for --%s flag: must be one of %s", d, flag.Name, strings.Join(valid, ",")) } - default: - logrus.Fatal("invalid value provided for --cni flag") + } + + for _, c := range disabledCharts.UnsortedList() { + clx.Set("disable", c) } } diff --git a/pkg/pebinaryexecutor/pebinary.go b/pkg/pebinaryexecutor/pebinary.go index ab1e9683b4..059b90a6e6 100644 --- a/pkg/pebinaryexecutor/pebinary.go +++ b/pkg/pebinaryexecutor/pebinary.go @@ -46,19 +46,20 @@ var ( ) type PEBinaryConfig struct { - ManifestsDir string - ImagesDir string - Resolver *images.Resolver + CNIPlugin win.CNIPlugin CloudProvider *CloudProviderConfig - CISMode bool + Resolver *images.Resolver + ManifestsDir string DataDir string AuditPolicyFile string KubeletPath string + CNIName string + ImagesDir string KubeConfigKubeProxy string + IngressController string + CISMode bool DisableETCD bool IsServer bool - CNIName string - CNIPlugin win.CNIPlugin } type CloudProviderConfig struct { @@ -105,7 +106,7 @@ func (p *PEBinaryConfig) Bootstrap(ctx context.Context, nodeConfig *config.Node, } if p.IsServer { - return bootstrap.UpdateManifests(p.Resolver, nodeConfig, cfg) + return bootstrap.UpdateManifests(p.Resolver, p.IngressController, nodeConfig, cfg) } restConfig, err := clientcmd.BuildConfigFromFlags("", nodeConfig.AgentConfig.KubeConfigK3sController) diff --git a/pkg/podexecutor/staticpod.go b/pkg/podexecutor/staticpod.go index ee74cfa154..b120489e5d 100644 --- a/pkg/podexecutor/staticpod.go +++ b/pkg/podexecutor/staticpod.go @@ -105,25 +105,25 @@ type ControlPlaneProbeConfs struct { } type StaticPodConfig struct { + Resolver *images.Resolver + stopKubelet context.CancelFunc + CloudProvider *CloudProviderConfig ControlPlaneResources - ControlPlaneProbeConfs + DataDir string + RuntimeEndpoint string + ManifestsDir string + IngressController string + ImagesDir string + AuditPolicyFile string + PSAConfigFile string + KubeletPath string ControlPlaneEnv ControlPlaneMounts - ManifestsDir string - ImagesDir string - Resolver *images.Resolver - CloudProvider *CloudProviderConfig - DataDir string - AuditPolicyFile string - PSAConfigFile string - KubeletPath string - RuntimeEndpoint string + ControlPlaneProbeConfs CISMode bool DisableETCD bool ExternalDatabase bool IsServer bool - - stopKubelet context.CancelFunc } type CloudProviderConfig struct { @@ -159,7 +159,7 @@ func (s *StaticPodConfig) Bootstrap(_ context.Context, nodeConfig *daemonconfig. return err } if s.IsServer { - return bootstrap.UpdateManifests(s.Resolver, nodeConfig, cfg) + return bootstrap.UpdateManifests(s.Resolver, s.IngressController, nodeConfig, cfg) } // Remove the kube-proxy static pod manifest before starting the agent. diff --git a/pkg/rke2/rke2.go b/pkg/rke2/rke2.go index 8d5c832743..aecec1df28 100644 --- a/pkg/rke2/rke2.go +++ b/pkg/rke2/rke2.go @@ -22,6 +22,7 @@ import ( "github.com/pkg/errors" "github.com/rancher/rke2/pkg/controllers/cisnetworkpolicy" "github.com/rancher/rke2/pkg/images" + "github.com/rancher/wrangler/v3/pkg/slice" "github.com/sirupsen/logrus" "github.com/urfave/cli" @@ -62,6 +63,30 @@ type ExtraEnv struct { CloudControllerManager cli.StringSlice } +var ( + DisableItems = []string{"rke2-coredns", "rke2-metrics-server", "rke2-snapshot-controller", "rke2-snapshot-controller-crd", "rke2-snapshot-validation-webhook"} + CNIItems = []string{"calico", "canal", "cilium", "flannel"} + IngressItems = []string{"ingress-nginx", "traefik"} + + CNIFlag = &cli.StringSliceFlag{ + Name: "cni", + Usage: "(networking) CNI Plugins to deploy, one of none, " + strings.Join(CNIItems, ", ") + "; optionally with multus as the first value to enable the multus meta-plugin (default: canal)", + EnvVar: "RKE2_CNI", + Value: &cli.StringSlice{}, + } + IngressControllerFlag = &cli.StringSliceFlag{ + Name: "ingress-controller", + Usage: "(networking) Ingress Controllers to deploy, one of none, " + strings.Join(IngressItems, ", ") + "; the first value will be set as the default ingress class (default: ingress-nginx)", + EnvVar: "RKE_INGRESS_CONTROLLER", + Value: &cli.StringSlice{}, + } + ServiceLBFlag = &cli.BoolFlag{ + Name: "enable-servicelb", + Usage: "(components) Enable rke2 default cloud controller manager's service controller", + EnvVar: "RKE2_ENABLE_SERVICELB", + } +) + // Valid CIS Profile versions const ( CISProfile123 = "cis-1.23" @@ -115,7 +140,7 @@ func Server(clx *cli.Context, cfg Config) error { var leaderControllers rawServer.CustomControllers - cnis := clx.StringSlice("cni") + cnis := *CNIFlag.Value if cisMode && (len(cnis) == 0 || slice.ContainsString(cnis, "canal")) { leaderControllers = append(leaderControllers, cisnetworkpolicy.Controller) } else { diff --git a/pkg/rke2/rke2_linux.go b/pkg/rke2/rke2_linux.go index 7c1bb14b79..e21b43f8c0 100644 --- a/pkg/rke2/rke2_linux.go +++ b/pkg/rke2/rke2_linux.go @@ -140,6 +140,11 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta containerRuntimeEndpoint = containerdSock } + var ingressControllerName string + if IngressControllerFlag.Value != nil && len(*IngressControllerFlag.Value) > 0 { + ingressControllerName = (*IngressControllerFlag.Value)[0] + } + return &podexecutor.StaticPodConfig{ Resolver: resolver, ImagesDir: agentImagesDir, @@ -154,6 +159,7 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*podexecutor.Sta DisableETCD: clx.Bool("disable-etcd"), ExternalDatabase: ExternalDatabase, IsServer: isServer, + IngressController: ingressControllerName, ControlPlaneResources: *controlPlaneResources, ControlPlaneProbeConfs: *controlPlaneProbeConfs, ControlPlaneEnv: *extraEnv, diff --git a/pkg/rke2/rke2_windows.go b/pkg/rke2/rke2_windows.go index 54cb196920..ba011275ff 100644 --- a/pkg/rke2/rke2_windows.go +++ b/pkg/rke2/rke2_windows.go @@ -58,17 +58,23 @@ func initExecutor(clx *cli.Context, cfg Config, isServer bool) (*pebinaryexecuto cfg.KubeletPath = "kubelet" } + var ingressControllerName string + if IngressControllerFlag.Value != nil && len(*IngressControllerFlag.Value) > 0 { + ingressControllerName = (*IngressControllerFlag.Value)[0] + } + return &pebinaryexecutor.PEBinaryConfig{ - Resolver: resolver, - ImagesDir: agentImagesDir, - ManifestsDir: agentManifestsDir, - CISMode: isCISMode(clx), - CloudProvider: cpConfig, - DataDir: dataDir, - AuditPolicyFile: clx.String("audit-policy-file"), - KubeletPath: cfg.KubeletPath, - DisableETCD: clx.Bool("disable-etcd"), - IsServer: isServer, - CNIName: "", + Resolver: resolver, + ImagesDir: agentImagesDir, + ManifestsDir: agentManifestsDir, + CISMode: isCISMode(clx), + CloudProvider: cpConfig, + DataDir: dataDir, + AuditPolicyFile: clx.String("audit-policy-file"), + KubeletPath: cfg.KubeletPath, + DisableETCD: clx.Bool("disable-etcd"), + IsServer: isServer, + IngressController: ingressControllerName, + CNIName: "", }, nil } From e56bb439688f0a34251559f3fd6b1ea00818479b Mon Sep 17 00:00:00 2001 From: Brad Davidson Date: Wed, 22 May 2024 22:44:21 +0000 Subject: [PATCH 3/3] Update ingress charts Signed-off-by: Brad Davidson --- charts/chart_versions.yaml | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/charts/chart_versions.yaml b/charts/chart_versions.yaml index 88a4974cb3..4638c51a2f 100644 --- a/charts/chart_versions.yaml +++ b/charts/chart_versions.yaml @@ -14,9 +14,15 @@ charts: - version: 1.29.002 filename: /charts/rke2-coredns.yaml bootstrap: true - - version: 4.10.101 + - version: 4.10.102 filename: /charts/rke2-ingress-nginx.yaml bootstrap: false + - version: 25.0.000 + filename: /charts/rke2-traefik.yaml + bootstrap: false + - version: 25.0.000 + filename: /charts/rke2-traefik-crd.yaml + bootstrap: false - version: 3.12.002 filename: /charts/rke2-metrics-server.yaml bootstrap: false