Merge pull request spidernet-io#3842 from cyclinder/sysctl_cp_v0.9

spiderpool-agent: support to configure the sysctl config

charts/spiderpool/templates/daemonset.yaml

@@ -201,9 +201,14 @@ spec:
         {{- with .Values.spiderpoolAgent.extraEnv }}
         {{- toYaml . | nindent 8 }}
         {{- end }}
-        {{- with .Values.spiderpoolAgent.securityContext }}
+        {{- if or .Values.spiderpoolAgent.tuneSysctlConfig .Values.spiderpoolAgent.securityContext }}
         securityContext:
-        {{- toYaml . | nindent 8 }}
+          {{- if .Values.spiderpoolAgent.tuneSysctlConfig }}
+          privileged: true
+          {{- end }}
+          {{- with .Values.spiderpoolAgent.securityContext }}
+          {{- toYaml . | nindent 10 }}
+          {{- end }}
         {{- end }}
         volumeMounts:
         - name: config-path

charts/spiderpool/values.yaml

@@ -429,6 +429,9 @@ spiderpoolAgent:
       ## @param spiderpoolAgent.resources.requests.memory the memory requests of spiderpoolAgent pod
       memory: 128Mi
 
+  ## @param spiderpoolAgent.tuneSysctlConfig enable to set required sysctl on each node to run spiderpool. refer to [Spiderpool-agent](https://spidernet-io.github.io/spiderpool/dev/reference/spiderpool-agent/) for details
+  tuneSysctlConfig: true
+
   ## @param spiderpoolAgent.securityContext the security Context of spiderpoolAgent pod
   securityContext: {}
   # runAsUser: 0

cmd/spiderpool-agent/cmd/config.go

@@ -103,6 +103,7 @@ type Config struct {
 	EnableStatefulSet                 bool   `yaml:"enableStatefulSet"`
 	EnableKubevirtStaticIP            bool   `yaml:"enableKubevirtStaticIP"`
 	EnableSpiderSubnet                bool   `yaml:"enableSpiderSubnet"`
+	TuneSysctlConfig                  bool   `yaml:"tuneSysctlConfig"`
 	ClusterSubnetDefaultFlexibleIPNum int    `yaml:"clusterSubnetDefaultFlexibleIPNumber"`
 }
 

cmd/spiderpool-agent/cmd/daemon.go

@@ -17,6 +17,7 @@ import (
 
 	"github.com/google/gops/agent"
 	"github.com/grafana/pyroscope-go"
+	"go.uber.org/zap"
 	apiruntime "k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/client-go/rest"
 	"k8s.io/utils/ptr"
@@ -27,6 +28,7 @@ import (
 	"github.com/spidernet-io/spiderpool/pkg/kubevirtmanager"
 	"github.com/spidernet-io/spiderpool/pkg/logutils"
 	"github.com/spidernet-io/spiderpool/pkg/namespacemanager"
+	"github.com/spidernet-io/spiderpool/pkg/networking/sysctl"
 	"github.com/spidernet-io/spiderpool/pkg/nodemanager"
 	"github.com/spidernet-io/spiderpool/pkg/openapi"
 	"github.com/spidernet-io/spiderpool/pkg/podmanager"
@@ -73,6 +75,15 @@ func DaemonMain() {
 	}
 	logger.Sugar().Infof("Spiderpool-agent config: %+v", agentContext.Cfg)
 
+	// setup sysctls
+	if agentContext.Cfg.TuneSysctlConfig {
+		if err := sysctlConfig(agentContext.Cfg.EnableIPv4, agentContext.Cfg.EnableIPv6); err != nil {
+			logger.Sugar().Fatal(err)
+		}
+	} else {
+		logger.Sugar().Infof("setSysctlConfig is disabled.")
+	}
+
 	// Set up gops.
 	if agentContext.Cfg.GopsListenPort != "" {
 		address := "127.0.0.1:" + agentContext.Cfg.GopsListenPort
@@ -412,3 +423,26 @@ func initAgentServiceManagers(ctx context.Context) {
 		logger.Info("Feature SpiderSubnet is disabled")
 	}
 }
+
+// sysctlConfig set default sysctl configs,Notice: ignore not exist sysctl configs as
+// possible.
+func sysctlConfig(enableIPv4, enableIPv6 bool) error {
+	// setup default sysctl config
+	for _, sc := range sysctl.DefaultSysctlConfig {
+		if (enableIPv4 && sc.IsIPv4) || (enableIPv6 && sc.IsIPv6) {
+			logger.Info("Setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value))
+			err := sysctl.SetSysctl(sc.Name, sc.Value)
+			if err == nil {
+				logger.Debug("success to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value))
+				continue
+			}
+
+			if !errors.Is(err, os.ErrNotExist) {
+				logger.Error("failed to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value), zap.Error(err))
+				return err
+			}
+			logger.Warn("skip to setup sysctl", zap.String("sysctl", sc.Name), zap.String("value", sc.Value), zap.Error(err))
+		}
+	}
+	return nil
+}

docs/reference/spiderpool-agent.md

@@ -27,6 +27,29 @@ Run the spiderpool agent daemon.
 | SPIDERPOOL_IPPOOL_MAX_ALLOCATED_IPS             | 5000    | Max number of IP that a single IP pool can provide.                                             |
 | SPIDERPOOL_ENABLED_RELEASE_CONFLICT_IPS         | true    | Enable/disable release conflict IPs.                                                            |
 
+## spiderpool-agent helps set sysctl configs for each node
+
+To optimize the kernel network configuration of a node, spiderpool-agent will by default configure the following kernel parameters:
+
+| sysctl config | value | description |
+| -------------| ------| ------------|
+| net.ipv4.neigh.default.gc_thresh3 | 28160 | This is the hard maximum number of entries to keep in the ARP cache. The garbage collector will always run if there are more than this number of entries in the cache. for ipv4  |
+| net.ipv6.neigh.default.gc_thresh3 | 28160 | This is the hard maximum number of entries to keep in the ARP cache. The garbage collector will always run if there are more than this number of entries in the cache. for ipv6. Note: this is only avaliable in some low kernel version.|
+| net.ipv4.conf.all.arp_notify | 1 |  Generate gratuitous arp requests when device is brought up or hardware address changes.|
+| net.ipv4.conf.all.forwarding | 1 | enable ipv4 forwarding |
+| net.ipv4.conf.all.forwarding | 1 | enable ipv6 forwarding |
+
+To optimize the kernel network configuration of a node, spiderpool-agent configures some kernel parameters (such as a, etc.) by default. Some kernel parameters can only be set in certain kernel versions, so we will ignore the "kernel parameter does not exist" error when configure the kernel parameters.
+
+Users can edit the `spiderpoolAgent.securityContext` field of values.yaml in the chart before installing spiderpool to update the kernel parameters that need additional configuration, or manually edit spiderpool-agent daemonSet after installing Spiderpool, and then restart spiderpool-agent pods:
+
+Users can disable this feature by following command when installing Spiderpool:
+
+```
+helm install spiderpool -n kube-system --set global.tuneSysctlConfig=false
+```
+
+Or configure the spiderpool-conf configMap, set tuneSysctlConfig to false and restart the spiderpool-agent pods.
 
 ## spiderpool-agent shutdown
 

pkg/networking/sysctl/sysctl.go

@@ -5,11 +5,55 @@ package sysctl
 
 import (
 	"fmt"
+	"os"
+	"path/filepath"
+	"strings"
+
 	"github.com/containernetworking/plugins/pkg/ns"
 	"github.com/containernetworking/plugins/pkg/utils/sysctl"
-	"os"
 )
 
+// DefaultSysctlConfig is the default sysctl config for the node
+var DefaultSysctlConfig = []struct {
+	Name           string
+	Value          string
+	IsIPv4, IsIPv6 bool
+}{
+	// In order to avoid large-scale cluster arp_table overflow, resulting in
+	// pods not being able to communicate or pods not being able to start due
+	// to the inability to insert static arp table entries, it is necessary
+	// to appropriately increase and adjust its value. more details see:
+	// https://github.com/spidernet-io/spiderpool/issues/3587
+	{
+		Name: "net.ipv4.neigh.default.gc_thresh3",
+		// Assuming a node is full of underlay pods (110) and their subnet
+		// mask is 16 bits ( 2 ^ 8 = 256 IPs), the value is 110 * 256 = 28160
+		Value:  "28160",
+		IsIPv4: true,
+	},
+	{
+		// this sysctl may not be available at low kernel levels,
+		// so we'll ignore it at this point.
+		Name:   "net.ipv6.neigh.default.gc_thresh3",
+		Value:  "28160",
+		IsIPv6: true,
+	},
+	// send gratitous ARP when device or address change
+	{
+		Name:   "net.ipv4.conf.all.arp_notify",
+		Value:  "1",
+		IsIPv4: true,
+	}, {
+		Name:   "net.ipv4.conf.all.forwarding",
+		Value:  "1",
+		IsIPv4: true,
+	}, {
+		Name:   "net.ipv6.conf.all.forwarding",
+		Value:  "1",
+		IsIPv6: true,
+	},
+}
+
 // SysctlRPFilter set rp_filter value for host netns and specify netns
 func SysctlRPFilter(netns ns.NetNS, value int32) error {
 	var err error
@@ -77,3 +121,20 @@ func EnableIpv6Sysctl(netns ns.NetNS) error {
 	})
 	return err
 }
+
+func SetSysctl(sysConfig string, value string) error {
+	// sysConfig: net.ipv6.neigh.default.gc_thresh3
+	// to: net/ipv6/neigh/default/gc_thresh3
+	sysConfig = strings.ReplaceAll(sysConfig, ".", "/")
+
+	_, err := os.Stat(filepath.Join("/proc/sys", sysConfig))
+	if err != nil {
+		return err
+	}
+
+	if _, err := sysctl.Sysctl(sysConfig, value); err != nil {
+		return err
+	}
+
+	return nil
+}

test/scripts/install-multus.sh

@@ -377,7 +377,8 @@ EOF
 
 kubectl wait --for=condition=ready -l app.kubernetes.io/component=spiderpool-agent --timeout=100s pod -n kube-system --kubeconfig ${E2E_KUBECONFIG} || \
 ( kubectl get pod -n kube-system  --kubeconfig ${E2E_KUBECONFIG} ; \
- kubectl logs -n kube-system -l job-name=spiderpool-init --kubeconfig ${E2E_KUBECONFIG} ; exit 1 )
+  kubectl logs -n kube-system -l app.kubernetes.io/component=spiderpool-agent --kubeconfig ${E2E_KUBECONFIG} ; \
+  kubectl logs -n kube-system -l job-name=spiderpool-init --kubeconfig ${E2E_KUBECONFIG} ; exit 1 )
 
 Install::MultusCR
 Install::SpiderpoolCR