From 8f54016f9fcf01b24ffda5c469f11de9cd71fac4 Mon Sep 17 00:00:00 2001 From: Wenying Dong Date: Wed, 16 Sep 2020 03:32:31 +0000 Subject: [PATCH] [IPv6] Add support for dual-stack when using kube-proxy for Service (#1200) 1. Add a config item for IPv6 Serivce CIDR if using kube-proxy to provide Service functions. 2. Output IPv6 traffic from host gateway if its destination is a Service address. 3. Use ct_mark to identify Service traffic and output the reply packet to the host gateway to ensure the DNAT processing in iptables. --- build/yamls/antrea-aks.yml | 15 +- build/yamls/antrea-eks.yml | 15 +- build/yamls/antrea-gke.yml | 15 +- build/yamls/antrea-ipsec.yml | 15 +- build/yamls/antrea.yml | 15 +- build/yamls/base/conf/antrea-agent.conf | 9 +- cmd/antrea-agent/agent.go | 7 + cmd/antrea-agent/config.go | 5 + cmd/antrea-agent/options.go | 8 +- pkg/agent/agent.go | 7 +- pkg/agent/openflow/client.go | 18 +- pkg/agent/openflow/pipeline.go | 206 +++++++++++++------- pkg/agent/openflow/testing/mock_openflow.go | 2 +- pkg/agent/route/route_linux.go | 24 +-- pkg/agent/route/route_windows.go | 1 + pkg/agent/util/iptables/iptables.go | 72 ++++--- pkg/agent/util/net.go | 10 +- pkg/support/dump_others.go | 3 +- test/integration/agent/openflow_test.go | 97 ++++++--- 19 files changed, 366 insertions(+), 178 deletions(-) diff --git a/build/yamls/antrea-aks.yml b/build/yamls/antrea-aks.yml index 5e0b20d8e3d..04a4cbd65eb 100644 --- a/build/yamls/antrea-aks.yml +++ b/build/yamls/antrea-aks.yml @@ -1017,7 +1017,8 @@ data: featureGates: # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on - # Service traffic. + # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster + # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged. AntreaProxy: true # Enable traceflow which provides packet tracing feature to diagnose network issue. @@ -1062,6 +1063,12 @@ data: # for the GRE tunnel type. #enableIPSecTunnel: false + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack + # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by + # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. + # No default value for this field. + #serviceCIDRv6: + # Determines how traffic is encapsulated. It has the following options # encap(default): Inter-node Pod traffic is always encapsulated and Pod to outbound traffic is masqueraded. # noEncap: Inter-node Pod traffic is not encapsulated, but Pod to outbound traffic is masqueraded. @@ -1142,7 +1149,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-c5t898htbb + name: antrea-config-52dtdgg8kk namespace: kube-system --- apiVersion: v1 @@ -1249,7 +1256,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-c5t898htbb + name: antrea-config-52dtdgg8kk name: antrea-config - name: antrea-controller-tls secret: @@ -1480,7 +1487,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-c5t898htbb + name: antrea-config-52dtdgg8kk name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-eks.yml b/build/yamls/antrea-eks.yml index 6fa7c909319..fbd9c7cc4b8 100644 --- a/build/yamls/antrea-eks.yml +++ b/build/yamls/antrea-eks.yml @@ -1017,7 +1017,8 @@ data: featureGates: # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on - # Service traffic. + # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster + # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged. AntreaProxy: true # Enable traceflow which provides packet tracing feature to diagnose network issue. @@ -1062,6 +1063,12 @@ data: # for the GRE tunnel type. #enableIPSecTunnel: false + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack + # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by + # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. + # No default value for this field. + #serviceCIDRv6: + # Determines how traffic is encapsulated. It has the following options # encap(default): Inter-node Pod traffic is always encapsulated and Pod to outbound traffic is masqueraded. # noEncap: Inter-node Pod traffic is not encapsulated, but Pod to outbound traffic is masqueraded. @@ -1142,7 +1149,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-c5t898htbb + name: antrea-config-52dtdgg8kk namespace: kube-system --- apiVersion: v1 @@ -1249,7 +1256,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-c5t898htbb + name: antrea-config-52dtdgg8kk name: antrea-config - name: antrea-controller-tls secret: @@ -1482,7 +1489,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-c5t898htbb + name: antrea-config-52dtdgg8kk name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-gke.yml b/build/yamls/antrea-gke.yml index 6230a12303a..6abea5b8262 100644 --- a/build/yamls/antrea-gke.yml +++ b/build/yamls/antrea-gke.yml @@ -1017,7 +1017,8 @@ data: featureGates: # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on - # Service traffic. + # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster + # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged. AntreaProxy: true # Enable traceflow which provides packet tracing feature to diagnose network issue. @@ -1062,6 +1063,12 @@ data: # for the GRE tunnel type. #enableIPSecTunnel: false + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack + # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by + # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. + # No default value for this field. + #serviceCIDRv6: + # Determines how traffic is encapsulated. It has the following options # encap(default): Inter-node Pod traffic is always encapsulated and Pod to outbound traffic is masqueraded. # noEncap: Inter-node Pod traffic is not encapsulated, but Pod to outbound traffic is masqueraded. @@ -1142,7 +1149,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-t47k4gbhkb + name: antrea-config-c9th62mgbg namespace: kube-system --- apiVersion: v1 @@ -1249,7 +1256,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-t47k4gbhkb + name: antrea-config-c9th62mgbg name: antrea-config - name: antrea-controller-tls secret: @@ -1480,7 +1487,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-t47k4gbhkb + name: antrea-config-c9th62mgbg name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea-ipsec.yml b/build/yamls/antrea-ipsec.yml index b6b4fb38461..ece7760bffa 100644 --- a/build/yamls/antrea-ipsec.yml +++ b/build/yamls/antrea-ipsec.yml @@ -1017,7 +1017,8 @@ data: featureGates: # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on - # Service traffic. + # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster + # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged. # AntreaProxy: false # Enable traceflow which provides packet tracing feature to diagnose network issue. @@ -1067,6 +1068,12 @@ data: # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. #serviceCIDR: 10.96.0.0/12 + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack + # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by + # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. + # No default value for this field. + #serviceCIDRv6: + # Determines how traffic is encapsulated. It has the following options # encap(default): Inter-node Pod traffic is always encapsulated and Pod to outbound traffic is masqueraded. # noEncap: Inter-node Pod traffic is not encapsulated, but Pod to outbound traffic is masqueraded. @@ -1147,7 +1154,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-4k2khg5c24 + name: antrea-config-5ghkmg42fd namespace: kube-system --- apiVersion: v1 @@ -1263,7 +1270,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-4k2khg5c24 + name: antrea-config-5ghkmg42fd name: antrea-config - name: antrea-controller-tls secret: @@ -1529,7 +1536,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-4k2khg5c24 + name: antrea-config-5ghkmg42fd name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/antrea.yml b/build/yamls/antrea.yml index 5fef5f5eb23..076a2b23305 100644 --- a/build/yamls/antrea.yml +++ b/build/yamls/antrea.yml @@ -1017,7 +1017,8 @@ data: featureGates: # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on - # Service traffic. + # Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster + # before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged. # AntreaProxy: false # Enable traceflow which provides packet tracing feature to diagnose network issue. @@ -1067,6 +1068,12 @@ data: # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. #serviceCIDR: 10.96.0.0/12 + # ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack + # cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by + # --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. + # No default value for this field. + #serviceCIDRv6: + # Determines how traffic is encapsulated. It has the following options # encap(default): Inter-node Pod traffic is always encapsulated and Pod to outbound traffic is masqueraded. # noEncap: Inter-node Pod traffic is not encapsulated, but Pod to outbound traffic is masqueraded. @@ -1147,7 +1154,7 @@ metadata: annotations: {} labels: app: antrea - name: antrea-config-b8h2ghckdc + name: antrea-config-cc8ch94hd2 namespace: kube-system --- apiVersion: v1 @@ -1254,7 +1261,7 @@ spec: key: node-role.kubernetes.io/master volumes: - configMap: - name: antrea-config-b8h2ghckdc + name: antrea-config-cc8ch94hd2 name: antrea-config - name: antrea-controller-tls secret: @@ -1485,7 +1492,7 @@ spec: operator: Exists volumes: - configMap: - name: antrea-config-b8h2ghckdc + name: antrea-config-cc8ch94hd2 name: antrea-config - hostPath: path: /etc/cni/net.d diff --git a/build/yamls/base/conf/antrea-agent.conf b/build/yamls/base/conf/antrea-agent.conf index d1de76350b1..af618b00d1c 100644 --- a/build/yamls/base/conf/antrea-agent.conf +++ b/build/yamls/base/conf/antrea-agent.conf @@ -2,7 +2,8 @@ featureGates: # Enable AntreaProxy which provides ServiceLB for in-cluster Services in antrea-agent. # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on -# Service traffic. +# Service traffic. Antrea proxy doesn't support an IPv6 only cluster or a Dual-Stack cluster +# before PR #1102[https://github.com/vmware-tanzu/antrea/pull/1102] is merged. # AntreaProxy: false # Enable traceflow which provides packet tracing feature to diagnose network issue. @@ -52,6 +53,12 @@ featureGates: # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. #serviceCIDR: 10.96.0.0/12 +# ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack +# cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by +# --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. +# No default value for this field. +#serviceCIDRv6: + # Determines how traffic is encapsulated. It has the following options # encap(default): Inter-node Pod traffic is always encapsulated and Pod to outbound traffic is masqueraded. # noEncap: Inter-node Pod traffic is not encapsulated, but Pod to outbound traffic is masqueraded. diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 20e2f96179a..201470e74a7 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -96,6 +96,12 @@ func run(o *Options) error { features.DefaultFeatureGate.Enabled(features.AntreaPolicy)) _, serviceCIDRNet, _ := net.ParseCIDR(o.config.ServiceCIDR) + var serviceCIDRNetv6 *net.IPNet + // Todo: use FeatureGate to check if IPv6 is enabled and then read configuration item "ServiceCIDRv6". + if o.config.ServiceCIDRv6 != "" { + _, serviceCIDRNetv6, _ = net.ParseCIDR(o.config.ServiceCIDRv6) + } + _, encapMode := config.GetTrafficEncapModeFromStr(o.config.TrafficEncapMode) networkConfig := &config.NetworkConfig{ TunnelType: ovsconfig.TunnelType(o.config.TunnelType), @@ -121,6 +127,7 @@ func run(o *Options) error { o.config.HostGateway, o.config.DefaultMTU, serviceCIDRNet, + serviceCIDRNetv6, networkConfig, features.DefaultFeatureGate.Enabled(features.AntreaProxy)) err = agentInitializer.Initialize() diff --git a/cmd/antrea-agent/config.go b/cmd/antrea-agent/config.go index 2ed462a98fa..c7818caeb1e 100644 --- a/cmd/antrea-agent/config.go +++ b/cmd/antrea-agent/config.go @@ -68,6 +68,11 @@ type AgentConfig struct { // AntreaProxy is enabled, this parameter is not needed and will be ignored if provided. // Default is 10.96.0.0/12 ServiceCIDR string `yaml:"serviceCIDR,omitempty"` + // ClusterIP CIDR range for IPv6 Services. It's required when using kube-proxy to provide IPv6 Service in a Dual-Stack + // cluster or an IPv6 only cluster. The value should be the same as the configuration for kube-apiserver specified by + // --service-cluster-ip-range. When AntreaProxy is enabled, this parameter is not needed. + // No default value for this field. + ServiceCIDRv6 string `yaml:"serviceCIDRv6,omitempty"` // Whether or not to enable IPSec (ESP) encryption for Pod traffic across Nodes. IPSec encryption // is supported only for the GRE tunnel type. Antrea uses Preshared Key (PSK) for IKE // authentication. When IPSec tunnel is enabled, the PSK value must be passed to Antrea Agent diff --git a/cmd/antrea-agent/options.go b/cmd/antrea-agent/options.go index a92aff1b188..f754c634d39 100644 --- a/cmd/antrea-agent/options.go +++ b/cmd/antrea-agent/options.go @@ -84,7 +84,13 @@ func (o *Options) validate(args []string) error { // Validate service CIDR configuration _, _, err := net.ParseCIDR(o.config.ServiceCIDR) if err != nil { - return fmt.Errorf("service CIDR %s is invalid", o.config.ServiceCIDR) + return fmt.Errorf("Service CIDR %s is invalid", o.config.ServiceCIDR) + } + if o.config.ServiceCIDRv6 != "" { + _, _, err := net.ParseCIDR(o.config.ServiceCIDRv6) + if err != nil { + return fmt.Errorf("Service CIDR v6 %s is invalid", o.config.ServiceCIDRv6) + } } if o.config.TunnelType != ovsconfig.VXLANTunnel && o.config.TunnelType != ovsconfig.GeneveTunnel && o.config.TunnelType != ovsconfig.GRETunnel && o.config.TunnelType != ovsconfig.STTTunnel { diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 1015b7993ad..f1e19699f4b 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -64,6 +64,7 @@ type Initializer struct { hostGateway string // name of gateway port on the OVS bridge mtu int serviceCIDR *net.IPNet // K8s Service ClusterIP CIDR + serviceCIDRv6 *net.IPNet // K8s Service ClusterIP CIDR in IPv6 networkConfig *config.NetworkConfig nodeConfig *config.NodeConfig enableProxy bool @@ -79,6 +80,7 @@ func NewInitializer( hostGateway string, mtu int, serviceCIDR *net.IPNet, + serviceCIDRv6 *net.IPNet, networkConfig *config.NetworkConfig, enableProxy bool) *Initializer { return &Initializer{ @@ -91,6 +93,7 @@ func NewInitializer( hostGateway: hostGateway, mtu: mtu, serviceCIDR: serviceCIDR, + serviceCIDRv6: serviceCIDRv6, networkConfig: networkConfig, enableProxy: enableProxy, } @@ -313,8 +316,8 @@ func (i *Initializer) initOpenFlowPipeline() error { // from local Pods to any Service address can be forwarded to the host gateway interface // correctly. Otherwise packets might be dropped by egress rules before they are DNATed to // backend Pods. - if err := i.ofClient.InstallClusterServiceCIDRFlows(i.serviceCIDR, gatewayOFPort); err != nil { - klog.Errorf("Failed to setup openflow entries for Cluster Service CIDR %s: %v", i.serviceCIDR, err) + if err := i.ofClient.InstallClusterServiceCIDRFlows([]*net.IPNet{i.serviceCIDR, i.serviceCIDRv6}, gatewayOFPort); err != nil { + klog.Errorf("Failed to setup OpenFlow entries for Service CIDRs: %v", err) return err } } else { diff --git a/pkg/agent/openflow/client.go b/pkg/agent/openflow/client.go index a403757eeae..171b1db3104 100644 --- a/pkg/agent/openflow/client.go +++ b/pkg/agent/openflow/client.go @@ -52,7 +52,7 @@ type Client interface { // InstallClusterServiceCIDRFlows sets up the appropriate flows so that traffic can reach // the different Services running in the Cluster. This method needs to be invoked once with // the Cluster Service CIDR as a parameter. - InstallClusterServiceCIDRFlows(serviceNet *net.IPNet, gatewayOFPort uint32) error + InstallClusterServiceCIDRFlows(serviceNets []*net.IPNet, gatewayOFPort uint32) error // InstallClusterServiceFlows sets up the appropriate flows so that traffic can reach // the different Services running in the Cluster. This method needs to be invoked once. @@ -471,23 +471,23 @@ func (c *client) InstallClusterServiceFlows() error { return nil } -func (c *client) InstallClusterServiceCIDRFlows(serviceNet *net.IPNet, gatewayOFPort uint32) error { - flow := c.serviceCIDRDNATFlow(serviceNet, gatewayOFPort) - if err := c.ofEntryOperations.Add(flow); err != nil { + +func (c *client) InstallClusterServiceCIDRFlows(serviceNets []*net.IPNet, gatewayOFPort uint32) error { + flows := c.serviceCIDRDNATFlows(serviceNets, gatewayOFPort) + if err := c.ofEntryOperations.AddAll(flows); err != nil { return err } - c.defaultServiceFlows = []binding.Flow{flow} + c.defaultServiceFlows = flows return nil } func (c *client) InstallGatewayFlows(gatewayAddrs []net.IP, gatewayMAC net.HardwareAddr, gatewayOFPort uint32) error { flows := []binding.Flow{ c.gatewayClassifierFlow(gatewayOFPort, cookie.Default), - c.ctRewriteDstMACFlow(gatewayMAC, cookie.Default), c.l2ForwardCalcFlow(gatewayMAC, gatewayOFPort, cookie.Default), } - hasIPv6Addr := util.ContainIPv6Addr(gatewayAddrs) - flows = append(flows, c.gatewayIPSpoofGuardFlows(gatewayOFPort, hasIPv6Addr, cookie.Default)...) + hasV4, hasV6 := util.CheckAddressFamilies(gatewayAddrs) + flows = append(flows, c.gatewayIPSpoofGuardFlows(gatewayOFPort, hasV4, hasV6, cookie.Default)...) // Add ARP SpoofGuard flow for local gateway interface. gwIPv4 := util.GetIPv4Addr(gatewayAddrs) @@ -496,7 +496,7 @@ func (c *client) InstallGatewayFlows(gatewayAddrs []net.IP, gatewayMAC net.Hardw } // Add flow to ensure the liveness check packet could be forwarded correctly. flows = append(flows, c.localProbeFlow(gatewayAddrs, cookie.Default)...) - + flows = append(flows, c.ctRewriteDstMACFlow(gatewayMAC, hasV4, hasV6, cookie.Default)...) // In NoEncap , no traffic from tunnel port if c.encapMode.SupportsEncap() { flows = append(flows, c.l3ToGatewayFlow(gatewayAddrs, gatewayMAC, cookie.Default)...) diff --git a/pkg/agent/openflow/pipeline.go b/pkg/agent/openflow/pipeline.go index 39a167e31c2..1882e4e04cd 100644 --- a/pkg/agent/openflow/pipeline.go +++ b/pkg/agent/openflow/pipeline.go @@ -520,7 +520,7 @@ func (c *client) connectionTrackFlows(category cookie.Category) []binding.Flow { connectionTrackTable := c.pipeline[conntrackTable] connectionTrackStateTable := c.pipeline[conntrackStateTable] connectionTrackCommitTable := c.pipeline[conntrackCommitTable] - var flows []binding.Flow + flows := c.conntrackBasicFlows(category) if c.enableProxy { flows = append(flows, // Replace the default flow with multiple resubmits actions. @@ -543,52 +543,103 @@ func (c *client) connectionTrackFlows(category cookie.Category) []binding.Flow { Done(), ) } else { + flows = append(flows, c.kubeProxyFlows(category)...) + } + + // TODO: following flows should move to function "kubeProxyFlows". Since another PR(#1198) is trying + // to polish the relevant logic, code refactoring is needed after that PR is merged. + if c.nodeConfig.PodIPv4CIDR != nil { + flows = append(flows, + connectionTrackStateTable.BuildFlow(priorityHigh).MatchProtocol(binding.ProtocolIP). + MatchRegRange(int(marksReg), markTrafficFromGateway, binding.Range{0, 15}). + MatchCTMark(gatewayCTMark). + MatchCTStateNew(false).MatchCTStateTrk(true). + Action().GotoTable(connectionTrackStateTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + connectionTrackCommitTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). + MatchRegRange(int(marksReg), markTrafficFromGateway, binding.Range{0, 15}). + MatchCTStateNew(true).MatchCTStateTrk(true). + Action().CT(true, connectionTrackCommitTable.GetNext(), CtZone).LoadToMark(gatewayCTMark).CTDone(). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + ) + } + if c.nodeConfig.PodIPv6CIDR != nil { + flows = append(flows, + connectionTrackStateTable.BuildFlow(priorityHigh).MatchProtocol(binding.ProtocolIPv6). + MatchRegRange(int(marksReg), markTrafficFromGateway, binding.Range{0, 15}). + MatchCTMark(gatewayCTMark). + MatchCTStateNew(false).MatchCTStateTrk(true). + Action().GotoTable(connectionTrackStateTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + connectionTrackCommitTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6). + MatchRegRange(int(marksReg), markTrafficFromGateway, binding.Range{0, 15}). + MatchCTStateNew(true).MatchCTStateTrk(true). + Action().CT(true, connectionTrackCommitTable.GetNext(), CtZoneV6).LoadToMark(gatewayCTMark).CTDone(). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + ) + } + return flows +} + +func (c *client) conntrackBasicFlows(category cookie.Category) []binding.Flow { + connectionTrackStateTable := c.pipeline[conntrackStateTable] + connectionTrackCommitTable := c.pipeline[conntrackCommitTable] + var flows []binding.Flow + if c.nodeConfig.PodIPv4CIDR != nil { + flows = append(flows, + connectionTrackStateTable.BuildFlow(priorityLow).MatchProtocol(binding.ProtocolIP). + MatchCTStateInv(true).MatchCTStateTrk(true). + Action().Drop(). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + connectionTrackCommitTable.BuildFlow(priorityLow).MatchProtocol(binding.ProtocolIP). + MatchCTStateNew(true).MatchCTStateTrk(true). + Action().CT(true, connectionTrackCommitTable.GetNext(), CtZone).CTDone(). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + ) + } + if c.nodeConfig.PodIPv6CIDR != nil { + flows = append(flows, + connectionTrackStateTable.BuildFlow(priorityLow).MatchProtocol(binding.ProtocolIPv6). + MatchCTStateInv(true).MatchCTStateTrk(true). + Action().Drop(). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + connectionTrackCommitTable.BuildFlow(priorityLow).MatchProtocol(binding.ProtocolIPv6). + MatchCTStateNew(true).MatchCTStateTrk(true). + Action().CT(true, connectionTrackCommitTable.GetNext(), CtZoneV6).CTDone(). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done(), + ) + } + return flows +} + +func (c *client) kubeProxyFlows(category cookie.Category) []binding.Flow { + connectionTrackTable := c.pipeline[conntrackTable] + var flows []binding.Flow + if c.nodeConfig.PodIPv4CIDR != nil { flows = append(flows, connectionTrackTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). Action().CT(false, connectionTrackTable.GetNext(), CtZone).CTDone(). Cookie(c.cookieAllocator.Request(category).Raw()). Done(), + ) + } + if c.nodeConfig.PodIPv6CIDR != nil { + flows = append(flows, connectionTrackTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6). Action().CT(false, connectionTrackTable.GetNext(), CtZoneV6).CTDone(). Cookie(c.cookieAllocator.Request(category).Raw()). Done(), ) } - return append(flows, - connectionTrackStateTable.BuildFlow(priorityHigh).MatchProtocol(binding.ProtocolIP). - MatchRegRange(int(marksReg), markTrafficFromGateway, binding.Range{0, 15}). - MatchCTMark(gatewayCTMark). - MatchCTStateNew(false).MatchCTStateTrk(true). - Action().GotoTable(connectionTrackStateTable.GetNext()). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - connectionTrackStateTable.BuildFlow(priorityLow).MatchProtocol(binding.ProtocolIP). - MatchCTStateInv(true).MatchCTStateTrk(true). - Action().Drop(). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - connectionTrackStateTable.BuildFlow(priorityLow).MatchProtocol(binding.ProtocolIPv6). - MatchCTStateInv(true).MatchCTStateTrk(true). - Action().Drop(). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - connectionTrackCommitTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). - MatchRegRange(int(marksReg), markTrafficFromGateway, binding.Range{0, 15}). - MatchCTStateNew(true).MatchCTStateTrk(true). - Action().CT(true, connectionTrackCommitTable.GetNext(), CtZone).LoadToMark(gatewayCTMark).CTDone(). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - connectionTrackCommitTable.BuildFlow(priorityLow).MatchProtocol(binding.ProtocolIP). - MatchCTStateNew(true).MatchCTStateTrk(true). - Action().CT(true, connectionTrackCommitTable.GetNext(), CtZone).CTDone(). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - connectionTrackCommitTable.BuildFlow(priorityLow).MatchProtocol(binding.ProtocolIPv6). - MatchCTStateNew(true).MatchCTStateTrk(true). - Action().CT(true, connectionTrackCommitTable.GetNext(), CtZoneV6).CTDone(). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done(), - ) + return flows } // TODO: Use DuplicateToBuilder or integrate this function into original one to avoid unexpected difference. @@ -612,16 +663,29 @@ func (c *client) traceflowConnectionTrackFlows(dataplaneTag uint8, category cook } // ctRewriteDstMACFlow rewrites the destination MAC with local host gateway MAC if the packets has set ct_mark but not sent from the host gateway. -func (c *client) ctRewriteDstMACFlow(gatewayMAC net.HardwareAddr, category cookie.Category) binding.Flow { +func (c *client) ctRewriteDstMACFlow(gatewayMAC net.HardwareAddr, hasV4Addr, hasV6Addr bool, category cookie.Category) []binding.Flow { connectionTrackStateTable := c.pipeline[conntrackStateTable] macData, _ := strconv.ParseUint(strings.Replace(gatewayMAC.String(), ":", "", -1), 16, 64) - return connectionTrackStateTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). - MatchCTMark(gatewayCTMark). - MatchCTStateNew(false).MatchCTStateTrk(true). - Action().LoadRange(binding.NxmFieldDstMAC, macData, binding.Range{0, 47}). - Action().GotoTable(connectionTrackStateTable.GetNext()). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done() + var flows []binding.Flow + if hasV4Addr { + flows = append(flows, connectionTrackStateTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). + MatchCTMark(gatewayCTMark). + MatchCTStateNew(false).MatchCTStateTrk(true). + Action().LoadRange(binding.NxmFieldDstMAC, macData, binding.Range{0, 47}). + Action().GotoTable(connectionTrackStateTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } + if hasV6Addr { + flows = append(flows, connectionTrackStateTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6). + MatchCTMark(gatewayCTMark). + MatchCTStateNew(false).MatchCTStateTrk(true). + Action().LoadRange(binding.NxmFieldDstMAC, macData, binding.Range{0, 47}). + Action().GotoTable(connectionTrackStateTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } + return flows } // serviceLBBypassFlow makes packets that belong to a tracked connection bypass @@ -699,7 +763,7 @@ func (c *client) l3FlowsToPod(localGatewayMAC net.HardwareAddr, podInterfaceIPs l3FwdTable := c.pipeline[l3ForwardingTable] var flows []binding.Flow for _, ip := range podInterfaceIPs { - ipProtocol := parseIPProtocol(ip) + ipProtocol := getIPProtocol(ip) flowBuilder := l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol) if c.enableProxy { flowBuilder = flowBuilder.MatchRegRange(int(marksReg), macRewriteMark, macRewriteMarkRange) @@ -724,7 +788,7 @@ func (c *client) l3ToPodFlow(podInterfaceIPs []net.IP, podInterfaceMAC net.Hardw l3FwdTable := c.pipeline[l3ForwardingTable] var flows []binding.Flow for _, ip := range podInterfaceIPs { - ipProtocol := parseIPProtocol(ip) + ipProtocol := getIPProtocol(ip) flows = append(flows, l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol). MatchDstIP(ip). Action().SetDstMAC(podInterfaceMAC). @@ -753,7 +817,7 @@ func (c *client) l3ToGatewayFlow(localGatewayIPs []net.IP, localGatewayMAC net.H l3FwdTable := c.pipeline[l3ForwardingTable] var flows []binding.Flow for _, ip := range localGatewayIPs { - ipProtocol := parseIPProtocol(ip) + ipProtocol := getIPProtocol(ip) flows = append(flows, l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol). MatchDstMAC(globalVirtualMAC). MatchDstIP(ip). @@ -772,7 +836,7 @@ func (c *client) l3FwdFlowToRemote( tunnelPeer net.IP, tunOFPort uint32, category cookie.Category) binding.Flow { - ipProto := parseIPProtocol(peerSubnet.IP) + ipProto := getIPProtocol(peerSubnet.IP) return c.pipeline[l3ForwardingTable].BuildFlow(priorityNormal).MatchProtocol(ipProto). MatchDstIPNet(peerSubnet). Action().DecTTL(). @@ -797,7 +861,7 @@ func (c *client) l3FwdFlowToRemoteViaGW( localGatewayMAC net.HardwareAddr, peerSubnet net.IPNet, category cookie.Category) binding.Flow { - ipProto := parseIPProtocol(peerSubnet.IP) + ipProto := getIPProtocol(peerSubnet.IP) l3FwdTable := c.pipeline[l3ForwardingTable] return l3FwdTable.BuildFlow(priorityNormal).MatchProtocol(ipProto). MatchDstIPNet(peerSubnet). @@ -852,7 +916,7 @@ func (c *client) podIPSpoofGuardFlow(ifIPs []net.IP, ifMAC net.HardwareAddr, ifO ipSpoofGuardTable := ipPipeline[spoofGuardTable] var flows []binding.Flow for _, ifIP := range ifIPs { - ipProtocol := parseIPProtocol(ifIP) + ipProtocol := getIPProtocol(ifIP) if ipProtocol == binding.ProtocolIP { flows = append(flows, ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(ipProtocol). MatchInPort(ifOFPort). @@ -874,7 +938,7 @@ func (c *client) podIPSpoofGuardFlow(ifIPs []net.IP, ifMAC net.HardwareAddr, ifO return flows } -func parseIPProtocol(ip net.IP) binding.Protocol { +func getIPProtocol(ip net.IP) binding.Protocol { var ipProtocol binding.Protocol if ip.To4() != nil { ipProtocol = binding.ProtocolIP @@ -932,15 +996,17 @@ func (c *client) sessionAffinityReselectFlow() binding.Flow { } // gatewayIPSpoofGuardFlow generates the flow to skip spoof guard checking for traffic sent from gateway interface. -func (c *client) gatewayIPSpoofGuardFlows(gatewayOFPort uint32, hasIPv6Addr bool, category cookie.Category) []binding.Flow { +func (c *client) gatewayIPSpoofGuardFlows(gatewayOFPort uint32, hasIPv4Addr, hasIPv6Addr bool, category cookie.Category) []binding.Flow { ipPipeline := c.pipeline ipSpoofGuardTable := ipPipeline[spoofGuardTable] var flows []binding.Flow - flows = append(flows, ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). - MatchInPort(gatewayOFPort). - Action().GotoTable(ipSpoofGuardTable.GetNext()). - Cookie(c.cookieAllocator.Request(category).Raw()). - Done()) + if hasIPv4Addr { + flows = append(flows, ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). + MatchInPort(gatewayOFPort). + Action().GotoTable(ipSpoofGuardTable.GetNext()). + Cookie(c.cookieAllocator.Request(category).Raw()). + Done()) + } if hasIPv6Addr { flows = append(flows, ipSpoofGuardTable.BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIPv6). MatchInPort(gatewayOFPort). @@ -952,14 +1018,22 @@ func (c *client) gatewayIPSpoofGuardFlows(gatewayOFPort uint32, hasIPv6Addr bool } // serviceCIDRDNATFlow generates flows to match dst IP in service CIDR and output to host gateway interface directly. -func (c *client) serviceCIDRDNATFlow(serviceCIDR *net.IPNet, gatewayOFPort uint32) binding.Flow { - return c.pipeline[dnatTable].BuildFlow(priorityNormal).MatchProtocol(binding.ProtocolIP). - MatchDstIPNet(*serviceCIDR). - Action().LoadRegRange(int(portCacheReg), gatewayOFPort, ofPortRegRange). - Action().LoadRegRange(int(marksReg), portFoundMark, ofPortMarkRange). - Action().GotoTable(conntrackCommitTable). - Cookie(c.cookieAllocator.Request(cookie.Service).Raw()). - Done() + +func (c *client) serviceCIDRDNATFlows(serviceCIDRs []*net.IPNet, gatewayOFPort uint32) []binding.Flow { + var flows []binding.Flow + for _, serviceCIDR := range serviceCIDRs { + if serviceCIDR != nil { + ipProto := getIPProtocol(serviceCIDR.IP) + flows = append(flows, c.pipeline[dnatTable].BuildFlow(priorityNormal).MatchProtocol(ipProto). + MatchDstIPNet(*serviceCIDR). + Action().LoadRegRange(int(portCacheReg), gatewayOFPort, ofPortRegRange). + Action().LoadRegRange(int(marksReg), portFoundMark, ofPortMarkRange). + Action().GotoTable(conntrackCommitTable). + Cookie(c.cookieAllocator.Request(cookie.Service).Raw()). + Done()) + } + } + return flows } // serviceNeedLBFlow generates flows to mark packets as LB needed. @@ -1245,7 +1319,7 @@ func (c *client) defaultDropFlow(tableID binding.TableIDType, matchKey int, matc func (c *client) localProbeFlow(localGatewayIPs []net.IP, category cookie.Category) []binding.Flow { var flows []binding.Flow for _, ip := range localGatewayIPs { - ipProtocol := parseIPProtocol(ip) + ipProtocol := getIPProtocol(ip) flows = append(flows, c.pipeline[IngressRuleTable].BuildFlow(priorityHigh). MatchProtocol(ipProtocol). MatchSrcIP(ip). diff --git a/pkg/agent/openflow/testing/mock_openflow.go b/pkg/agent/openflow/testing/mock_openflow.go index 458770b9b37..996e7df15e5 100644 --- a/pkg/agent/openflow/testing/mock_openflow.go +++ b/pkg/agent/openflow/testing/mock_openflow.go @@ -238,7 +238,7 @@ func (mr *MockClientMockRecorder) InstallBridgeUplinkFlows(arg0, arg1 interface{ } // InstallClusterServiceCIDRFlows mocks base method -func (m *MockClient) InstallClusterServiceCIDRFlows(arg0 *net.IPNet, arg1 uint32) error { +func (m *MockClient) InstallClusterServiceCIDRFlows(arg0 []*net.IPNet, arg1 uint32) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "InstallClusterServiceCIDRFlows", arg0, arg1) ret0, _ := ret[0].(error) diff --git a/pkg/agent/route/route_linux.go b/pkg/agent/route/route_linux.go index c61b0a479b4..228f24a3eaf 100644 --- a/pkg/agent/route/route_linux.go +++ b/pkg/agent/route/route_linux.go @@ -59,10 +59,9 @@ var ( // Client takes care of routing container packets in host network, coordinating ip route, ip rule, iptables and ipset. type Client struct { - nodeConfig *config.NodeConfig - encapMode config.TrafficEncapModeType - serviceCIDR *net.IPNet - ipt *iptables.Client + nodeConfig *config.NodeConfig + encapMode config.TrafficEncapModeType + ipt *iptables.Client // nodeRoutes caches ip routes to remote Pods. It's a map of podCIDR to routes. nodeRoutes sync.Map // nodeNeighbors caches IPv6 Neighbors to remote host gateway @@ -70,16 +69,11 @@ type Client struct { } // NewClient returns a route client. +// TODO: remove param serviceCIDR after kube-proxy is replaced by Antrea Proxy. This param is not used in this file; +// leaving it here is to be compatible with the implementation on Windows. func NewClient(serviceCIDR *net.IPNet, encapMode config.TrafficEncapModeType) (*Client, error) { - ipt, err := iptables.New() - if err != nil { - return nil, fmt.Errorf("error creating IPTables instance: %v", err) - } - return &Client{ - serviceCIDR: serviceCIDR, - encapMode: encapMode, - ipt: ipt, + encapMode: encapMode, }, nil } @@ -178,8 +172,10 @@ func (c *Client) writeEKSMangleRule(iptablesData *bytes.Buffer) { // initIPTables ensure that the iptables infrastructure we use is set up. // It's idempotent and can safely be called on every startup. func (c *Client) initIPTables() error { - if c.nodeConfig.PodIPv6CIDR != nil { - c.ipt.SetIPv6Supported(true) + var err error + c.ipt, err = iptables.New(c.nodeConfig.PodIPv4CIDR != nil, c.nodeConfig.PodIPv6CIDR != nil) + if err != nil { + return fmt.Errorf("error creating IPTables instance: %v", err) } // Create the antrea managed chains and link them to built-in chains. // We cannot use iptables-restore for these jump rules because there diff --git a/pkg/agent/route/route_windows.go b/pkg/agent/route/route_windows.go index 9d4f5077c40..4aba113eaf9 100644 --- a/pkg/agent/route/route_windows.go +++ b/pkg/agent/route/route_windows.go @@ -44,6 +44,7 @@ type Client struct { } // NewClient returns a route client. +// Todo: remove param serviceCIDR after kube-proxy is replaced by Antrea Proxy completely. func NewClient(serviceCIDR *net.IPNet, encapMode config.TrafficEncapModeType) (*Client, error) { nr := netroute.New() return &Client{ diff --git a/pkg/agent/util/iptables/iptables.go b/pkg/agent/util/iptables/iptables.go index 65195ee4333..939e813cf04 100644 --- a/pkg/agent/util/iptables/iptables.go +++ b/pkg/agent/util/iptables/iptables.go @@ -50,23 +50,33 @@ const ( var restoreWaitSupportedMinVersion = semver.Version{Major: 1, Minor: 6, Patch: 2} type Client struct { - ipt *iptables.IPTables - ip6t *iptables.IPTables - // restoreWaitSupported indicates whether iptables-restore supports --wait flag. + ipts []*iptables.IPTables + // restoreWaitSupported indicates whether iptables-restore (or ip6tables-restore) supports --wait flag. restoreWaitSupported bool - enableIPv6 bool } -func New() (*Client, error) { - ipt, err := iptables.New() - if err != nil { - return nil, fmt.Errorf("error creating IPTables instance: %v", err) +func New(enableIPV4, enableIPV6 bool) (*Client, error) { + var ipts []*iptables.IPTables + var restoreWaitSupported bool + if enableIPV4 { + ipt, err := iptables.New() + if err != nil { + return nil, fmt.Errorf("error creating IPTables instance: %v", err) + } + ipts = append(ipts, ipt) + restoreWaitSupported = isRestoreWaitSupported(ipt) } - ip6t, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) - if err != nil { - return nil, fmt.Errorf("error creating IPTables instance for IPv6: %v", err) + if enableIPV6 { + ip6t, err := iptables.NewWithProtocol(iptables.ProtocolIPv6) + if err != nil { + return nil, fmt.Errorf("error creating IPTables instance for IPv6: %v", err) + } + ipts = append(ipts, ip6t) + if !restoreWaitSupported { + restoreWaitSupported = isRestoreWaitSupported(ip6t) + } } - return &Client{ipt: ipt, ip6t: ip6t, restoreWaitSupported: isRestoreWaitSupported(ipt)}, nil + return &Client{ipts: ipts, restoreWaitSupported: restoreWaitSupported}, nil } func isRestoreWaitSupported(ipt *iptables.IPTables) bool { @@ -77,8 +87,8 @@ func isRestoreWaitSupported(ipt *iptables.IPTables) bool { // ensureChain checks if target chain already exists, creates it if not. func (c *Client) EnsureChain(table string, chain string) error { - ipts := c.getIptablesInstances() - for _, ipt := range ipts { + for idx := range c.ipts { + ipt := c.ipts[idx] oriChains, err := ipt.ListChains(table) if err != nil { return fmt.Errorf("error listing existing chains in table %s: %v", table, err) @@ -94,18 +104,10 @@ func (c *Client) EnsureChain(table string, chain string) error { return nil } -func (c *Client) getIptablesInstances() []*iptables.IPTables { - ipts := []*iptables.IPTables{c.ipt} - if c.enableIPv6 { - ipts = append(ipts, c.ip6t) - } - return ipts -} - // ensureRule checks if target rule already exists, appends it if not. func (c *Client) EnsureRule(table string, chain string, ruleSpec []string) error { - ipts := c.getIptablesInstances() - for _, ipt := range ipts { + for idx := range c.ipts { + ipt := c.ipts[idx] exist, err := ipt.Exists(table, chain, ruleSpec...) if err != nil { return fmt.Errorf("error checking if rule %v exists in table %s chain %s: %v", ruleSpec, table, chain, err) @@ -163,11 +165,23 @@ func (c *Client) Restore(data []byte, flush bool, useIPv6 bool) error { // Save calls iptables-saves to dump chains and tables in iptables. func (c *Client) Save() ([]byte, error) { - return exec.Command("iptables-save", "-c").CombinedOutput() -} - -func (c *Client) SetIPv6Supported(val bool) { - c.enableIPv6 = val + var output []byte + for idx := range c.ipts { + var cmd string + ipt := c.ipts[idx] + switch ipt.Proto() { + case iptables.ProtocolIPv6: + cmd = "ip6tables-save" + default: + cmd = "iptables-save" + } + data, err := exec.Command(cmd, "-c").CombinedOutput() + if err != nil { + return nil, err + } + output = append(output, data...) + } + return output, nil } func contains(chains []string, targetChain string) bool { diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index 9d2fe1b419b..124bf36698c 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -132,13 +132,15 @@ func GetIPv4Addr(ips []net.IP) net.IP { return nil } -func ContainIPv6Addr(ips []net.IP) bool { +func CheckAddressFamilies(ips []net.IP) (hasV4, hasV6 bool) { for _, ip := range ips { - if ip.To4() == nil { - return true + if ip.To4() != nil { + hasV4 = true + } else { + hasV6 = true } } - return false + return } func GetIPWithFamily(ips []net.IP, addrFamily uint8) (net.IP, error) { diff --git a/pkg/support/dump_others.go b/pkg/support/dump_others.go index 0789773d5c5..0515fd43bce 100644 --- a/pkg/support/dump_others.go +++ b/pkg/support/dump_others.go @@ -52,7 +52,8 @@ func (d *agentDumper) DumpHostNetworkInfo(basedir string) error { } func (d *agentDumper) dumpIPTables(basedir string) error { - c, err := iptables.New() + nodeConfig := d.aq.GetNodeConfig() + c, err := iptables.New(nodeConfig.PodIPv4CIDR != nil, nodeConfig.PodIPv6CIDR != nil) if err != nil { return err } diff --git a/test/integration/agent/openflow_test.go b/test/integration/agent/openflow_test.go index 9c17d5b6a7d..56d035cc942 100644 --- a/test/integration/agent/openflow_test.go +++ b/test/integration/agent/openflow_test.go @@ -84,8 +84,15 @@ type testConfig struct { tunnelOFPort uint32 serviceCIDR *net.IPNet globalMAC net.HardwareAddr + enableIPv6 bool + enableIPv4 bool } +var ( + _, podIPv4CIDR, _ = net.ParseCIDR("192.168.1.0/24") + _, podIPv6CIDR, _ = net.ParseCIDR("fd74:ca9b:172:19::/64") +) + func TestConnectivityFlows(t *testing.T) { // Initialize ovs metrics (Prometheus) to test them metrics.InitializeOVSMetrics() @@ -228,10 +235,17 @@ func testReplayFlows(t *testing.T) { } func testInitialize(t *testing.T, config *testConfig) { - if _, err := c.Initialize(roundInfo, &config1.NodeConfig{}, config1.TrafficEncapModeEncap, config1.HostGatewayOFPort); err != nil { + nodeConfig := &config1.NodeConfig{} + if config.enableIPv4 { + nodeConfig.PodIPv4CIDR = podIPv4CIDR + } + if config.enableIPv6 { + nodeConfig.PodIPv6CIDR = podIPv6CIDR + } + if _, err := c.Initialize(roundInfo, nodeConfig, config1.TrafficEncapModeEncap, config1.HostGatewayOFPort); err != nil { t.Errorf("Failed to initialize openflow client: %v", err) } - for _, tableFlow := range prepareDefaultFlows() { + for _, tableFlow := range prepareDefaultFlows(config) { ofTestUtils.CheckFlowExists(t, ovsCtlClient, tableFlow.tableID, true, tableFlow.flows) } checkOVSFlowMetrics(t, c) @@ -444,6 +458,7 @@ func TestIPv6ConnectivityFlows(t *testing.T) { testInitialize, testInstallNodeFlows, testInstallPodFlows, + testInstallGatewayFlows, testUninstallPodFlows, testUninstallNodeFlows, } { @@ -645,6 +660,8 @@ func prepareConfiguration() *testConfig { tunnelOFPort: uint32(2), serviceCIDR: serviceCIDR, globalMAC: vMAC, + enableIPv4: true, + enableIPv6: false, } } @@ -681,6 +698,8 @@ func prepareIPv6Configuration() *testConfig { tunnelOFPort: uint32(2), serviceCIDR: serviceCIDR, globalMAC: vMAC, + enableIPv4: false, + enableIPv6: true, } } @@ -766,15 +785,6 @@ func prepareGatewayFlows(gwIPs []net.IP, gwMAC net.HardwareAddr, gwOFPort uint32 }, }, }, - { - uint8(31), - []*ofTestUtils.ExpectFlow{ - { - MatchStr: "priority=200,ct_state=-new+trk,ct_mark=0x20,ip", - ActStr: fmt.Sprintf("load:0x%s->NXM_OF_ETH_DST[],goto_table:42", strings.Replace(gwMAC.String(), ":", "", -1)), - }, - }, - }, { uint8(80), []*ofTestUtils.ExpectFlow{ @@ -787,9 +797,11 @@ func prepareGatewayFlows(gwIPs []net.IP, gwMAC net.HardwareAddr, gwOFPort uint32 } for _, gwIP := range gwIPs { - var ipProtoStr string + var ipProtoStr, nwSrcStr, nwDstStr string if gwIP.To4() != nil { ipProtoStr = "ip" + nwSrcStr = "nw_src" + nwDstStr = "nw_dst" flows = append(flows, expectTableFlows{ uint8(10), @@ -806,13 +818,15 @@ func prepareGatewayFlows(gwIPs []net.IP, gwMAC net.HardwareAddr, gwOFPort uint32 }) } else { ipProtoStr = "ipv6" + nwSrcStr = "ipv6_src" + nwDstStr = "ipv6_dst" } flows = append(flows, expectTableFlows{ uint8(70), []*ofTestUtils.ExpectFlow{ { - MatchStr: fmt.Sprintf("priority=200,%s,dl_dst=%s,nw_dst=%s", ipProtoStr, vMAC.String(), gwIP.String()), + MatchStr: fmt.Sprintf("priority=200,%s,dl_dst=%s,%s=%s", ipProtoStr, vMAC.String(), nwDstStr, gwIP.String()), ActStr: fmt.Sprintf("set_field:%s->eth_dst,goto_table:80", gwMAC.String()), }, }, @@ -821,11 +835,20 @@ func prepareGatewayFlows(gwIPs []net.IP, gwMAC net.HardwareAddr, gwOFPort uint32 tableID: uint8(90), flows: []*ofTestUtils.ExpectFlow{ { - MatchStr: fmt.Sprintf("priority=210,%s,nw_src=%s", ipProtoStr, gwIP.String()), + MatchStr: fmt.Sprintf("priority=210,%s,%s=%s", ipProtoStr, nwSrcStr, gwIP.String()), ActStr: "goto_table:105", }, }, }, + expectTableFlows{ + uint8(31), + []*ofTestUtils.ExpectFlow{ + { + MatchStr: fmt.Sprintf("priority=200,ct_state=-new+trk,ct_mark=0x20,%s", ipProtoStr), + ActStr: fmt.Sprintf("load:0x%s->NXM_OF_ETH_DST[],goto_table:42", strings.Replace(gwMAC.String(), ":", "", -1)), + }, + }, + }, ) } @@ -891,8 +914,37 @@ func prepareServiceHelperFlows() []expectTableFlows { } } -func prepareDefaultFlows() []expectTableFlows { +func prepareDefaultFlows(config *testConfig) []expectTableFlows { + table31Flows := expectTableFlows{ + tableID: 31, + flows: []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "resubmit(,40),resubmit(,41)"}}, + } + table105Flows := expectTableFlows{ + tableID: 105, + flows: []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:106"}}, + } + if config.enableIPv4 { + table31Flows.flows = append(table31Flows.flows, + &ofTestUtils.ExpectFlow{MatchStr: "priority=210,ct_state=-new+trk,ct_mark=0x20,ip,reg0=0x1/0xffff", ActStr: "goto_table:42"}, + &ofTestUtils.ExpectFlow{MatchStr: "priority=190,ct_state=+inv+trk,ip", ActStr: "drop"}, + ) + table105Flows.flows = append(table105Flows.flows, + &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk,ip,reg0=0x1/0xffff", ActStr: "ct(commit,table=106,zone=65520,exec(load:0x20->NXM_NX_CT_MARK[])"}, + &ofTestUtils.ExpectFlow{MatchStr: "priority=190,ct_state=+new+trk,ip", ActStr: "ct(commit,table=106,zone=65520)"}, + ) + } + if config.enableIPv6 { + table31Flows.flows = append(table31Flows.flows, + &ofTestUtils.ExpectFlow{MatchStr: "priority=210,ct_state=-new+trk,ct_mark=0x20,ipv6,reg0=0x1/0xffff", ActStr: "goto_table:42"}, + &ofTestUtils.ExpectFlow{MatchStr: "priority=190,ct_state=+inv+trk,ipv6", ActStr: "drop"}, + ) + table105Flows.flows = append(table105Flows.flows, + &ofTestUtils.ExpectFlow{MatchStr: "priority=200,ct_state=+new+trk,ipv6,reg0=0x1/0xffff", ActStr: "ct(commit,table=106,zone=65510,exec(load:0x20->NXM_NX_CT_MARK[])"}, + &ofTestUtils.ExpectFlow{MatchStr: "priority=190,ct_state=+new+trk,ipv6", ActStr: "ct(commit,table=106,zone=65510)"}, + ) + } return []expectTableFlows{ + table31Flows, table105Flows, { uint8(0), []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "drop"}}, @@ -914,14 +966,6 @@ func prepareDefaultFlows() []expectTableFlows { {MatchStr: "priority=200,ip", ActStr: "ct(table=31,zone=65520,nat)"}, }, }, - { - uint8(31), - []*ofTestUtils.ExpectFlow{ - {MatchStr: "priority=210,ct_state=-new+trk,ct_mark=0x20,ip,reg0=0x1/0xffff", ActStr: "goto_table:42"}, - {MatchStr: "priority=190,ct_state=+inv+trk,ip", ActStr: "drop"}, - {MatchStr: "priority=0", ActStr: "resubmit(,40),resubmit(,41)"}, - }, - }, { uint8(42), []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:50"}}, @@ -958,13 +1002,6 @@ func prepareDefaultFlows() []expectTableFlows { uint8(101), []*ofTestUtils.ExpectFlow{{MatchStr: "priority=0", ActStr: "goto_table:105"}}, }, - { - uint8(105), - []*ofTestUtils.ExpectFlow{ - {MatchStr: "priority=200,ct_state=+new+trk,ip,reg0=0x1/0xffff", ActStr: "ct(commit,table=106,zone=65520,exec(load:0x20->NXM_NX_CT_MARK[])"}, - {MatchStr: "priority=190,ct_state=+new+trk,ip", ActStr: "ct(commit,table=106,zone=65520)"}, - {MatchStr: "priority=0", ActStr: "goto_table:106"}}, - }, { uint8(110), []*ofTestUtils.ExpectFlow{