diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh index a92e79f643a..604574cb04d 100755 --- a/ci/jenkins/test.sh +++ b/ci/jenkins/test.sh @@ -371,8 +371,9 @@ function deliver_antrea_windows { sleep 5 # Some tests need us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13 image but it is not for windows/amd64 10.0.17763 # Use e2eteam/agnhost:2.13 instead - harbor_images=("sigwindowstools-kube-proxy:v1.18.0" "agnhost:2.13" "agnhost:2.13" "agnhost:2.29" "e2eteam-jessie-dnsutils:1.0" "e2eteam-pause:3.2") - antrea_images=("sigwindowstools/kube-proxy:v1.18.0" "e2eteam/agnhost:2.13" "us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13" "k8s.gcr.io/e2e-test-images/agnhost:2.29" "e2eteam/jessie-dnsutils:1.0" "e2eteam/pause:3.2") + harbor_images=("sigwindowstools-kube-proxy:v1.18.0" "agnhost:2.13" "agnhost:2.13" "agnhost:2.13" "agnhost:2.29" "e2eteam-jessie-dnsutils:1.0" "e2eteam-jessie-dnsutils:1.0" "e2eteam-pause:3.2" "e2eteam-pause:3.2" "e2eteam-busybox:1.29-windows-amd64-1809") + antrea_images=("sigwindowstools/kube-proxy:v1.18.0" "e2eteam/agnhost:2.13" "us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13" "k8sprow.azurecr.io/kubernetes-e2e-test-images/agnhost:2.13" "k8s.gcr.io/e2e-test-images/agnhost:2.29" "e2eteam/jessie-dnsutils:1.0" "gcr.io/kubernetes-e2e-test-images/jessie-dnsutils:1.0" "e2eteam/pause:3.2" "k8s.gcr.io/pause:3.2" "docker.io/library/busybox:1.29") + common_images=("mcr.microsoft.com/windows/servercore/iis:latest") # Pull necessary images in advance to avoid transient error for i in "${!harbor_images[@]}"; do ssh -o StrictHostKeyChecking=no -n Administrator@${IP} "docker pull -q ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} && docker tag ${DOCKER_REGISTRY}/antrea/${harbor_images[i]} ${antrea_images[i]}" || true diff --git a/cmd/antrea-agent/agent.go b/cmd/antrea-agent/agent.go index 2db8da16567..71fc267ab01 100644 --- a/cmd/antrea-agent/agent.go +++ b/cmd/antrea-agent/agent.go @@ -231,6 +231,7 @@ func run(o *Options) error { k8sClient, crdClient, ovsBridgeClient, + ovsctl.NewClient(o.config.OVSBridge), ofClient, routeClient, ifaceStore, @@ -267,6 +268,7 @@ func run(o *Options) error { k8sClient, informerFactory, ofClient, + ovsctl.NewClient(o.config.OVSBridge), ovsBridgeClient, routeClient, ifaceStore, diff --git a/hack/update-codegen-dockerized.sh b/hack/update-codegen-dockerized.sh index 8b21d5d2871..9d8acf30aa5 100755 --- a/hack/update-codegen-dockerized.sh +++ b/hack/update-codegen-dockerized.sh @@ -51,6 +51,9 @@ function generate_mocks { "pkg/agent/querier AgentQuerier testing" "pkg/agent/route Interface testing" "pkg/agent/ipassigner IPAssigner testing" + "pkg/agent/util/ipset Interface testing" + "pkg/agent/util/iptables Interface testing mock_iptables_linux.go" # Must specify linux.go suffix, otherwise compilation would fail on windows platform as source file has linux build tag. + "pkg/agent/util/netlink Interface testing mock_netlink_linux.go" "pkg/antctl AntctlClient ." "pkg/controller/networkpolicy EndpointQuerier testing" "pkg/controller/querier ControllerQuerier testing" @@ -67,21 +70,29 @@ function generate_mocks { current_year=$(date +"%Y") sed -i "s/YEAR/${current_year}/g" hack/boilerplate/license_header.raw.txt for target in "${MOCKGEN_TARGETS[@]}"; do - read -r package interfaces mock_package <<<"${target}" - package_name=$(basename "${package}") - if [[ "${mock_package}" == "." ]]; then # generate mocks in same package as src - $GOPATH/bin/mockgen \ - -copyright_file hack/boilerplate/license_header.raw.txt \ - -destination "${package}/mock_${package_name}_test.go" \ - -package="${package_name}" \ - "${ANTREA_PKG}/${package}" "${interfaces}" - else # generate mocks in subpackage - $GOPATH/bin/mockgen \ - -copyright_file hack/boilerplate/license_header.raw.txt \ - -destination "${package}/${mock_package}/mock_${package_name}.go" \ - -package="${mock_package}" \ - "${ANTREA_PKG}/${package}" "${interfaces}" + read -r src_package interfaces dst_package_name dst_file_name <<<"${target}" + src_package_name=$(basename "${src_package}") + # Generate mocks in the same package as src if dst_file_name is ".", otherwise create a sub package. + if [[ "${dst_package_name}" == "." ]]; then + package="${src_package_name}" + if [ -n "${dst_file_name}" ]; then + destination="${src_package}/${dst_file_name}" + else + destination="${src_package}/mock_${src_package_name}_test.go" + fi + else + package="${dst_package_name}" + if [ -n "${dst_file_name}" ]; then + destination="${src_package}/${dst_package_name}/${dst_file_name}" + else + destination="${src_package}/${dst_package_name}/mock_${src_package_name}.go" + fi fi + $GOPATH/bin/mockgen \ + -copyright_file hack/boilerplate/license_header.raw.txt \ + -destination "${destination}" \ + -package "${package}" \ + "${ANTREA_PKG}/${src_package}" "${interfaces}" done git checkout HEAD -- hack/boilerplate/license_header.raw.txt } diff --git a/pkg/agent/agent.go b/pkg/agent/agent.go index 50bfc54caaf..8a0afc77132 100644 --- a/pkg/agent/agent.go +++ b/pkg/agent/agent.go @@ -95,6 +95,7 @@ type Initializer struct { client clientset.Interface crdClient versioned.Interface ovsBridgeClient ovsconfig.OVSBridgeClient + ovsCtlClient ovsctl.OVSCtlClient ofClient openflow.Client routeClient route.Interface wireGuardClient wireguard.Interface @@ -122,6 +123,7 @@ func NewInitializer( k8sClient clientset.Interface, crdClient versioned.Interface, ovsBridgeClient ovsconfig.OVSBridgeClient, + ovsCtlClient ovsctl.OVSCtlClient, ofClient openflow.Client, routeClient route.Interface, ifaceStore interfacestore.InterfaceStore, @@ -142,6 +144,7 @@ func NewInitializer( ) *Initializer { return &Initializer{ ovsBridgeClient: ovsBridgeClient, + ovsCtlClient: ovsCtlClient, client: k8sClient, crdClient: crdClient, ifaceStore: ifaceStore, @@ -279,7 +282,6 @@ func (i *Initializer) initInterfaceStore() error { return intf } ifaceList := make([]*interfacestore.InterfaceConfig, 0, len(ovsPorts)) - ovsCtlClient := ovsctl.NewClient(i.ovsBridge) for index := range ovsPorts { port := &ovsPorts[index] ovsPort := &interfacestore.OVSPortConfig{ @@ -297,6 +299,8 @@ func (i *Initializer) initInterfaceStore() error { case interfacestore.AntreaUplink: intf = parseUplinkInterfaceFunc(port, ovsPort) case interfacestore.AntreaTunnel: + fallthrough + case interfacestore.AntreaIPsecTunnel: intf = parseTunnelInterfaceFunc(port, ovsPort) case interfacestore.AntreaHost: if port.Name == i.ovsBridge { @@ -314,9 +318,6 @@ func (i *Initializer) initInterfaceStore() error { intf = cniserver.ParseOVSPortInterfaceConfig(port, ovsPort, true) case interfacestore.AntreaTrafficControl: intf = trafficcontrol.ParseTrafficControlInterfaceConfig(port, ovsPort) - if err := ovsCtlClient.SetPortNoFlood(int(ovsPort.OFPort)); err != nil { - klog.ErrorS(err, "Failed to set port with no-flood config", "PortName", port.Name) - } default: klog.InfoS("Unknown Antrea interface type", "type", interfaceType) } @@ -340,7 +341,11 @@ func (i *Initializer) initInterfaceStore() error { fallthrough case port.IFType == ovsconfig.STTTunnel: intf = parseTunnelInterfaceFunc(port, ovsPort) - antreaIFType = interfacestore.AntreaTunnel + if intf.Type == interfacestore.IPSecTunnelInterface { + antreaIFType = interfacestore.AntreaIPsecTunnel + } else { + antreaIFType = interfacestore.AntreaTunnel + } case port.Name == i.ovsBridge: intf = nil antreaIFType = interfacestore.AntreaHost @@ -368,6 +373,26 @@ func (i *Initializer) initInterfaceStore() error { return nil } +func (i *Initializer) restorePortConfigs() error { + interfaces := i.ifaceStore.ListInterfaces() + for _, intf := range interfaces { + switch intf.Type { + case interfacestore.IPSecTunnelInterface: + fallthrough + case interfacestore.TrafficControlInterface: + if intf.OFPort < 0 { + klog.InfoS("Skipped setting no-flood for port due to invalid ofPort", "port", intf.InterfaceName, "ofport", intf.OFPort) + continue + } + if err := i.ovsCtlClient.SetPortNoFlood(int(intf.OFPort)); err != nil { + return fmt.Errorf("failed to set no-flood for port %s: %w", intf.InterfaceName, err) + } + klog.InfoS("Set no-flood for port", "port", intf.InterfaceName) + } + } + return nil +} + // Initialize sets up agent initial configurations. func (i *Initializer) Initialize() error { klog.Info("Setting up node network") @@ -386,6 +411,10 @@ func (i *Initializer) Initialize() error { return err } + if err := i.restorePortConfigs(); err != nil { + return err + } + // initializeWireGuard must be executed after setupOVSBridge as it requires gateway addresses on the OVS bridge. if i.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeWireGuard { if err := i.initializeWireGuard(); err != nil { @@ -487,14 +516,15 @@ func persistRoundNum(num uint64, bridgeClient ovsconfig.OVSBridgeClient, interva // initOpenFlowPipeline sets up necessary Openflow entries, including pipeline, classifiers, conn_track, and gateway flows // Every time the agent is (re)started, we go through the following sequence: -// 1. agent determines the new round number (this is done by incrementing the round number -// persisted in OVSDB, or if it's not available by picking round 1). -// 2. any existing flow for which the round number matches the round number obtained from step 1 -// is deleted. -// 3. all required flows are installed, using the round number obtained from step 1. -// 4. after convergence, all existing flows for which the round number matches the previous round -// number (i.e. the round number which was persisted in OVSDB, if any) are deleted. -// 5. the new round number obtained from step 1 is persisted to OVSDB. +// 1. agent determines the new round number (this is done by incrementing the round number +// persisted in OVSDB, or if it's not available by picking round 1). +// 2. any existing flow for which the round number matches the round number obtained from step 1 +// is deleted. +// 3. all required flows are installed, using the round number obtained from step 1. +// 4. after convergence, all existing flows for which the round number matches the previous round +// number (i.e. the round number which was persisted in OVSDB, if any) are deleted. +// 5. the new round number obtained from step 1 is persisted to OVSDB. +// // The rationale for not persisting the new round number until after all previous flows have been // deleted is to avoid a situation in which some stale flows are never deleted because of successive // agent restarts (with the agent crashing before step 4 can be completed). With the sequence @@ -552,6 +582,13 @@ func (i *Initializer) initOpenFlowPipeline() error { i.ofClient.ReplayFlows() klog.Info("Flow replay completed") + klog.InfoS("Restoring OF port configs to OVS bridge") + if err := i.restorePortConfigs(); err != nil { + klog.ErrorS(err, "Failed to restore OF port configs") + } else { + klog.InfoS("Port configs restoration completed") + } + if i.ovsBridgeClient.GetOVSDatapathType() == ovsconfig.OVSDatapathNetdev { // we don't set flow-restore-wait when using the OVS netdev datapath return @@ -561,7 +598,7 @@ func (i *Initializer) initOpenFlowPipeline() error { // happen that ovsBridgeClient's connection is not ready when ofClient completes flow replay. We retry it // with a timeout that is longer time than ovsBridgeClient's maximum connecting retry interval (8 seconds) // to ensure the flag can be removed successfully. - err := wait.PollImmediate(200*time.Millisecond, 10*time.Second, func() (done bool, err error) { + err = wait.PollImmediate(200*time.Millisecond, 10*time.Second, func() (done bool, err error) { if err := i.FlowRestoreComplete(); err != nil { return false, nil } diff --git a/pkg/agent/agent_test.go b/pkg/agent/agent_test.go index 266b80ff86a..614823bddd1 100644 --- a/pkg/agent/agent_test.go +++ b/pkg/agent/agent_test.go @@ -37,6 +37,7 @@ import ( "antrea.io/antrea/pkg/agent/types" "antrea.io/antrea/pkg/ovs/ovsconfig" ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing" + ovsctltest "antrea.io/antrea/pkg/ovs/ovsctl/testing" "antrea.io/antrea/pkg/util/env" "antrea.io/antrea/pkg/util/ip" ) @@ -518,3 +519,78 @@ func mockConfigureLinkAddress(returnedErr error) func() { configureLinkAddresses = originalConfigureLinkAddresses } } + +func TestRestorePortConfigs(t *testing.T) { + ipsecTunnelInterface := interfacestore.NewIPSecTunnelInterface("antrea-ipsec1", + ovsconfig.GeneveTunnel, + "node1", + net.ParseIP("1.1.1.1"), + "abcdefg", + "node1") + ipsecTunnelInterface.OVSPortConfig = &interfacestore.OVSPortConfig{OFPort: 11, PortUUID: "uuid1"} + tunnelInterface := interfacestore.NewTunnelInterface(defaultTunInterfaceName, + ovsconfig.GeneveTunnel, + 0, + net.ParseIP("1.1.1.10"), + true) + tunnelInterface.OVSPortConfig = &interfacestore.OVSPortConfig{OFPort: 12} + trafficControlInterface1 := interfacestore.NewTrafficControlInterface("antrea-tap1") + trafficControlInterface1.OVSPortConfig = &interfacestore.OVSPortConfig{OFPort: 13, PortUUID: "uuid3"} + trafficControlInterface2 := interfacestore.NewTrafficControlInterface("antrea-tap2") + trafficControlInterface2.OVSPortConfig = &interfacestore.OVSPortConfig{OFPort: -1, PortUUID: "uuid3"} + + tests := []struct { + name string + existingInterfaces []*interfacestore.InterfaceConfig + expectedOVSCtlCalls func(client *ovsctltest.MockOVSCtlClientMockRecorder) + expectedErr string + }{ + { + name: "success", + existingInterfaces: []*interfacestore.InterfaceConfig{ + ipsecTunnelInterface, + tunnelInterface, + trafficControlInterface1, + trafficControlInterface2, + }, + expectedOVSCtlCalls: func(client *ovsctltest.MockOVSCtlClientMockRecorder) { + client.SetPortNoFlood(11).Return(nil) + client.SetPortNoFlood(13).Return(nil) + }, + }, + { + name: "fail", + existingInterfaces: []*interfacestore.InterfaceConfig{ + { + InterfaceName: "antrea-tap1", + Type: interfacestore.TrafficControlInterface, + OVSPortConfig: &interfacestore.OVSPortConfig{OFPort: 10, PortUUID: "uuid3"}, + }, + }, + expectedOVSCtlCalls: func(client *ovsctltest.MockOVSCtlClientMockRecorder) { + client.SetPortNoFlood(10).Return(fmt.Errorf("server unavailable")) + }, + expectedErr: "failed to set no-flood for port antrea-tap1: server unavailable", + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + controller := mock.NewController(t) + defer controller.Finish() + mockOVSCtlClient := ovsctltest.NewMockOVSCtlClient(controller) + ifaceStore := interfacestore.NewInterfaceStore() + initializer := &Initializer{ + ifaceStore: ifaceStore, + ovsCtlClient: mockOVSCtlClient, + } + ifaceStore.Initialize(tt.existingInterfaces) + tt.expectedOVSCtlCalls(mockOVSCtlClient.EXPECT()) + err := initializer.restorePortConfigs() + if tt.expectedErr == "" { + assert.NoError(t, err) + } else { + assert.ErrorContains(t, err, tt.expectedErr) + } + }) + } +} diff --git a/pkg/agent/controller/noderoute/node_route_controller.go b/pkg/agent/controller/noderoute/node_route_controller.go index bcfae3ce5b4..acee38cc98e 100644 --- a/pkg/agent/controller/noderoute/node_route_controller.go +++ b/pkg/agent/controller/noderoute/node_route_controller.go @@ -40,9 +40,9 @@ import ( "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/agent/wireguard" "antrea.io/antrea/pkg/ovs/ovsconfig" + "antrea.io/antrea/pkg/ovs/ovsctl" utilip "antrea.io/antrea/pkg/util/ip" "antrea.io/antrea/pkg/util/k8s" - "antrea.io/antrea/pkg/util/runtime" ) const ( @@ -65,6 +65,7 @@ type Controller struct { kubeClient clientset.Interface ovsBridgeClient ovsconfig.OVSBridgeClient ofClient openflow.Client + ovsCtlClient ovsctl.OVSCtlClient routeClient route.Interface interfaceStore interfacestore.InterfaceStore networkConfig *config.NetworkConfig @@ -72,7 +73,6 @@ type Controller struct { nodeInformer coreinformers.NodeInformer nodeLister corelisters.NodeLister nodeListerSynced cache.InformerSynced - svcLister corelisters.ServiceLister queue workqueue.RateLimitingInterface // installedNodes records routes and flows installation states of Nodes. // The key is the host name of the Node, the value is the nodeRouteInfo of the Node. @@ -92,6 +92,7 @@ func NewNodeRouteController( kubeClient clientset.Interface, informerFactory informers.SharedInformerFactory, client openflow.Client, + ovsCtlClient ovsctl.OVSCtlClient, ovsBridgeClient ovsconfig.OVSBridgeClient, routeClient route.Interface, interfaceStore interfacestore.InterfaceStore, @@ -102,11 +103,11 @@ func NewNodeRouteController( ipsecCertificateManager ipseccertificate.Manager, ) *Controller { nodeInformer := informerFactory.Core().V1().Nodes() - svcLister := informerFactory.Core().V1().Services() controller := &Controller{ kubeClient: kubeClient, ovsBridgeClient: ovsBridgeClient, ofClient: client, + ovsCtlClient: ovsCtlClient, routeClient: routeClient, interfaceStore: interfaceStore, networkConfig: networkConfig, @@ -114,7 +115,6 @@ func NewNodeRouteController( nodeInformer: nodeInformer, nodeLister: nodeInformer.Lister(), nodeListerSynced: nodeInformer.Informer().HasSynced, - svcLister: svcLister.Lister(), queue: workqueue.NewNamedRateLimitingQueue(workqueue.NewItemExponentialFailureRateLimiter(minRetryDelay, maxRetryDelay), "noderoute"), installedNodes: cache.NewIndexer(nodeRouteInfoKeyFunc, cache.Indexers{nodeRouteInfoPodCIDRIndexName: nodeRouteInfoPodCIDRIndexFunc}), wireGuardClient: wireguardClient, @@ -203,27 +203,10 @@ func (c *Controller) removeStaleGatewayRoutes() error { desiredPodCIDRs = append(desiredPodCIDRs, podCIDRs...) } - // TODO: This is not the best place to keep the ClusterIP Service routes. - desiredClusterIPSvcIPs := map[string]bool{} - if c.proxyAll && runtime.IsWindowsPlatform() { - // The route for virtual IP -> antrea-gw0 should be always kept. - desiredClusterIPSvcIPs[config.VirtualServiceIPv4.String()] = true - - svcs, err := c.svcLister.List(labels.Everything()) - for _, svc := range svcs { - for _, ip := range svc.Spec.ClusterIPs { - desiredClusterIPSvcIPs[ip] = true - } - } - if err != nil { - return fmt.Errorf("error when listing ClusterIP Service IPs: %v", err) - } - } - // routeClient will remove orphaned routes whose destinations are not in desiredPodCIDRs. // If proxyAll enabled, it will also remove routes that are for Windows ClusterIP Services // which no longer exist. - if err := c.routeClient.Reconcile(desiredPodCIDRs, desiredClusterIPSvcIPs); err != nil { + if err := c.routeClient.Reconcile(desiredPodCIDRs); err != nil { return err } return nil @@ -244,7 +227,7 @@ func (c *Controller) removeStaleTunnelPorts() error { // will not include it in the set. desiredInterfaces := make(map[string]bool) // knownInterfaces is the list of interfaces currently in the local cache. - knownInterfaces := c.interfaceStore.GetInterfaceKeysByType(interfacestore.TunnelInterface) + knownInterfaces := c.interfaceStore.GetInterfaceKeysByType(interfacestore.IPSecTunnelInterface) if c.networkConfig.TrafficEncryptionMode == config.TrafficEncryptionModeIPSec { for _, node := range nodes { @@ -669,15 +652,14 @@ func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int3 } c.interfaceStore.DeleteInterface(interfaceConfig) exists = false - } else { - if interfaceConfig.OFPort != 0 { - klog.V(2).InfoS("Found cached IPsec tunnel interface", "node", nodeName, "interface", interfaceConfig.InterfaceName, "port", interfaceConfig.OFPort) - return interfaceConfig.OFPort, nil - } } } + if !exists { - ovsExternalIDs := map[string]interface{}{ovsExternalIDNodeName: nodeName} + ovsExternalIDs := map[string]interface{}{ + ovsExternalIDNodeName: nodeName, + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaIPsecTunnel, + } portUUID, err := c.ovsBridgeClient.CreateTunnelPortExt( portName, c.networkConfig.TunnelType, @@ -713,6 +695,12 @@ func (c *Controller) createIPSecTunnelPort(nodeName string, nodeIP net.IP) (int3 // Let NodeRouteController retry at errors. return 0, fmt.Errorf("failed to get of_port of IPsec tunnel port for Node %s", nodeName) } + + // Set the port with no-flood to reject ARP flood packets. + if err := c.ovsCtlClient.SetPortNoFlood(int(ofPort)); err != nil { + return 0, fmt.Errorf("failed to set port %s with no-flood config: %w", portName, err) + } + interfaceConfig.OFPort = ofPort return ofPort, nil } diff --git a/pkg/agent/controller/noderoute/node_route_controller_test.go b/pkg/agent/controller/noderoute/node_route_controller_test.go index fd70d204270..a324dd43695 100644 --- a/pkg/agent/controller/noderoute/node_route_controller_test.go +++ b/pkg/agent/controller/noderoute/node_route_controller_test.go @@ -35,6 +35,7 @@ import ( "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/ovs/ovsconfig" ovsconfigtest "antrea.io/antrea/pkg/ovs/ovsconfig/testing" + ovsctltest "antrea.io/antrea/pkg/ovs/ovsctl/testing" utilip "antrea.io/antrea/pkg/util/ip" ) @@ -58,6 +59,7 @@ type fakeController struct { ovsClient *ovsconfigtest.MockOVSBridgeClient routeClient *routetest.MockInterface interfaceStore interfacestore.InterfaceStore + ovsCtlClient *ovsctltest.MockOVSCtlClient } type fakeIPsecCertificateManager struct{} @@ -75,7 +77,9 @@ func newController(t *testing.T, networkConfig *config.NetworkConfig) (*fakeCont routeClient := routetest.NewMockInterface(ctrl) interfaceStore := interfacestore.NewInterfaceStore() ipsecCertificateManager := &fakeIPsecCertificateManager{} - c := NewNodeRouteController(clientset, informerFactory, ofClient, ovsClient, routeClient, interfaceStore, networkConfig, &config.NodeConfig{GatewayConfig: &config.GatewayConfig{ + ovsCtlClient := ovsctltest.NewMockOVSCtlClient(ctrl) + + c := NewNodeRouteController(clientset, informerFactory, ofClient, ovsCtlClient, ovsClient, routeClient, interfaceStore, networkConfig, &config.NodeConfig{GatewayConfig: &config.GatewayConfig{ IPv4: nil, MAC: gatewayMAC, }}, nil, false, ipsecCertificateManager) @@ -86,6 +90,7 @@ func newController(t *testing.T, networkConfig *config.NetworkConfig) (*fakeCont ofClient: ofClient, ovsClient: ovsClient, routeClient: routeClient, + ovsCtlClient: ovsCtlClient, interfaceStore: interfaceStore, }, ctrl.Finish } @@ -254,7 +259,7 @@ func setup(t *testing.T, ifaces []*interfacestore.InterfaceConfig, authenticatio func TestRemoveStaleTunnelPorts(t *testing.T) { c, closeFn := setup(t, []*interfacestore.InterfaceConfig{ { - Type: interfacestore.TunnelInterface, + Type: interfacestore.IPSecTunnelInterface, InterfaceName: util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-1"), TunnelInterfaceConfig: &interfacestore.TunnelInterfaceConfig{ NodeName: "xyz-k8s-0-1", @@ -302,7 +307,7 @@ func TestRemoveStaleTunnelPorts(t *testing.T) { func TestCreateIPSecTunnelPortPSK(t *testing.T) { c, closeFn := setup(t, []*interfacestore.InterfaceConfig{ { - Type: interfacestore.TunnelInterface, + Type: interfacestore.IPSecTunnelInterface, InterfaceName: "mismatchedname", TunnelInterfaceConfig: &interfacestore.TunnelInterfaceConfig{ NodeName: "xyz-k8s-0-2", @@ -315,7 +320,7 @@ func TestCreateIPSecTunnelPortPSK(t *testing.T) { }, }, { - Type: interfacestore.TunnelInterface, + Type: interfacestore.IPSecTunnelInterface, InterfaceName: util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-3"), TunnelInterfaceConfig: &interfacestore.TunnelInterfaceConfig{ NodeName: "xyz-k8s-0-3", @@ -339,16 +344,25 @@ func TestCreateIPSecTunnelPortPSK(t *testing.T) { node1PortName := util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-1") node2PortName := util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-2") + node3PortName := util.GenerateNodeTunnelInterfaceName("xyz-k8s-0-3") c.ovsClient.EXPECT().CreateTunnelPortExt( node1PortName, ovsconfig.TunnelType("vxlan"), int32(0), false, "", nodeIP1.String(), "", "changeme", nil, - map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1"}).Times(1) + map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1", + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaIPsecTunnel, + }).Times(1) c.ovsClient.EXPECT().CreateTunnelPortExt( node2PortName, ovsconfig.TunnelType("vxlan"), int32(0), false, "", nodeIP2.String(), "", "changeme", nil, - map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-2"}).Times(1) + map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-2", + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaIPsecTunnel, + }).Times(1) c.ovsClient.EXPECT().GetOFPort(node1PortName, false).Return(int32(1), nil) + c.ovsCtlClient.EXPECT().SetPortNoFlood(1) c.ovsClient.EXPECT().GetOFPort(node2PortName, false).Return(int32(2), nil) + c.ovsCtlClient.EXPECT().SetPortNoFlood(2) + c.ovsClient.EXPECT().GetOFPort(node3PortName, false).Return(int32(5), nil) + c.ovsCtlClient.EXPECT().SetPortNoFlood(5) c.ovsClient.EXPECT().DeletePort("123").Times(1) tests := []struct { @@ -405,8 +419,11 @@ func TestCreateIPSecTunnelPortCert(t *testing.T) { c.ovsClient.EXPECT().CreateTunnelPortExt( node1PortName, ovsconfig.TunnelType("vxlan"), int32(0), false, "", nodeIP1.String(), "xyz-k8s-0-1", "", nil, - map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1"}).Times(1) + map[string]interface{}{ovsExternalIDNodeName: "xyz-k8s-0-1", + interfacestore.AntreaInterfaceTypeKey: interfacestore.AntreaIPsecTunnel, + }).Times(1) c.ovsClient.EXPECT().GetOFPort(node1PortName, false).Return(int32(1), nil) + c.ovsCtlClient.EXPECT().SetPortNoFlood(1) tests := []struct { name string diff --git a/pkg/agent/interfacestore/interface_cache.go b/pkg/agent/interfacestore/interface_cache.go index 1d35378ae1c..479593452ab 100644 --- a/pkg/agent/interfacestore/interface_cache.go +++ b/pkg/agent/interfacestore/interface_cache.go @@ -86,8 +86,8 @@ func getInterfaceKey(obj interface{}) (string, error) { var key string if interfaceConfig.Type == ContainerInterface { key = util.GenerateContainerInterfaceKey(interfaceConfig.ContainerID) - } else if interfaceConfig.Type == TunnelInterface && interfaceConfig.NodeName != "" { - // Tunnel interface for a Node. + } else if interfaceConfig.Type == IPSecTunnelInterface { + // IPsec tunnel interface for a Node. key = util.GenerateNodeTunnelInterfaceKey(interfaceConfig.NodeName) } else { // Use the interface name as the key by default. @@ -123,6 +123,15 @@ func (c *interfaceCache) GetInterface(interfaceKey string) (*InterfaceConfig, bo return iface.(*InterfaceConfig), found } +// ListInterfacesByType lists all interfaces from local cache. +func (c *interfaceCache) ListInterfaces() []*InterfaceConfig { + interfaceConfigs := make([]*InterfaceConfig, 0) + for _, iface := range c.cache.List() { + interfaceConfigs = append(interfaceConfigs, iface.(*InterfaceConfig)) + } + return interfaceConfigs +} + // GetInterfaceByName retrieves interface from local cache given the interface // name. func (c *interfaceCache) GetInterfaceByName(interfaceName string) (*InterfaceConfig, bool) { diff --git a/pkg/agent/interfacestore/interface_cache_test.go b/pkg/agent/interfacestore/interface_cache_test.go new file mode 100644 index 00000000000..ba7b4ea770a --- /dev/null +++ b/pkg/agent/interfacestore/interface_cache_test.go @@ -0,0 +1,248 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package interfacestore + +import ( + "net" + "reflect" + "testing" + + "github.com/stretchr/testify/assert" + + "antrea.io/antrea/pkg/agent/util" + "antrea.io/antrea/pkg/ovs/ovsconfig" +) + +var ( + podMAC, _ = net.ParseMAC("11:22:33:44:55:66") + podIP = net.ParseIP("1.2.3.4") + gwIP = net.ParseIP("1.2.3.1") + hostIP = net.ParseIP("2.2.2.2") + ipsecTunnelIP = net.ParseIP("2.2.2.3") + nodeName = "n1" + peerNodeName = "n2" +) + +func TestNewInterfaceStore(t *testing.T) { + t.Run("testContainerInterface", testContainerInterface) + t.Run("testGatewayInterface", testGatewayInterface) + t.Run("testTunnelInterface", testTunnelInterface) + t.Run("testUplinkInterface", testUplinkInterface) + t.Run("testExternalEntityInterface", testEntityInterface) +} + +func testContainerInterface(t *testing.T) { + store := NewInterfaceStore() + containerInterface := NewContainerInterface("ns0p0c0", "c0", "p0", "ns0", podMAC, []net.IP{podIP}, 2) + containerInterface.OVSPortConfig = &OVSPortConfig{ + OFPort: 12, + PortUUID: "1234567890", + } + containerInterfaceKey := util.GenerateContainerInterfaceKey(containerInterface.ContainerID) + store.Initialize([]*InterfaceConfig{containerInterface}) + assert.Equal(t, 1, store.Len()) + storedIface, exists := store.GetInterface(containerInterfaceKey) + assert.True(t, exists) + assert.True(t, reflect.DeepEqual(storedIface, containerInterface)) + // The name of Container InterfaceConfig is not the key in InterfaceStore + _, exists = store.GetInterface(containerInterface.InterfaceName) + assert.False(t, exists) + _, exists = store.GetInterfaceByName(containerInterface.InterfaceName) + assert.True(t, exists) + _, exists = store.GetContainerInterface(containerInterface.ContainerID) + assert.True(t, exists) + _, exists = store.GetInterfaceByIP(podIP.String()) + assert.True(t, exists) + _, exists = store.GetInterfaceByOFPort(uint32(containerInterface.OVSPortConfig.OFPort)) + assert.True(t, exists) + ifaces := store.GetContainerInterfacesByPod(containerInterface.PodName, containerInterface.PodNamespace) + assert.Equal(t, 1, len(ifaces)) + assert.True(t, reflect.DeepEqual(ifaces[0], containerInterface)) + ifaceNames := store.GetInterfaceKeysByType(ContainerInterface) + assert.Equal(t, 1, len(ifaceNames)) + assert.Equal(t, containerInterfaceKey, ifaceNames[0]) + assert.Equal(t, 1, store.GetContainerInterfaceNum()) + store.DeleteInterface(containerInterface) + assert.Equal(t, 0, store.GetContainerInterfaceNum()) + _, exists = store.GetContainerInterface(containerInterface.ContainerID) + assert.False(t, exists) + _, exists = store.GetInterfaceByIP(containerInterface.IPs[0].String()) + assert.False(t, exists) + containerInterface.IPs = nil + store.AddInterface(containerInterface) + assert.Equal(t, 1, store.GetContainerInterfaceNum()) + _, exists = store.GetInterfaceByIP(podIP.String()) + assert.False(t, exists) +} + +func testGatewayInterface(t *testing.T) { + gatewayInterface := NewGatewayInterface("antrea-gw0", util.GenerateRandomMAC()) + gatewayInterface.IPs = []net.IP{gwIP} + gatewayInterface.OVSPortConfig = &OVSPortConfig{ + OFPort: 13, + PortUUID: "1234567890", + } + testGeneralInterface(t, gatewayInterface, GatewayInterface) +} + +func testTunnelInterface(t *testing.T) { + store := NewInterfaceStore() + tunnelInterface := NewTunnelInterface("antrea-tun0", ovsconfig.GeneveTunnel, 6081, hostIP, false) + tunnelInterface.OVSPortConfig = &OVSPortConfig{ + OFPort: 14, + PortUUID: "1234567890", + } + tunnelInterface.IPs = []net.IP{hostIP} + ipsecTunnelInterface := NewIPSecTunnelInterface("antrea-ipsec0", ovsconfig.ERSPANTunnel, nodeName, hostIP, "abcdefg", peerNodeName) + ipsecTunnelInterface.OVSPortConfig = &OVSPortConfig{ + OFPort: 15, + PortUUID: "1234567890", + } + ipsecTunnelInterface.IPs = []net.IP{ipsecTunnelIP} + store.Initialize([]*InterfaceConfig{tunnelInterface, ipsecTunnelInterface}) + assert.Equal(t, 2, store.Len()) + for _, tunIface := range []*InterfaceConfig{tunnelInterface, ipsecTunnelInterface} { + storedIface, exists := store.GetInterfaceByName(tunIface.InterfaceName) + assert.True(t, exists) + assert.True(t, reflect.DeepEqual(storedIface, tunIface)) + storedIface, exists = store.GetInterfaceByIP(tunIface.IPs[0].String()) + assert.True(t, exists) + assert.True(t, reflect.DeepEqual(storedIface, tunIface)) + storedIface, exists = store.GetInterfaceByOFPort(uint32(tunIface.OVSPortConfig.OFPort)) + assert.True(t, exists) + assert.True(t, reflect.DeepEqual(storedIface, tunIface)) + } + + ipsecTunnelKey := util.GenerateNodeTunnelInterfaceKey(ipsecTunnelInterface.NodeName) + storedIface, exists := store.GetInterface(ipsecTunnelKey) + assert.True(t, exists) + assert.True(t, reflect.DeepEqual(storedIface, ipsecTunnelInterface)) + _, exists = store.GetInterface(ipsecTunnelInterface.InterfaceName) + assert.False(t, exists) + storedIface, exists = store.GetInterface(tunnelInterface.InterfaceName) + assert.True(t, exists) + assert.True(t, reflect.DeepEqual(storedIface, tunnelInterface)) + storedIface, exists = store.GetNodeTunnelInterface(nodeName) + assert.True(t, exists) + assert.True(t, reflect.DeepEqual(storedIface, ipsecTunnelInterface)) + _, exists = store.GetNodeTunnelInterface(peerNodeName) + assert.False(t, exists) + + ifaceNames := store.GetInterfaceKeysByType(TunnelInterface) + assert.Equal(t, 1, len(ifaceNames)) + ipsecIfaceNames := store.GetInterfaceKeysByType(IPSecTunnelInterface) + assert.Equal(t, 1, len(ipsecIfaceNames)) + store.DeleteInterface(ipsecTunnelInterface) + assert.Equal(t, 0, len(store.GetInterfaceKeysByType(IPSecTunnelInterface))) + _, exists = store.GetInterfaceByName(ipsecTunnelInterface.InterfaceName) + assert.False(t, exists) + store.AddInterface(ipsecTunnelInterface) + ifaceNames = store.GetInterfaceKeysByType(IPSecTunnelInterface) + assert.Equal(t, 1, len(ifaceNames)) + _, exists = store.GetInterfaceByName(ipsecTunnelInterface.InterfaceName) + assert.True(t, exists) +} + +func testUplinkInterface(t *testing.T) { + uplinkInterface := NewUplinkInterface("ens224") + uplinkInterface.IPs = []net.IP{hostIP} + uplinkInterface.OVSPortConfig = &OVSPortConfig{ + OFPort: 16, + PortUUID: "1234567890", + } + testGeneralInterface(t, uplinkInterface, UplinkInterface) +} + +func testEntityInterface(t *testing.T) { + store := NewInterfaceStore() + portConfig := &OVSPortConfig{OFPort: 18, PortUUID: "123456789"} + uplinkConfig := &OVSPortConfig{OFPort: 19, PortUUID: "987654321"} + entityIPv4 := net.ParseIP("2.3.4.5") + entityIPv6 := net.ParseIP("abcd::1234") + entityIPs := []net.IP{ + entityIPv4, + entityIPv6, + } + entityInterface := newExternalEntityInterface("vm1-ens192", entityIPs, "ens192", "ns2", portConfig, uplinkConfig) + store.Initialize([]*InterfaceConfig{entityInterface}) + storedIface, exists := store.GetInterface(entityInterface.InterfaceName) + assert.True(t, exists) + assert.True(t, reflect.DeepEqual(storedIface, entityInterface)) + assert.Equal(t, entityIPv4, storedIface.GetIPv4Addr()) + assert.Equal(t, entityIPv6, storedIface.GetIPv6Addr()) + _, exists = store.GetInterfaceByName(entityInterface.InterfaceName) + assert.True(t, exists) + for _, entityIP := range entityInterface.IPs { + _, exists = store.GetInterfaceByIP(entityIP.String()) + assert.True(t, exists) + } + _, exists = store.GetInterfaceByOFPort(uint32(entityInterface.OVSPortConfig.OFPort)) + assert.True(t, exists) + _, exists = store.GetInterfaceByOFPort(uint32(entityInterface.UplinkPort.OFPort)) + assert.False(t, exists) + ifaces := store.GetInterfacesByEntity(entityInterface.EntityName, entityInterface.EntityNamespace) + assert.Equal(t, 1, len(ifaces)) + assert.True(t, reflect.DeepEqual(ifaces[0], entityInterface)) + ifaceNames := store.GetInterfaceKeysByType(ExternalEntityInterface) + assert.Equal(t, 1, len(ifaceNames)) + assert.Equal(t, entityInterface.InterfaceName, ifaceNames[0]) + store.DeleteInterface(entityInterface) + assert.Equal(t, 0, len(store.GetInterfaceKeysByType(ExternalEntityInterface))) + store.AddInterface(entityInterface) + assert.Equal(t, 1, len(store.GetInterfaceKeysByType(ExternalEntityInterface))) +} + +func testGeneralInterface(t *testing.T, ifaceConfig *InterfaceConfig, ifaceType InterfaceType) { + store := NewInterfaceStore() + store.Initialize([]*InterfaceConfig{ifaceConfig}) + storedIface, exists := store.GetInterface(ifaceConfig.InterfaceName) + assert.True(t, exists) + assert.True(t, reflect.DeepEqual(storedIface, ifaceConfig)) + _, exists = store.GetInterfaceByName(ifaceConfig.InterfaceName) + assert.True(t, exists) + _, exists = store.GetInterfaceByIP(ifaceConfig.IPs[0].String()) + assert.True(t, exists) + _, exists = store.GetInterfaceByOFPort(uint32(ifaceConfig.OVSPortConfig.OFPort)) + assert.True(t, exists) + fooPort := uint32(1) + _, exists = store.GetInterfaceByOFPort(fooPort) + assert.False(t, exists) + ifaceNames := store.GetInterfaceKeysByType(ifaceType) + assert.Equal(t, 1, len(ifaceNames)) + assert.Equal(t, ifaceConfig.InterfaceName, ifaceNames[0]) + store.DeleteInterface(ifaceConfig) + assert.Equal(t, 0, len(store.GetInterfaceKeysByType(ifaceType))) + store.AddInterface(ifaceConfig) + ifaceNames = store.GetInterfaceKeysByType(ifaceType) + assert.Equal(t, 1, len(ifaceNames)) + assert.Equal(t, ifaceConfig.InterfaceName, ifaceNames[0]) + ifaces := store.GetInterfacesByType(ifaceType) + assert.Equal(t, 1, len(ifaces)) + assert.Equal(t, ifaceConfig, ifaces[0]) +} + +func newExternalEntityInterface(name string, entityIPs []net.IP, entityName string, entityNamespace string, ovsPortConfig, uplinkPortConfig *OVSPortConfig) *InterfaceConfig { + return &InterfaceConfig{ + Type: ExternalEntityInterface, + InterfaceName: name, + IPs: entityIPs, + OVSPortConfig: ovsPortConfig, + EntityInterfaceConfig: &EntityInterfaceConfig{ + EntityName: entityName, + EntityNamespace: entityNamespace, + UplinkPort: uplinkPortConfig, + }, + } +} diff --git a/pkg/agent/interfacestore/testing/mock_interfacestore.go b/pkg/agent/interfacestore/testing/mock_interfacestore.go index 3e8aa50fbc5..70519cf5256 100644 --- a/pkg/agent/interfacestore/testing/mock_interfacestore.go +++ b/pkg/agent/interfacestore/testing/mock_interfacestore.go @@ -257,3 +257,17 @@ func (mr *MockInterfaceStoreMockRecorder) Len() *gomock.Call { mr.mock.ctrl.T.Helper() return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Len", reflect.TypeOf((*MockInterfaceStore)(nil).Len)) } + +// ListInterfaces mocks base method +func (m *MockInterfaceStore) ListInterfaces() []*interfacestore.InterfaceConfig { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListInterfaces") + ret0, _ := ret[0].([]*interfacestore.InterfaceConfig) + return ret0 +} + +// ListInterfaces indicates an expected call of ListInterfaces +func (mr *MockInterfaceStoreMockRecorder) ListInterfaces() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListInterfaces", reflect.TypeOf((*MockInterfaceStore)(nil).ListInterfaces)) +} diff --git a/pkg/agent/interfacestore/types.go b/pkg/agent/interfacestore/types.go index 688875dd4af..d0576ffbf14 100644 --- a/pkg/agent/interfacestore/types.go +++ b/pkg/agent/interfacestore/types.go @@ -37,6 +37,8 @@ const ( TrafficControlInterface // ExternalEntityInterface is used to mark current interface is for ExternalEntity Endpoint ExternalEntityInterface + // IPSecTunnelInterface is used to mark current interface is for IPSec tunnel port + IPSecTunnelInterface AntreaInterfaceTypeKey = "antrea-type" AntreaGateway = "gateway" @@ -45,6 +47,7 @@ const ( AntreaUplink = "uplink" AntreaHost = "host" AntreaTrafficControl = "traffic-control" + AntreaIPsecTunnel = "ipsec-tunnel" AntreaUnset = "" ) @@ -110,6 +113,7 @@ type InterfaceConfig struct { type InterfaceStore interface { Initialize(interfaces []*InterfaceConfig) AddInterface(interfaceConfig *InterfaceConfig) + ListInterfaces() []*InterfaceConfig DeleteInterface(interfaceConfig *InterfaceConfig) GetInterface(interfaceKey string) (*InterfaceConfig, bool) GetInterfaceByName(interfaceName string) (*InterfaceConfig, bool) @@ -164,7 +168,7 @@ func NewTunnelInterface(tunnelName string, tunnelType ovsconfig.TunnelType, dest // Node. func NewIPSecTunnelInterface(interfaceName string, tunnelType ovsconfig.TunnelType, nodeName string, nodeIP net.IP, psk, remoteName string) *InterfaceConfig { tunnelConfig := &TunnelInterfaceConfig{Type: tunnelType, NodeName: nodeName, RemoteIP: nodeIP, PSK: psk, RemoteName: remoteName} - return &InterfaceConfig{InterfaceName: interfaceName, Type: TunnelInterface, TunnelInterfaceConfig: tunnelConfig} + return &InterfaceConfig{InterfaceName: interfaceName, Type: IPSecTunnelInterface, TunnelInterfaceConfig: tunnelConfig} } // NewUplinkInterface creates InterfaceConfig for the uplink interface. diff --git a/pkg/agent/nodeportlocal/rules/iptable_rule.go b/pkg/agent/nodeportlocal/rules/iptable_rule.go index def5bca618d..5d68bb7c527 100644 --- a/pkg/agent/nodeportlocal/rules/iptable_rule.go +++ b/pkg/agent/nodeportlocal/rules/iptable_rule.go @@ -110,7 +110,7 @@ func (ipt *iptablesRules) AddAllRules(nplList []PodNodePort) error { } } writeLine(iptablesData, "COMMIT") - if err := ipt.table.Restore(iptablesData.Bytes(), false, false); err != nil { + if err := ipt.table.Restore(iptablesData.String(), false, false); err != nil { return err } return nil diff --git a/pkg/agent/proxy/proxier.go b/pkg/agent/proxy/proxier.go index 360d1f1e994..525233b86c4 100644 --- a/pkg/agent/proxy/proxier.go +++ b/pkg/agent/proxy/proxier.go @@ -27,7 +27,7 @@ import ( corev1 "k8s.io/api/core/v1" discovery "k8s.io/api/discovery/v1" "k8s.io/apimachinery/pkg/runtime" - k8sapitypes "k8s.io/apimachinery/pkg/types" + apimachinerytypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/informers" "k8s.io/client-go/tools/record" @@ -108,6 +108,13 @@ type proxier struct { // oversizeServiceSet records the Services that have more than 800 Endpoints. oversizeServiceSet sets.String + // serviceIPRouteReferences tracks the references of Service IP routes. The key is the Service IP and the value is + // the set of ServiceInfo strings. Because a Service could have multiple ports and each port will generate a + // ServicePort (which is the unit of the processing), a Service IP route may be required by several ServicePorts. + // With the references, we install a route exactly once as long as it's used by any ServicePorts and uninstall it + // exactly once when it's no longer used by any ServicePorts. + // It applies to ClusterIP and LoadBalancerIP. + serviceIPRouteReferences map[string]sets.String // syncedOnce returns true if the proxier has synced rules at least once. syncedOnce bool syncedOnceMutex sync.RWMutex @@ -150,6 +157,7 @@ func (p *proxier) removeStaleServices() { continue } svcInfo := svcPort.(*types.ServiceInfo) + svcInfoStr := svcInfo.String() klog.V(2).Infof("Removing stale Service: %s %s", svcPortName.Name, svcInfo.String()) if p.oversizeServiceSet.Has(svcPortName.String()) { p.oversizeServiceSet.Delete(svcPortName.String()) @@ -158,20 +166,25 @@ func (p *proxier) removeStaleServices() { klog.ErrorS(err, "Failed to remove flows of Service", "Service", svcPortName) continue } - + // Remove NodePort and ClusterIP flows and configurations. if p.proxyAll { - // Remove NodePort flows and configurations. if svcInfo.NodePort() > 0 { if err := p.uninstallNodePortService(uint16(svcInfo.NodePort()), svcInfo.OFProtocol); err != nil { - klog.ErrorS(err, "Failed to remove flows and configurations of Service", "Service", svcPortName) + klog.ErrorS(err, "Error when uninstalling NodePort flows and configurations for Service", "ServicePortName", svcPortName, "ServiceInfo", svcInfoStr) + continue + } + } + if svcInfo.ClusterIP() != nil { + if err := p.deleteRouteForServiceIP(svcInfoStr, svcInfo.ClusterIP(), p.routeClient.DeleteClusterIPRoute); err != nil { + klog.ErrorS(err, "Failed to remove ClusterIP Service routes", "Service", svcPortName) continue } } } // Remove LoadBalancer flows and configurations. if p.proxyLoadBalancerIPs && len(svcInfo.LoadBalancerIPStrings()) > 0 { - if err := p.uninstallLoadBalancerService(svcInfo.LoadBalancerIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol); err != nil { - klog.ErrorS(err, "Failed to remove flows and configurations of Service", "Service", svcPortName) + if err := p.uninstallLoadBalancerService(svcInfoStr, svcInfo.LoadBalancerIPStrings(), uint16(svcInfo.Port()), svcInfo.OFProtocol); err != nil { + klog.ErrorS(err, "Error when uninstalling LoadBalancer flows and configurations for Service", "ServicePortName", svcPortName, "ServiceInfo", svcInfoStr) continue } } @@ -319,43 +332,80 @@ func (p *proxier) uninstallNodePortService(svcPort uint16, protocol binding.Prot return nil } -func (p *proxier) installLoadBalancerService(groupID binding.GroupIDType, loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool) error { +func (p *proxier) installLoadBalancerService(svcInfoStr string, groupID binding.GroupIDType, loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol, affinityTimeout uint16, nodeLocalExternal bool) error { for _, ingress := range loadBalancerIPStrings { if ingress != "" { - if err := p.ofClient.InstallServiceFlows(groupID, net.ParseIP(ingress), svcPort, protocol, affinityTimeout, nodeLocalExternal, corev1.ServiceTypeLoadBalancer); err != nil { - return fmt.Errorf("failed to install Service LoadBalancer load balancing flows: %w", err) + ip := net.ParseIP(ingress) + if err := p.ofClient.InstallServiceFlows(groupID, ip, svcPort, protocol, affinityTimeout, nodeLocalExternal, corev1.ServiceTypeLoadBalancer); err != nil { + return fmt.Errorf("failed to install LoadBalancer load balancing flows: %w", err) + } + if p.proxyAll { + if err := p.addRouteForServiceIP(svcInfoStr, ip, p.routeClient.AddLoadBalancer); err != nil { + return fmt.Errorf("failed to install LoadBalancer traffic redirecting routes: %w", err) + } } } } - if p.proxyAll { - if err := p.routeClient.AddLoadBalancer(loadBalancerIPStrings); err != nil { - return fmt.Errorf("failed to install Service LoadBalancer traffic redirecting flows: %w", err) + return nil +} + +func (p *proxier) addRouteForServiceIP(svcInfoStr string, ip net.IP, addRouteFn func(net.IP) error) error { + ipStr := ip.String() + references, exists := p.serviceIPRouteReferences[ipStr] + // If the IP was not referenced by any Service port, install a route for it. + // Otherwise, just reference it. + if !exists { + if err := addRouteFn(ip); err != nil { + return err } + references = sets.NewString(svcInfoStr) + p.serviceIPRouteReferences[ipStr] = references + } else { + references.Insert(svcInfoStr) } - return nil } -func (p *proxier) uninstallLoadBalancerService(loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol) error { +func (p *proxier) uninstallLoadBalancerService(svcInfoStr string, loadBalancerIPStrings []string, svcPort uint16, protocol binding.Protocol) error { for _, ingress := range loadBalancerIPStrings { if ingress != "" { - if err := p.ofClient.UninstallServiceFlows(net.ParseIP(ingress), svcPort, protocol); err != nil { - return fmt.Errorf("failed to remove Service LoadBalancer load balancing flows: %w", err) + ip := net.ParseIP(ingress) + if err := p.ofClient.UninstallServiceFlows(ip, svcPort, protocol); err != nil { + return fmt.Errorf("failed to remove LoadBalancer load balancing flows: %w", err) + } + if p.proxyAll { + if err := p.deleteRouteForServiceIP(svcInfoStr, ip, p.routeClient.DeleteLoadBalancer); err != nil { + return fmt.Errorf("failed to remove LoadBalancer traffic redirecting routes: %w", err) + } } } } - if p.proxyAll { - if err := p.routeClient.DeleteLoadBalancer(loadBalancerIPStrings); err != nil { - return fmt.Errorf("failed to remove Service LoadBalancer traffic redirecting flows: %w", err) + return nil +} + +func (p *proxier) deleteRouteForServiceIP(svcInfoStr string, ip net.IP, deleteRouteFn func(net.IP) error) error { + ipStr := ip.String() + references, exists := p.serviceIPRouteReferences[ipStr] + // If the IP was not referenced by this Service port, skip it. + if exists && references.Has(svcInfoStr) { + // Delete the IP only if this Service port is the last one referencing it. + // Otherwise, just dereference it. + if references.Len() == 1 { + if err := deleteRouteFn(ip); err != nil { + return err + } + delete(p.serviceIPRouteReferences, ipStr) + } else { + references.Delete(svcInfoStr) } } - return nil } func (p *proxier) installServices() { for svcPortName, svcPort := range p.serviceMap { svcInfo := svcPort.(*types.ServiceInfo) + svcInfoStr := svcInfo.String() endpointsInstalled, ok := p.endpointsInstalledMap[svcPortName] if !ok { endpointsInstalled = map[string]k8sproxy.Endpoint{} @@ -599,9 +649,9 @@ func (p *proxier) installServices() { } } // If previous Service which has ClusterIP should be removed, remove ClusterIP routes. - if svcInfo.ClusterIP() != nil { - if err := p.routeClient.DeleteClusterIPRoute(pSvcInfo.ClusterIP()); err != nil { - klog.ErrorS(err, "Failed to remove ClusterIP Service routes", "Service", svcPortName) + if pSvcInfo.ClusterIP() != nil { + if err := p.deleteRouteForServiceIP(pSvcInfo.String(), pSvcInfo.ClusterIP(), p.routeClient.DeleteClusterIPRoute); err != nil { + klog.ErrorS(err, "Error when uninstalling ClusterIP route for Service", "ServicePortName", svcPortName, "ServiceInfo", svcInfoStr) continue } } @@ -621,8 +671,8 @@ func (p *proxier) installServices() { // is created, the routing target IP block will be recalculated for expansion to be able to route the new // created ClusterIP. Deleting a ClusterIP will not shrink the target routing IP block. The Service CIDR // can be finally calculated after creating enough ClusterIPs. - if err := p.routeClient.AddClusterIPRoute(svcInfo.ClusterIP()); err != nil { - klog.ErrorS(err, "Failed to install ClusterIP route of Service", "Service", svcPortName) + if err := p.addRouteForServiceIP(svcInfo.String(), svcInfo.ClusterIP(), p.routeClient.AddClusterIPRoute); err != nil { + klog.ErrorS(err, "Error when installing ClusterIP route for Service", "ServicePortName", svcPortName, "ServiceInfo", svcInfoStr) continue } @@ -649,15 +699,15 @@ func (p *proxier) installServices() { } // Remove LoadBalancer flows and configurations. if len(toDelete) > 0 { - if err := p.uninstallLoadBalancerService(toDelete, uint16(pSvcInfo.Port()), pSvcInfo.OFProtocol); err != nil { - klog.ErrorS(err, "Failed to remove flows and configurations of Service", "Service", svcPortName) + if err := p.uninstallLoadBalancerService(pSvcInfo.String(), toDelete, uint16(pSvcInfo.Port()), pSvcInfo.OFProtocol); err != nil { + klog.ErrorS(err, "Error when uninstalling LoadBalancer flows and configurations for Service", "ServicePortName", svcPortName, "ServiceInfo", svcInfoStr) continue } } // Install LoadBalancer flows and configurations. if len(toAdd) > 0 { - if err := p.installLoadBalancerService(nGroupID, toAdd, uint16(svcInfo.Port()), svcInfo.OFProtocol, uint16(affinityTimeout), svcInfo.NodeLocalExternal()); err != nil { - klog.ErrorS(err, "Failed to install LoadBalancer flows and configurations of Service", "Service", svcPortName) + if err := p.installLoadBalancerService(svcInfo.String(), nGroupID, toAdd, uint16(svcInfo.Port()), svcInfo.OFProtocol, uint16(affinityTimeout), svcInfo.NodeLocalExternal()); err != nil { + klog.ErrorS(err, "Error when installing LoadBalancer flows and configurations for Service", "Service", "ServicePortName", svcPortName, "ServiceInfo", svcInfoStr) continue } } @@ -908,7 +958,7 @@ func (p *proxier) GetProxyProvider() k8sproxy.Provider { } func (p *proxier) GetServiceFlowKeys(serviceName, namespace string) ([]string, []binding.GroupIDType, bool) { - namespacedName := k8sapitypes.NamespacedName{Namespace: namespace, Name: serviceName} + namespacedName := apimachinerytypes.NamespacedName{Namespace: namespace, Name: serviceName} p.serviceEndpointsMapsMutex.Lock() defer p.serviceEndpointsMapsMutex.Unlock() @@ -985,6 +1035,7 @@ func NewProxier( endpointsInstalledMap: types.EndpointsMap{}, endpointsMap: types.EndpointsMap{}, endpointReferenceCounter: map[string]int{}, + serviceIPRouteReferences: map[string]sets.String{}, nodeLabels: map[string]string{}, serviceStringMap: map[string]k8sproxy.ServicePortName{}, oversizeServiceSet: sets.NewString(), diff --git a/pkg/agent/proxy/proxier_test.go b/pkg/agent/proxy/proxier_test.go index f55b4ed2987..d0bc7442728 100644 --- a/pkg/agent/proxy/proxier_test.go +++ b/pkg/agent/proxy/proxier_test.go @@ -24,13 +24,17 @@ import ( "github.com/golang/mock/gomock" "github.com/stretchr/testify/assert" corev1 "k8s.io/api/core/v1" + discovery "k8s.io/api/discovery/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/runtime" apimachinerytypes "k8s.io/apimachinery/pkg/types" "k8s.io/apimachinery/pkg/util/intstr" + "k8s.io/apimachinery/pkg/util/sets" "k8s.io/client-go/tools/record" "k8s.io/component-base/metrics/testutil" + "k8s.io/utils/pointer" + agentconfig "antrea.io/antrea/pkg/agent/config" "antrea.io/antrea/pkg/agent/openflow" ofmock "antrea.io/antrea/pkg/agent/openflow/testing" "antrea.io/antrea/pkg/agent/proxy/metrics" @@ -104,6 +108,7 @@ func makeTestEndpoints(namespace, name string, eptFunc func(*corev1.Endpoints)) type proxyOptions struct { proxyAllEnabled bool proxyLoadBalancerIPs bool + endpointSliceEnabled bool } type proxyOptionsFn func(*proxyOptions) @@ -116,6 +121,10 @@ func withoutProxyLoadBalancerIPs(o *proxyOptions) { o.proxyLoadBalancerIPs = false } +func withEndpointSlice(o *proxyOptions) { + o.endpointSliceEnabled = true +} + func NewFakeProxier(routeClient route.Interface, ofClient openflow.Client, nodePortAddresses []net.IP, groupIDAllocator openflow.GroupAllocator, isIPv6 bool, options ...proxyOptionsFn) *proxier { hostname := "localhost" eventBroadcaster := record.NewBroadcaster() @@ -132,6 +141,7 @@ func NewFakeProxier(routeClient route.Interface, ofClient openflow.Client, nodeP o := &proxyOptions{ proxyAllEnabled: false, proxyLoadBalancerIPs: true, + endpointSliceEnabled: false, } for _, fn := range options { @@ -145,6 +155,7 @@ func NewFakeProxier(routeClient route.Interface, ofClient openflow.Client, nodeP serviceInstalledMap: k8sproxy.ServiceMap{}, endpointsInstalledMap: types.EndpointsMap{}, endpointReferenceCounter: map[string]int{}, + serviceIPRouteReferences: map[string]sets.String{}, endpointsMap: types.EndpointsMap{}, groupCounter: types.NewGroupCounter(groupIDAllocator, make(chan string, 100)), ofClient: ofClient, @@ -154,8 +165,12 @@ func NewFakeProxier(routeClient route.Interface, ofClient openflow.Client, nodeP nodePortAddresses: nodePortAddresses, proxyAll: o.proxyAllEnabled, proxyLoadBalancerIPs: o.proxyLoadBalancerIPs, + hostname: hostname, } p.runner = k8sproxy.NewBoundedFrequencyRunner(componentName, p.syncProxyRules, time.Second, 30*time.Second, 2) + if o.endpointSliceEnabled { + p.endpointsChanges = newEndpointsChangesTracker(hostname, o.endpointSliceEnabled, isIPv6) + } return p } @@ -351,7 +366,7 @@ func testLoadBalancer(t *testing.T, nodePortAddresses []net.IP, svcIP, ep1IP, ep } mockRouteClient.EXPECT().AddClusterIPRoute(svcIP).Times(1) if proxyLoadBalancerIPs { - mockRouteClient.EXPECT().AddLoadBalancer([]string{loadBalancerIP.String()}).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIP).Times(1) } mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(svcNodePort), bindingProtocol).Times(1) @@ -749,6 +764,7 @@ func testClusterIPRemoval(t *testing.T, svcIP net.IP, epIP net.IP, isIPv6 bool) mockOFClient.EXPECT().UninstallServiceFlows(svcIP, uint16(svcPort), bindingProtocol).Times(1) mockOFClient.EXPECT().UninstallEndpointFlows(bindingProtocol, gomock.Any()).Times(1) mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(1) + mockRouteClient.EXPECT().DeleteClusterIPRoute(svcIP).Times(1) fp.syncProxyRules() fp.serviceChanges.OnServiceUpdate(service, nil) @@ -1300,3 +1316,176 @@ func TestMetrics(t *testing.T) { }) } } + +func makeEndpointSliceMap(proxier *proxier, allEndpoints ...*discovery.EndpointSlice) { + for i := range allEndpoints { + proxier.endpointsChanges.OnEndpointSliceUpdate(allEndpoints[i], false) + } + proxier.endpointsChanges.OnEndpointsSynced() +} + +func getMockClients(ctrl *gomock.Controller) (*ofmock.MockClient, *routemock.MockInterface) { + mockOFClient := ofmock.NewMockClient(ctrl) + mockRouteClient := routemock.NewMockInterface(ctrl) + return mockOFClient, mockRouteClient +} + +func TestLoadBalancerServiceWithMultiplePorts(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockOFClient, mockRouteClient := getMockClients(ctrl) + groupAllocator := openflow.NewGroupAllocator(false) + nodePortAddresses := []net.IP{net.ParseIP("0.0.0.0")} + fp := NewFakeProxier(mockRouteClient, mockOFClient, nodePortAddresses, groupAllocator, false, withProxyAll, withEndpointSlice) + + port80Str := "port80" + port80Int32 := int32(80) + port443Str := "port443" + port443Int32 := int32(443) + port30001Int32 := int32(30001) + port30002Int32 := int32(30002) + protocolTCP := corev1.ProtocolTCP + endpoint1Address := "192.168.0.11" + endpoint2Address := "192.168.1.11" + endpoint1NodeName := fp.hostname + endpoint2NodeName := "node2" + svc1IPv4 := net.ParseIP("10.20.30.41") + + svc := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc", + Namespace: "default", + }, + Spec: corev1.ServiceSpec{ + Ports: []corev1.ServicePort{ + { + Name: port80Str, + Protocol: protocolTCP, + Port: port80Int32, + TargetPort: intstr.FromInt(int(port80Int32)), + NodePort: port30001Int32, + }, + { + Name: port443Str, + Protocol: protocolTCP, + Port: port443Int32, + TargetPort: intstr.FromInt(int(port443Int32)), + NodePort: port30002Int32, + }, + }, + ClusterIP: svc1IPv4.String(), + ClusterIPs: []string{svc1IPv4.String()}, + Type: corev1.ServiceTypeLoadBalancer, + ExternalTrafficPolicy: corev1.ServiceExternalTrafficPolicyTypeLocal, + HealthCheckNodePort: 40000, + IPFamilies: []corev1.IPFamily{corev1.IPv4Protocol}, + }, + Status: corev1.ServiceStatus{ + LoadBalancer: corev1.LoadBalancerStatus{Ingress: []corev1.LoadBalancerIngress{ + {IP: loadBalancerIPv4.String()}, + }}, + }, + } + makeServiceMap(fp, svc) + + endpointSlice := &discovery.EndpointSlice{ + ObjectMeta: metav1.ObjectMeta{ + Name: "svc-x5ks2", + Namespace: svc.Namespace, + Labels: map[string]string{ + discovery.LabelServiceName: svc.Name, + }, + }, + AddressType: discovery.AddressTypeIPv4, + Endpoints: []discovery.Endpoint{ + { + Addresses: []string{ + endpoint1Address, + }, + Conditions: discovery.EndpointConditions{ + Ready: pointer.Bool(true), + Serving: pointer.Bool(false), + Terminating: pointer.Bool(false), + }, + NodeName: &endpoint1NodeName, + }, + { + Addresses: []string{ + endpoint2Address, + }, + Conditions: discovery.EndpointConditions{ + Ready: pointer.Bool(true), + Serving: pointer.Bool(false), + Terminating: pointer.Bool(false), + }, + NodeName: &endpoint2NodeName, + }, + }, + Ports: []discovery.EndpointPort{ + { + Name: &port80Str, + Port: &port80Int32, + Protocol: &protocolTCP, + }, + { + Name: &port443Str, + Port: &port443Int32, + Protocol: &protocolTCP, + }, + }, + } + makeEndpointSliceMap(fp, endpointSlice) + + localEndpointForPort80 := k8sproxy.NewBaseEndpointInfo(endpoint1Address, endpoint1NodeName, "", int(port80Int32), true, true, false, false, nil) + localEndpointForPort443 := k8sproxy.NewBaseEndpointInfo(endpoint1Address, endpoint1NodeName, "", int(port443Int32), true, true, false, false, nil) + remoteEndpointForPort80 := k8sproxy.NewBaseEndpointInfo(endpoint2Address, endpoint2NodeName, "", int(port80Int32), false, true, false, false, nil) + remoteEndpointForPort443 := k8sproxy.NewBaseEndpointInfo(endpoint2Address, endpoint2NodeName, "", int(port443Int32), false, true, false, false, nil) + + mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort80, remoteEndpointForPort80})).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, []k8sproxy.Endpoint{localEndpointForPort80}).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort80, remoteEndpointForPort80})).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(port80Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeClusterIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), agentconfig.VirtualNodePortDNATIPv4, uint16(port30001Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeNodePort).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), loadBalancerIPv4, uint16(port80Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(port30001Int32), binding.ProtocolTCP).Times(1) + // The route for the ClusterIP and the LoadBalancer IP should only be installed once. + mockRouteClient.EXPECT().AddClusterIPRoute(svc1IPv4).Times(1) + mockRouteClient.EXPECT().AddLoadBalancer(loadBalancerIPv4).Times(1) + + mockOFClient.EXPECT().InstallEndpointFlows(binding.ProtocolTCP, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort443, remoteEndpointForPort443})).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, []k8sproxy.Endpoint{localEndpointForPort443}).Times(1) + mockOFClient.EXPECT().InstallServiceGroup(gomock.Any(), false, gomock.InAnyOrder([]k8sproxy.Endpoint{localEndpointForPort443, remoteEndpointForPort443})).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), svc1IPv4, uint16(port443Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeClusterIP).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), agentconfig.VirtualNodePortDNATIPv4, uint16(port30002Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeNodePort).Times(1) + mockOFClient.EXPECT().InstallServiceFlows(gomock.Any(), loadBalancerIPv4, uint16(port443Int32), binding.ProtocolTCP, uint16(0), true, corev1.ServiceTypeLoadBalancer).Times(1) + mockRouteClient.EXPECT().AddNodePort(nodePortAddresses, uint16(port30002Int32), binding.ProtocolTCP).Times(1) + + fp.syncProxyRules() + + // Remove the service. + fp.serviceChanges.OnServiceUpdate(svc, nil) + fp.endpointsChanges.OnEndpointSliceUpdate(endpointSlice, true) + + mockOFClient.EXPECT().UninstallEndpointFlows(binding.ProtocolTCP, localEndpointForPort80) + mockOFClient.EXPECT().UninstallEndpointFlows(binding.ProtocolTCP, remoteEndpointForPort80) + mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(2) + mockOFClient.EXPECT().UninstallServiceFlows(svc1IPv4, uint16(port80Int32), binding.ProtocolTCP) + mockOFClient.EXPECT().UninstallServiceFlows(agentconfig.VirtualNodePortDNATIPv4, uint16(port30001Int32), binding.ProtocolTCP) + mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIPv4, uint16(port80Int32), binding.ProtocolTCP) + mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(port30001Int32), binding.ProtocolTCP) + + mockOFClient.EXPECT().UninstallEndpointFlows(binding.ProtocolTCP, localEndpointForPort443) + mockOFClient.EXPECT().UninstallEndpointFlows(binding.ProtocolTCP, remoteEndpointForPort443) + mockOFClient.EXPECT().UninstallServiceGroup(gomock.Any()).Times(2) + mockOFClient.EXPECT().UninstallServiceFlows(svc1IPv4, uint16(port443Int32), binding.ProtocolTCP) + mockOFClient.EXPECT().UninstallServiceFlows(agentconfig.VirtualNodePortDNATIPv4, uint16(port30002Int32), binding.ProtocolTCP) + mockOFClient.EXPECT().UninstallServiceFlows(loadBalancerIPv4, uint16(port443Int32), binding.ProtocolTCP) + mockRouteClient.EXPECT().DeleteNodePort(nodePortAddresses, uint16(port30002Int32), binding.ProtocolTCP) + // The route for the ClusterIP and the LoadBalancer IP should only be uninstalled once. + mockRouteClient.EXPECT().DeleteClusterIPRoute(svc1IPv4) + mockRouteClient.EXPECT().DeleteLoadBalancer(loadBalancerIPv4) + + fp.syncProxyRules() + + assert.Emptyf(t, fp.serviceIPRouteReferences, "serviceIPRouteReferences was not cleaned up after Service was removed") +} diff --git a/pkg/agent/route/interfaces.go b/pkg/agent/route/interfaces.go index f0e1fc04a57..c55e9f7ae5a 100644 --- a/pkg/agent/route/interfaces.go +++ b/pkg/agent/route/interfaces.go @@ -29,7 +29,7 @@ type Interface interface { // Reconcile should remove orphaned routes and related configuration based on the desired podCIDRs and Service IPs. // If IPv6 is enabled in the cluster, Reconcile should also remove the orphaned IPv6 neighbors. - Reconcile(podCIDRs []string, svcIPs map[string]bool) error + Reconcile(podCIDRs []string) error // AddRoutes should add routes to the provided podCIDR. // It should override the routes if they already exist, without error. @@ -65,11 +65,11 @@ type Interface interface { // ClusterIP Service traffic from host network. DeleteClusterIPRoute(svcIP net.IP) error - // AddLoadBalancer adds configurations when a LoadBalancer Service is created. - AddLoadBalancer(externalIPs []string) error + // AddLoadBalancer adds configurations when a LoadBalancer IP is added. + AddLoadBalancer(externalIP net.IP) error - // DeleteLoadBalancer deletes related configurations when a LoadBalancer Service is deleted. - DeleteLoadBalancer(externalIPs []string) error + // DeleteLoadBalancer deletes related configurations when a LoadBalancer IP is deleted. + DeleteLoadBalancer(externalIP net.IP) error // Run starts the sync loop. Run(stopCh <-chan struct{}) diff --git a/pkg/agent/route/route_linux.go b/pkg/agent/route/route_linux.go index 8f57d0d8040..1e3259ac477 100644 --- a/pkg/agent/route/route_linux.go +++ b/pkg/agent/route/route_linux.go @@ -36,6 +36,7 @@ import ( "antrea.io/antrea/pkg/agent/util" "antrea.io/antrea/pkg/agent/util/ipset" "antrea.io/antrea/pkg/agent/util/iptables" + utilnetlink "antrea.io/antrea/pkg/agent/util/netlink" "antrea.io/antrea/pkg/agent/util/sysctl" binding "antrea.io/antrea/pkg/ovs/openflow" "antrea.io/antrea/pkg/ovs/ovsconfig" @@ -94,7 +95,9 @@ type Client struct { nodeConfig *config.NodeConfig networkConfig *config.NetworkConfig noSNAT bool - ipt *iptables.Client + ipt iptables.Interface + ipset ipset.Interface + netlink utilnetlink.Interface // nodeRoutes caches ip routes to remote Pods. It's a map of podCIDR to routes. nodeRoutes sync.Map // nodeNeighbors caches IPv6 Neighbors to remote host gateway @@ -106,6 +109,7 @@ type Client struct { proxyAll bool connectUplinkToBridge bool multicastEnabled bool + isCloudEKS bool // serviceRoutes caches ip routes about Services. serviceRoutes sync.Map // serviceNeighbors caches neighbors. @@ -132,6 +136,9 @@ func NewClient(networkConfig *config.NetworkConfig, noSNAT, proxyAll, connectUpl proxyAll: proxyAll, multicastEnabled: multicastEnabled, connectUplinkToBridge: connectUplinkToBridge, + ipset: ipset.NewClient(), + netlink: utilnetlink.NewClient(), + isCloudEKS: env.IsCloudEKS(), }, nil } @@ -141,14 +148,20 @@ func (c *Client) Initialize(nodeConfig *config.NodeConfig, done func()) error { c.nodeConfig = nodeConfig c.iptablesInitialized = make(chan struct{}) + var err error // Sets up the ipset that will be used in iptables. - if err := c.syncIPSet(); err != nil { + if err = c.syncIPSet(); err != nil { return fmt.Errorf("failed to initialize ipset: %v", err) } + c.ipt, err = iptables.New(c.networkConfig.IPv4Enabled, c.networkConfig.IPv6Enabled) + if err != nil { + return fmt.Errorf("error creating IPTables instance: %v", err) + } // Sets up the iptables infrastructure required to route packets in host network. // It's called in a goroutine because xtables lock may not be acquired immediately. go func() { + klog.Info("Initializing iptables") defer done() defer close(c.iptablesInitialized) var backoffTime = 2 * time.Second @@ -216,7 +229,7 @@ func (c *Client) syncIPInfra() { } func (c *Client) syncRoutes() error { - routeList, err := netlink.RouteList(nil, netlink.FAMILY_ALL) + routeList, err := c.netlink.RouteList(nil, netlink.FAMILY_ALL) if err != nil { return err } @@ -233,7 +246,7 @@ func (c *Client) syncRoutes() error { if ok && routeEqual(route, r) { return true } - if err := netlink.RouteReplace(route); err != nil { + if err := c.netlink.RouteReplace(route); err != nil { klog.Errorf("Failed to add route to the gateway: %v", err) return false } @@ -296,10 +309,10 @@ func (c *Client) syncIPSet() error { if c.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { return nil } - if err := ipset.CreateIPSet(antreaPodIPSet, ipset.HashNet, false); err != nil { + if err := c.ipset.CreateIPSet(antreaPodIPSet, ipset.HashNet, false); err != nil { return err } - if err := ipset.CreateIPSet(antreaPodIP6Set, ipset.HashNet, true); err != nil { + if err := c.ipset.CreateIPSet(antreaPodIP6Set, ipset.HashNet, true); err != nil { return err } @@ -307,7 +320,7 @@ func (c *Client) syncIPSet() error { for _, podCIDR := range []*net.IPNet{c.nodeConfig.PodIPv4CIDR, c.nodeConfig.PodIPv6CIDR} { if podCIDR != nil { ipsetName := getIPSetName(podCIDR.IP) - if err := ipset.AddEntry(ipsetName, podCIDR.String()); err != nil { + if err := c.ipset.AddEntry(ipsetName, podCIDR.String()); err != nil { return err } } @@ -315,23 +328,23 @@ func (c *Client) syncIPSet() error { // If proxy full is enabled, create NodePort ipset. if c.proxyAll { - if err := ipset.CreateIPSet(antreaNodePortIPSet, ipset.HashIPPort, false); err != nil { + if err := c.ipset.CreateIPSet(antreaNodePortIPSet, ipset.HashIPPort, false); err != nil { return err } - if err := ipset.CreateIPSet(antreaNodePortIP6Set, ipset.HashIPPort, true); err != nil { + if err := c.ipset.CreateIPSet(antreaNodePortIP6Set, ipset.HashIPPort, true); err != nil { return err } c.nodePortsIPv4.Range(func(k, _ interface{}) bool { ipSetEntry := k.(string) - if err := ipset.AddEntry(antreaNodePortIPSet, ipSetEntry); err != nil { + if err := c.ipset.AddEntry(antreaNodePortIPSet, ipSetEntry); err != nil { return false } return true }) c.nodePortsIPv6.Range(func(k, _ interface{}) bool { ipSetEntry := k.(string) - if err := ipset.AddEntry(antreaNodePortIP6Set, ipSetEntry); err != nil { + if err := c.ipset.AddEntry(antreaNodePortIP6Set, ipSetEntry); err != nil { return false } return true @@ -339,31 +352,31 @@ func (c *Client) syncIPSet() error { } if c.connectUplinkToBridge { - if err := ipset.CreateIPSet(localAntreaFlexibleIPAMPodIPSet, ipset.HashIP, false); err != nil { + if err := c.ipset.CreateIPSet(localAntreaFlexibleIPAMPodIPSet, ipset.HashIP, false); err != nil { return err } - if err := ipset.CreateIPSet(localAntreaFlexibleIPAMPodIP6Set, ipset.HashIP, true); err != nil { + if err := c.ipset.CreateIPSet(localAntreaFlexibleIPAMPodIP6Set, ipset.HashIP, true); err != nil { return err } } if c.multicastEnabled && c.networkConfig.TrafficEncapMode.SupportsEncap() { - if err := ipset.CreateIPSet(clusterNodeIPSet, ipset.HashIP, false); err != nil { + if err := c.ipset.CreateIPSet(clusterNodeIPSet, ipset.HashIP, false); err != nil { return err } - if err := ipset.CreateIPSet(clusterNodeIP6Set, ipset.HashIP, true); err != nil { + if err := c.ipset.CreateIPSet(clusterNodeIP6Set, ipset.HashIP, true); err != nil { return err } c.clusterNodeIPs.Range(func(_, v interface{}) bool { ipsetEntry := v.(string) - if err := ipset.AddEntry(clusterNodeIPSet, ipsetEntry); err != nil { + if err := c.ipset.AddEntry(clusterNodeIPSet, ipsetEntry); err != nil { return false } return true }) c.clusterNodeIP6s.Range(func(_, v interface{}) bool { ipSetEntry := v.(string) - if err := ipset.AddEntry(clusterNodeIP6Set, ipSetEntry); err != nil { + if err := c.ipset.AddEntry(clusterNodeIP6Set, ipSetEntry); err != nil { return false } return true @@ -459,12 +472,6 @@ func (c *Client) writeEKSNATRules(iptablesData *bytes.Buffer) { // syncIPTables ensure that the iptables infrastructure we use is set up. // It's idempotent and can safely be called on every startup. func (c *Client) syncIPTables() error { - var err error - - c.ipt, err = iptables.New(c.networkConfig.IPv4Enabled, c.networkConfig.IPv6Enabled) - if err != nil { - return fmt.Errorf("error creating IPTables instance: %v", err) - } // Create the antrea managed chains and link them to built-in chains. // We cannot use iptables-restore for these jump rules because there // are non antrea managed rules in built-in chains. @@ -482,7 +489,7 @@ func (c *Client) syncIPTables() error { {iptables.MangleTable, iptables.PreRoutingChain, antreaMangleChain, "Antrea: jump to Antrea mangle rules"}, // TODO: unify the chain naming style {iptables.MangleTable, iptables.OutputChain, antreaOutputChain, "Antrea: jump to Antrea output rules"}, } - if c.proxyAll || env.IsCloudEKS() { + if c.proxyAll || c.isCloudEKS { jumpRules = append(jumpRules, jumpRule{iptables.NATTable, iptables.PreRoutingChain, antreaPreRoutingChain, "Antrea: jump to Antrea prerouting rules"}) } if c.proxyAll { @@ -524,7 +531,7 @@ func (c *Client) syncIPTables() error { false) // Setting --noflush to keep the previous contents (i.e. non antrea managed chains) of the tables. - if err := c.ipt.Restore(iptablesData.Bytes(), false, false); err != nil { + if err := c.ipt.Restore(iptablesData.String(), false, false); err != nil { return err } } @@ -541,7 +548,7 @@ func (c *Client) syncIPTables() error { snatMarkToIPv6, true) // Setting --noflush to keep the previous contents (i.e. non antrea managed chains) of the tables. - if err := c.ipt.Restore(iptablesData.Bytes(), false, true); err != nil { + if err := c.ipt.Restore(iptablesData.String(), false, true); err != nil { return err } } @@ -617,7 +624,7 @@ func (c *Client) restoreIptablesData(podCIDR *net.IPNet, // When Antrea is used to enforce NetworkPolicies in EKS, additional iptables // mangle rules are required. See https://github.com/antrea-io/antrea/issues/678. // These rules are only needed for IPv4. - if env.IsCloudEKS() && !isIPv6 { + if c.isCloudEKS && !isIPv6 { c.writeEKSMangleRules(iptablesData) } @@ -677,7 +684,7 @@ func (c *Client) restoreIptablesData(podCIDR *net.IPNet, writeLine(iptablesData, "COMMIT") writeLine(iptablesData, "*nat") - if c.proxyAll || env.IsCloudEKS() { + if c.proxyAll || c.isCloudEKS { writeLine(iptablesData, iptables.MakeChainLine(antreaPreRoutingChain)) } if c.proxyAll { @@ -784,7 +791,7 @@ func (c *Client) restoreIptablesData(podCIDR *net.IPNet, // When Antrea is used to enforce NetworkPolicies in EKS, additional iptables // nat rules are required. See https://github.com/antrea-io/antrea/issues/3946. // These rules are only needed for IPv4. - if env.IsCloudEKS() && !isIPv6 { + if c.isCloudEKS && !isIPv6 { c.writeEKSNATRules(iptablesData) } @@ -794,16 +801,19 @@ func (c *Client) restoreIptablesData(podCIDR *net.IPNet, func (c *Client) initIPRoutes() error { if c.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() { - gwLink := util.GetNetLink(c.nodeConfig.GatewayConfig.Name) + gwLink, err := c.netlink.LinkByName(c.nodeConfig.GatewayConfig.Name) + if err != nil { + return fmt.Errorf("error getting link %s: %v", c.nodeConfig.GatewayConfig.Name, err) + } if c.nodeConfig.NodeTransportIPv4Addr != nil { _, gwIP, _ := net.ParseCIDR(fmt.Sprintf("%s/32", c.nodeConfig.NodeTransportIPv4Addr.IP.String())) - if err := netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { + if err := c.netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { return fmt.Errorf("failed to add address %s to gw %s: %v", gwIP, gwLink.Attrs().Name, err) } } if c.nodeConfig.NodeTransportIPv6Addr != nil { _, gwIP, _ := net.ParseCIDR(fmt.Sprintf("%s/128", c.nodeConfig.NodeTransportIPv6Addr.IP.String())) - if err := netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { + if err := c.netlink.AddrReplace(gwLink, &netlink.Addr{IPNet: gwIP}); err != nil { return fmt.Errorf("failed to add address %s to gw %s: %v", gwIP, gwLink.Attrs().Name, err) } } @@ -833,14 +843,14 @@ func (c *Client) initServiceIPRoutes() error { // Reconcile removes orphaned podCIDRs from ipset and removes routes to orphaned podCIDRs // based on the desired podCIDRs. svcIPs are used for Windows only. -func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error { +func (c *Client) Reconcile(podCIDRs []string) error { desiredPodCIDRs := sets.NewString(podCIDRs...) // Get the peer IPv6 gateways from pod CIDRs desiredIPv6GWs := getIPv6Gateways(podCIDRs) // Remove orphaned podCIDRs from ipset. for _, ipsetName := range []string{antreaPodIPSet, antreaPodIP6Set} { - entries, err := ipset.ListEntries(ipsetName) + entries, err := c.ipset.ListEntries(ipsetName) if err != nil { return err } @@ -849,7 +859,7 @@ func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error { continue } klog.Infof("Deleting orphaned Pod IP %s from ipset and route table", entry) - if err := ipset.DelEntry(ipsetName, entry); err != nil { + if err := c.ipset.DelEntry(ipsetName, entry); err != nil { return err } _, cidr, err := net.ParseCIDR(entry) @@ -857,12 +867,11 @@ func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error { return err } route := &netlink.Route{Dst: cidr} - if err := netlink.RouteDel(route); err != nil && err != unix.ESRCH { + if err := c.netlink.RouteDel(route); err != nil && err != unix.ESRCH { return err } } } - // Remove any unknown routes on Antrea gateway. routes, err := c.listIPRoutesOnGW() if err != nil { @@ -887,7 +896,7 @@ func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error { } klog.Infof("Deleting unknown route %v", route) - if err := netlink.RouteDel(&route); err != nil && err != unix.ESRCH { + if err := c.netlink.RouteDel(&route); err != nil && err != unix.ESRCH { return err } } @@ -911,7 +920,7 @@ func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error { continue } klog.V(4).Infof("Deleting orphaned IPv6 neighbor %v", actualNeigh) - if err := netlink.NeighDel(actualNeigh); err != nil { + if err := c.netlink.NeighDel(actualNeigh); err != nil { return err } } @@ -931,11 +940,11 @@ func (c *Client) isServiceRoute(route *netlink.Route) bool { func (c *Client) listIPRoutesOnGW() ([]netlink.Route, error) { filter := &netlink.Route{ LinkIndex: c.nodeConfig.GatewayConfig.LinkIndex} - routes, err := netlink.RouteListFiltered(netlink.FAMILY_V4, filter, netlink.RT_FILTER_OIF) + routes, err := c.netlink.RouteListFiltered(netlink.FAMILY_V4, filter, netlink.RT_FILTER_OIF) if err != nil { return nil, err } - ipv6Routes, err := netlink.RouteListFiltered(netlink.FAMILY_V6, filter, netlink.RT_FILTER_OIF) + ipv6Routes, err := c.netlink.RouteListFiltered(netlink.FAMILY_V6, filter, netlink.RT_FILTER_OIF) if err != nil { return nil, err } @@ -958,7 +967,7 @@ func getIPv6Gateways(podCIDRs []string) sets.String { } func (c *Client) listIPv6NeighborsOnGateway() (map[string]*netlink.Neigh, error) { - neighs, err := netlink.NeighList(c.nodeConfig.GatewayConfig.LinkIndex, netlink.FAMILY_V6) + neighs, err := c.netlink.NeighList(c.nodeConfig.GatewayConfig.LinkIndex, netlink.FAMILY_V6) if err != nil { return nil, err } @@ -984,7 +993,7 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP podCIDRStr := podCIDR.String() ipsetName := getIPSetName(podCIDR.IP) // Add this podCIDR to antreaPodIPSet so that packets to them won't be masqueraded when they leave the host. - if err := ipset.AddEntry(ipsetName, podCIDRStr); err != nil { + if err := c.ipset.AddEntry(ipsetName, podCIDRStr); err != nil { return err } // Install routes to this Node. @@ -1030,7 +1039,7 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP klog.InfoS("Skip adding routes to peer", "node", nodeName, "ip", nodeIP, "podCIDR", podCIDR) } for _, route := range routes { - if err := netlink.RouteReplace(route); err != nil { + if err := c.netlink.RouteReplace(route); err != nil { return fmt.Errorf("failed to install route to peer %s (%s) with netlink. Route config: %s. Error: %v", nodeName, nodeIP, route.String(), err) } } @@ -1039,7 +1048,7 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP routeToNodeGwIPNetv6 := &netlink.Route{ Dst: &net.IPNet{IP: nodeGwIP, Mask: net.CIDRMask(128, 128)}, } - if err := netlink.RouteDel(routeToNodeGwIPNetv6); err == nil { + if err := c.netlink.RouteDel(routeToNodeGwIPNetv6); err == nil { klog.InfoS("Deleted route to peer gateway", "node", nodeName, "nodeIP", nodeIP, "nodeGatewayIP", nodeGwIP) } else if err != unix.ESRCH { return fmt.Errorf("failed to delete route to peer gateway on Node %s (%s) with netlink. Route config: %s. Error: %v", @@ -1050,7 +1059,7 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP Family: netlink.FAMILY_V6, IP: nodeGwIP, } - if err := netlink.NeighDel(neigh); err == nil { + if err := c.netlink.NeighDel(neigh); err == nil { klog.InfoS("Deleted neigh to peer gateway", "node", nodeName, "nodeIP", nodeIP, "nodeGatewayIP", nodeGwIP) c.nodeNeighbors.Delete(podCIDRStr) } else if err != unix.ENOENT { @@ -1066,7 +1075,7 @@ func (c *Client) AddRoutes(podCIDR *net.IPNet, nodeName string, nodeIP, nodeGwIP IP: nodeGwIP, HardwareAddr: globalVMAC, } - if err := netlink.NeighSet(neigh); err != nil { + if err := c.netlink.NeighSet(neigh); err != nil { return fmt.Errorf("failed to add neigh %v to gw %s: %v", neigh, c.nodeConfig.GatewayConfig.Name, err) } c.nodeNeighbors.Store(podCIDRStr, neigh) @@ -1085,7 +1094,7 @@ func (c *Client) DeleteRoutes(podCIDR *net.IPNet) error { podCIDRStr := podCIDR.String() ipsetName := getIPSetName(podCIDR.IP) // Delete this podCIDR from antreaPodIPSet as the CIDR is no longer for Pods. - if err := ipset.DelEntry(ipsetName, podCIDRStr); err != nil { + if err := c.ipset.DelEntry(ipsetName, podCIDRStr); err != nil { return err } @@ -1094,7 +1103,7 @@ func (c *Client) DeleteRoutes(podCIDR *net.IPNet) error { c.nodeRoutes.Delete(podCIDRStr) for _, r := range routes.([]*netlink.Route) { klog.V(4).Infof("Deleting route %v", r) - if err := netlink.RouteDel(r); err != nil && err != unix.ESRCH { + if err := c.netlink.RouteDel(r); err != nil && err != unix.ESRCH { c.nodeRoutes.Store(podCIDRStr, routes) return err } @@ -1106,7 +1115,7 @@ func (c *Client) DeleteRoutes(podCIDR *net.IPNet) error { if podCIDR.IP.To4() == nil { neigh, exists := c.nodeNeighbors.Load(podCIDRStr) if exists { - if err := netlink.NeighDel(neigh.(*netlink.Neigh)); err != nil { + if err := c.netlink.NeighDel(neigh.(*netlink.Neigh)); err != nil { return err } c.nodeNeighbors.Delete(podCIDRStr) @@ -1135,28 +1144,31 @@ func writeLine(buf *bytes.Buffer, words ...string) { // MigrateRoutesToGw moves routes (including assigned IP addresses if any) from link linkName to // host gateway. func (c *Client) MigrateRoutesToGw(linkName string) error { - gwLink := util.GetNetLink(c.nodeConfig.GatewayConfig.Name) - link, err := netlink.LinkByName(linkName) + gwLink, err := c.netlink.LinkByName(c.nodeConfig.GatewayConfig.Name) + if err != nil { + return fmt.Errorf("failed to get link %s: %w", c.nodeConfig.GatewayConfig.Name, err) + } + link, err := c.netlink.LinkByName(linkName) if err != nil { return fmt.Errorf("failed to get link %s: %w", linkName, err) } for _, family := range []int{netlink.FAMILY_V4, netlink.FAMILY_V6} { // Swap route first then address, otherwise route gets removed when address is removed. - routes, err := netlink.RouteList(link, family) + routes, err := c.netlink.RouteList(link, family) if err != nil { return fmt.Errorf("failed to get routes for link %s: %w", linkName, err) } for i := range routes { route := routes[i] route.LinkIndex = gwLink.Attrs().Index - if err = netlink.RouteReplace(&route); err != nil { + if err = c.netlink.RouteReplace(&route); err != nil { return fmt.Errorf("failed to add route %v to link %s: %w", &route, gwLink.Attrs().Name, err) } } // Swap address if any. - addrs, err := netlink.AddrList(link, family) + addrs, err := c.netlink.AddrList(link, family) if err != nil { return fmt.Errorf("failed to get addresses for %s: %w", linkName, err) } @@ -1165,11 +1177,11 @@ func (c *Client) MigrateRoutesToGw(linkName string) error { if addr.IP.IsLinkLocalMulticast() || addr.IP.IsLinkLocalUnicast() { continue } - if err = netlink.AddrDel(link, &addr); err != nil { + if err = c.netlink.AddrDel(link, &addr); err != nil { klog.Errorf("failed to delete addr %v from %s: %v", addr, link, err) } tmpAddr := &netlink.Addr{IPNet: addr.IPNet} - if err = netlink.AddrReplace(gwLink, tmpAddr); err != nil { + if err = c.netlink.AddrReplace(gwLink, tmpAddr); err != nil { return fmt.Errorf("failed to add addr %v to gw %s: %w", addr, gwLink.Attrs().Name, err) } } @@ -1179,16 +1191,18 @@ func (c *Client) MigrateRoutesToGw(linkName string) error { // UnMigrateRoutesFromGw moves route from gw to link linkName if provided; otherwise route is deleted func (c *Client) UnMigrateRoutesFromGw(route *net.IPNet, linkName string) error { - gwLink := util.GetNetLink(c.nodeConfig.GatewayConfig.Name) + gwLink, err := c.netlink.LinkByName(c.nodeConfig.GatewayConfig.Name) + if err != nil { + return fmt.Errorf("failed to get link %s: %w", c.nodeConfig.GatewayConfig.Name, err) + } var link netlink.Link - var err error if len(linkName) > 0 { - link, err = netlink.LinkByName(linkName) + link, err = c.netlink.LinkByName(linkName) if err != nil { return fmt.Errorf("failed to get link %s: %w", linkName, err) } } - routes, err := netlink.RouteList(gwLink, netlink.FAMILY_V4) + routes, err := c.netlink.RouteList(gwLink, netlink.FAMILY_V4) if err != nil { return fmt.Errorf("failed to get routes for link %s: %w", gwLink.Attrs().Name, err) } @@ -1197,9 +1211,9 @@ func (c *Client) UnMigrateRoutesFromGw(route *net.IPNet, linkName string) error if route.String() == rt.Dst.String() { if link != nil { rt.LinkIndex = link.Attrs().Index - return netlink.RouteReplace(&rt) + return c.netlink.RouteReplace(&rt) } - return netlink.RouteDel(&rt) + return c.netlink.RouteDel(&rt) } } return nil @@ -1252,13 +1266,13 @@ func (c *Client) addVirtualServiceIPRoute(isIPv6 bool) error { } neigh := generateNeigh(svcIP, linkIndex) - if err := netlink.NeighSet(neigh); err != nil { + if err := c.netlink.NeighSet(neigh); err != nil { return fmt.Errorf("failed to add new IP neighbour for %s: %w", svcIP, err) } c.serviceNeighbors.Store(svcIP.String(), neigh) route := generateRoute(svcIP, mask, nil, linkIndex, netlink.SCOPE_LINK) - if err := netlink.RouteReplace(route); err != nil { + if err := c.netlink.RouteReplace(route); err != nil { return fmt.Errorf("failed to install route for virtual Service IP %s: %w", svcIP.String(), err) } c.serviceRoutes.Store(svcIP.String(), route) @@ -1276,7 +1290,7 @@ func (c *Client) AddNodePort(nodePortAddresses []net.IP, port uint16, protocol b for i := range nodePortAddresses { ipSetEntry := fmt.Sprintf("%s,%s:%d", nodePortAddresses[i], transProtocol, port) - if err := ipset.AddEntry(ipSetName, ipSetEntry); err != nil { + if err := c.ipset.AddEntry(ipSetName, ipSetEntry); err != nil { return err } if isIPv6 { @@ -1297,7 +1311,7 @@ func (c *Client) DeleteNodePort(nodePortAddresses []net.IP, port uint16, protoco for i := range nodePortAddresses { ipSetEntry := fmt.Sprintf("%s,%s:%d", nodePortAddresses[i], transProtocol, port) - if err := ipset.DelEntry(ipSetName, ipSetEntry); err != nil { + if err := c.ipset.DelEntry(ipSetName, ipSetEntry); err != nil { return err } if isIPv6 { @@ -1347,7 +1361,7 @@ func (c *Client) AddClusterIPRoute(svcIP net.IP) error { // Generate a route with the new destination CIDR and install it. newClusterIPCIDRMask, _ := newClusterIPCIDR.Mask.Size() route := generateRoute(newClusterIPCIDR.IP, newClusterIPCIDRMask, gw, linkIndex, scope) - if err = netlink.RouteReplace(route); err != nil { + if err = c.netlink.RouteReplace(route); err != nil { return fmt.Errorf("failed to install new ClusterIP route: %w", err) } // Store the new destination CIDR. @@ -1382,7 +1396,7 @@ func (c *Client) AddClusterIPRoute(svcIP net.IP) error { // Remove stale routes. for _, rt := range staleRoutes { - if err = netlink.RouteDel(rt); err != nil { + if err = c.netlink.RouteDel(rt); err != nil { return fmt.Errorf("failed to uninstall stale ClusterIP route %s: %w", rt.String(), err) } klog.V(4).InfoS("Uninstalled stale ClusterIP route successfully", "stale route", rt) @@ -1402,7 +1416,7 @@ func (c *Client) addVirtualNodePortDNATIPRoute(isIPv6 bool) error { mask = ipv6AddrLength } route := generateRoute(vIP, mask, gw, linkIndex, netlink.SCOPE_UNIVERSE) - if err := netlink.RouteReplace(route); err != nil { + if err := c.netlink.RouteReplace(route); err != nil { return fmt.Errorf("failed to install routing entry for virtual NodePort DNAT IP %s: %w", vIP.String(), err) } klog.V(4).InfoS("Added virtual NodePort DNAT IP route", "route", route) @@ -1411,11 +1425,10 @@ func (c *Client) addVirtualNodePortDNATIPRoute(isIPv6 bool) error { return nil } -// addLoadBalancerIngressIPRoute is used to add routing entry which is used to route LoadBalancer ingress IP to Antrea +// AddLoadBalancer is used to add routing entry which is used to route LoadBalancer ingress IP to Antrea // gateway on host. -func (c *Client) addLoadBalancerIngressIPRoute(svcIPStr string) error { +func (c *Client) AddLoadBalancer(svcIP net.IP) error { linkIndex := c.nodeConfig.GatewayConfig.LinkIndex - svcIP := net.ParseIP(svcIPStr) isIPv6 := utilnet.IsIPv6(svcIP) var gw net.IP var mask int @@ -1428,7 +1441,7 @@ func (c *Client) addLoadBalancerIngressIPRoute(svcIPStr string) error { } route := generateRoute(svcIP, mask, gw, linkIndex, netlink.SCOPE_UNIVERSE) - if err := netlink.RouteReplace(route); err != nil { + if err := c.netlink.RouteReplace(route); err != nil { return fmt.Errorf("failed to install routing entry for LoadBalancer ingress IP %s: %w", svcIP.String(), err) } klog.V(4).InfoS("Added LoadBalancer ingress IP route", "route", route) @@ -1437,11 +1450,10 @@ func (c *Client) addLoadBalancerIngressIPRoute(svcIPStr string) error { return nil } -// deleteLoadBalancerIngressIPRoute is used to delete routing entry which is used to route LoadBalancer ingress IP to Antrea +// DeleteLoadBalancer is used to delete routing entry which is used to route LoadBalancer ingress IP to Antrea // gateway on host. -func (c *Client) deleteLoadBalancerIngressIPRoute(svcIPStr string) error { +func (c *Client) DeleteLoadBalancer(svcIP net.IP) error { linkIndex := c.nodeConfig.GatewayConfig.LinkIndex - svcIP := net.ParseIP(svcIPStr) isIPv6 := utilnet.IsIPv6(svcIP) var gw net.IP var mask int @@ -1454,7 +1466,7 @@ func (c *Client) deleteLoadBalancerIngressIPRoute(svcIPStr string) error { } route := generateRoute(svcIP, mask, gw, linkIndex, netlink.SCOPE_UNIVERSE) - if err := netlink.RouteDel(route); err != nil { + if err := c.netlink.RouteDel(route); err != nil { if err.Error() == "no such process" { klog.InfoS("Failed to delete LoadBalancer ingress IP route since the route has been deleted", "route", route) } else { @@ -1467,28 +1479,6 @@ func (c *Client) deleteLoadBalancerIngressIPRoute(svcIPStr string) error { return nil } -// AddLoadBalancer is used to add routing entries when a LoadBalancer Service is added. -func (c *Client) AddLoadBalancer(externalIPs []string) error { - for _, svcIPStr := range externalIPs { - if err := c.addLoadBalancerIngressIPRoute(svcIPStr); err != nil { - return err - } - } - - return nil -} - -// DeleteLoadBalancer is used to delete routing entries when a LoadBalancer Service is deleted. -func (c *Client) DeleteLoadBalancer(externalIPs []string) error { - for _, svcIPStr := range externalIPs { - if err := c.deleteLoadBalancerIngressIPRoute(svcIPStr); err != nil { - return err - } - } - - return nil -} - // AddLocalAntreaFlexibleIPAMPodRule is used to add IP to target ip set when an AntreaFlexibleIPAM Pod is added. An entry is added // for every Pod IP. func (c *Client) AddLocalAntreaFlexibleIPAMPodRule(podAddresses []net.IP) error { @@ -1509,7 +1499,7 @@ func (c *Client) AddLocalAntreaFlexibleIPAMPodRule(podAddresses []net.IP) error } ipSetEntry := podAddresses[i].String() ipSetName := getLocalAntreaFlexibleIPAMPodIPSetName(isIPv6) - if err := ipset.AddEntry(ipSetName, ipSetEntry); err != nil { + if err := c.ipset.AddEntry(ipSetName, ipSetEntry); err != nil { return err } } @@ -1525,7 +1515,7 @@ func (c *Client) DeleteLocalAntreaFlexibleIPAMPodRule(podAddresses []net.IP) err isIPv6 := podAddresses[i].To4() == nil ipSetEntry := podAddresses[i].String() ipSetName := getLocalAntreaFlexibleIPAMPodIPSetName(isIPv6) - if err := ipset.DelEntry(ipSetName, ipSetEntry); err != nil { + if err := c.ipset.DelEntry(ipSetName, ipSetEntry); err != nil { return err } } @@ -1543,12 +1533,12 @@ func (c *Client) addNodeIP(podCIDR *net.IPNet, nodeIP net.IP) error { } ipSetEntry := nodeIP.String() if nodeIP.To4() != nil { - if err := ipset.AddEntry(clusterNodeIPSet, ipSetEntry); err != nil { + if err := c.ipset.AddEntry(clusterNodeIPSet, ipSetEntry); err != nil { return err } c.clusterNodeIPs.Store(podCIDR.String(), ipSetEntry) } else { - if err := ipset.AddEntry(clusterNodeIP6Set, ipSetEntry); err != nil { + if err := c.ipset.AddEntry(clusterNodeIP6Set, ipSetEntry); err != nil { return err } c.clusterNodeIP6s.Store(podCIDR.String(), ipSetEntry) @@ -1570,7 +1560,7 @@ func (c *Client) deleteNodeIP(podCIDR *net.IPNet) error { return nil } ipSetEntry := obj.(string) - if err := ipset.DelEntry(clusterNodeIPSet, ipSetEntry); err != nil { + if err := c.ipset.DelEntry(clusterNodeIPSet, ipSetEntry); err != nil { return err } c.clusterNodeIPs.Delete(podCIDRStr) @@ -1580,7 +1570,7 @@ func (c *Client) deleteNodeIP(podCIDR *net.IPNet) error { return nil } ipSetEntry := obj.(string) - if err := ipset.DelEntry(clusterNodeIP6Set, ipSetEntry); err != nil { + if err := c.ipset.DelEntry(clusterNodeIP6Set, ipSetEntry); err != nil { return err } c.clusterNodeIP6s.Delete(podCIDRStr) diff --git a/pkg/agent/route/route_linux_test.go b/pkg/agent/route/route_linux_test.go new file mode 100644 index 00000000000..e16e6f92ff0 --- /dev/null +++ b/pkg/agent/route/route_linux_test.go @@ -0,0 +1,1598 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package route + +import ( + "fmt" + "net" + "sync" + "testing" + + "github.com/golang/mock/gomock" + "github.com/stretchr/testify/assert" + "github.com/vishvananda/netlink" + + "antrea.io/antrea/pkg/agent/config" + "antrea.io/antrea/pkg/agent/types" + "antrea.io/antrea/pkg/agent/util/ipset" + ipsettest "antrea.io/antrea/pkg/agent/util/ipset/testing" + "antrea.io/antrea/pkg/agent/util/iptables" + iptablestest "antrea.io/antrea/pkg/agent/util/iptables/testing" + netlinktest "antrea.io/antrea/pkg/agent/util/netlink/testing" + "antrea.io/antrea/pkg/ovs/openflow" + "antrea.io/antrea/pkg/ovs/ovsconfig" + "antrea.io/antrea/pkg/util/ip" +) + +func TestSyncRoutes(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockNetlink := netlinktest.NewMockInterface(ctrl) + + nodeRoute1 := &netlink.Route{Dst: ip.MustParseCIDR("192.168.1.0/24"), Gw: net.ParseIP("1.1.1.1")} + nodeRoute2 := &netlink.Route{Dst: ip.MustParseCIDR("192.168.2.0/24"), Gw: net.ParseIP("1.1.1.2")} + serviceRoute1 := &netlink.Route{Dst: ip.MustParseCIDR("169.254.0.253/32"), LinkIndex: 10} + serviceRoute2 := &netlink.Route{Dst: ip.MustParseCIDR("169.254.0.252/32"), Gw: net.ParseIP("169.254.0.253")} + mockNetlink.EXPECT().RouteList(nil, netlink.FAMILY_ALL).Return([]netlink.Route{*nodeRoute1, *serviceRoute1}, nil) + mockNetlink.EXPECT().RouteReplace(nodeRoute2) + mockNetlink.EXPECT().RouteReplace(serviceRoute2) + mockNetlink.EXPECT().RouteReplace(&netlink.Route{ + LinkIndex: 10, + Dst: ip.MustParseCIDR("192.168.0.0/24"), + Src: net.ParseIP("192.168.0.1"), + Scope: netlink.SCOPE_LINK, + }) + mockNetlink.EXPECT().RouteReplace(&netlink.Route{ + LinkIndex: 10, + Dst: ip.MustParseCIDR("aabb:ccdd::/64"), + Src: net.ParseIP("aabb:ccdd::1"), + Scope: netlink.SCOPE_LINK, + }) + + c := &Client{ + netlink: mockNetlink, + proxyAll: true, + nodeRoutes: sync.Map{}, + serviceRoutes: sync.Map{}, + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{LinkIndex: 10, IPv4: net.ParseIP("192.168.0.1"), IPv6: net.ParseIP("aabb:ccdd::1")}, + PodIPv4CIDR: ip.MustParseCIDR("192.168.0.0/24"), + PodIPv6CIDR: ip.MustParseCIDR("aabb:ccdd::/64"), + }, + } + c.nodeRoutes.Store("192.168.1.0/24", []*netlink.Route{nodeRoute1}) + c.nodeRoutes.Store("192.168.2.0/24", []*netlink.Route{nodeRoute2}) + c.serviceRoutes.Store("169.254.0.253/32", serviceRoute1) + c.serviceRoutes.Store("169.254.0.252/32", serviceRoute2) + + assert.NoError(t, c.syncRoutes()) +} + +func TestSyncIPSet(t *testing.T) { + podCIDRStr := "172.16.10.0/24" + _, podCIDR, _ := net.ParseCIDR(podCIDRStr) + podCIDRv6Str := "2001:ab03:cd04:55ef::/64" + _, podCIDRv6, _ := net.ParseCIDR(podCIDRv6Str) + tests := []struct { + name string + proxyAll bool + multicastEnabled bool + connectUplinkToBridge bool + networkConfig *config.NetworkConfig + nodeConfig *config.NodeConfig + nodePortsIPv4 []string + nodePortsIPv6 []string + clusterNodeIPs map[string]string + clusterNodeIP6s map[string]string + expectedCalls func(ipset *ipsettest.MockInterfaceMockRecorder) + }{ + { + name: "networkPolicyOnly", + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeNetworkPolicyOnly, + }, + expectedCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) {}, + }, + { + name: "noencap", + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeNoEncap, + IPv4Enabled: true, + IPv6Enabled: true, + }, + nodeConfig: &config.NodeConfig{ + PodIPv4CIDR: podCIDR, + PodIPv6CIDR: podCIDRv6, + }, + expectedCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.CreateIPSet(antreaPodIPSet, ipset.HashNet, false) + mockIPSet.CreateIPSet(antreaPodIP6Set, ipset.HashNet, true) + mockIPSet.AddEntry(antreaPodIPSet, podCIDRStr) + mockIPSet.AddEntry(antreaPodIP6Set, podCIDRv6Str) + }, + }, + { + name: "encap, proxyAll=true, multicastEnabled=true", + proxyAll: true, + multicastEnabled: true, + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeEncap, + IPv4Enabled: true, + IPv6Enabled: true, + }, + nodeConfig: &config.NodeConfig{ + PodIPv4CIDR: podCIDR, + PodIPv6CIDR: podCIDRv6, + }, + nodePortsIPv4: []string{"192.168.0.2,tcp:10000", "127.0.0.1,tcp:10000"}, + nodePortsIPv6: []string{"fe80::e643:4bff:fe44:ee,tcp:10000", "::1,tcp:10000"}, + clusterNodeIPs: map[string]string{"172.16.3.0/24": "192.168.0.3", "172.16.4.0/24": "192.168.0.4"}, + clusterNodeIP6s: map[string]string{"2001:ab03:cd04:5503::/64": "fe80::e643:4bff:fe03", "2001:ab03:cd04:5504::/64": "fe80::e643:4bff:fe04"}, + expectedCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.CreateIPSet(antreaPodIPSet, ipset.HashNet, false) + mockIPSet.CreateIPSet(antreaPodIP6Set, ipset.HashNet, true) + mockIPSet.AddEntry(antreaPodIPSet, podCIDRStr) + mockIPSet.AddEntry(antreaPodIP6Set, podCIDRv6Str) + mockIPSet.CreateIPSet(antreaNodePortIPSet, ipset.HashIPPort, false) + mockIPSet.CreateIPSet(antreaNodePortIP6Set, ipset.HashIPPort, true) + mockIPSet.AddEntry(antreaNodePortIPSet, "192.168.0.2,tcp:10000") + mockIPSet.AddEntry(antreaNodePortIPSet, "127.0.0.1,tcp:10000") + mockIPSet.AddEntry(antreaNodePortIP6Set, "fe80::e643:4bff:fe44:ee,tcp:10000") + mockIPSet.AddEntry(antreaNodePortIP6Set, "::1,tcp:10000") + mockIPSet.CreateIPSet(clusterNodeIPSet, ipset.HashIP, false) + mockIPSet.CreateIPSet(clusterNodeIP6Set, ipset.HashIP, true) + mockIPSet.AddEntry(clusterNodeIPSet, "192.168.0.3") + mockIPSet.AddEntry(clusterNodeIPSet, "192.168.0.4") + mockIPSet.AddEntry(clusterNodeIP6Set, "fe80::e643:4bff:fe03") + mockIPSet.AddEntry(clusterNodeIP6Set, "fe80::e643:4bff:fe04") + }, + }, + { + name: "noencap, connectUplinkToBridge=true", + connectUplinkToBridge: true, + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeNoEncap, + IPv4Enabled: true, + IPv6Enabled: true, + }, + nodeConfig: &config.NodeConfig{ + PodIPv4CIDR: podCIDR, + PodIPv6CIDR: podCIDRv6, + }, + expectedCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.CreateIPSet(antreaPodIPSet, ipset.HashNet, false) + mockIPSet.CreateIPSet(antreaPodIP6Set, ipset.HashNet, true) + mockIPSet.AddEntry(antreaPodIPSet, podCIDRStr) + mockIPSet.AddEntry(antreaPodIP6Set, podCIDRv6Str) + mockIPSet.CreateIPSet(localAntreaFlexibleIPAMPodIPSet, ipset.HashIP, false) + mockIPSet.CreateIPSet(localAntreaFlexibleIPAMPodIP6Set, ipset.HashIP, true) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + ipset := ipsettest.NewMockInterface(ctrl) + c := &Client{ipset: ipset, + networkConfig: tt.networkConfig, + nodeConfig: tt.nodeConfig, + proxyAll: tt.proxyAll, + multicastEnabled: tt.multicastEnabled, + connectUplinkToBridge: tt.connectUplinkToBridge, + nodePortsIPv4: sync.Map{}, + nodePortsIPv6: sync.Map{}, + clusterNodeIPs: sync.Map{}, + clusterNodeIP6s: sync.Map{}, + } + for _, nodePortIPv4 := range tt.nodePortsIPv4 { + c.nodePortsIPv4.Store(nodePortIPv4, struct{}{}) + } + for _, nodePortIPv6 := range tt.nodePortsIPv6 { + c.nodePortsIPv6.Store(nodePortIPv6, struct{}{}) + } + for cidr, nodeIP := range tt.clusterNodeIPs { + c.clusterNodeIPs.Store(cidr, nodeIP) + } + for cidr, nodeIP := range tt.clusterNodeIP6s { + c.clusterNodeIP6s.Store(cidr, nodeIP) + } + tt.expectedCalls(ipset.EXPECT()) + assert.NoError(t, c.syncIPSet()) + }) + } +} + +func TestSyncIPTables(t *testing.T) { + tests := []struct { + name string + isCloudEKS bool + proxyAll bool + multicastEnabled bool + connectUplinkToBridge bool + networkConfig *config.NetworkConfig + nodeConfig *config.NodeConfig + nodePortsIPv4 []string + nodePortsIPv6 []string + markToSNATIP map[uint32]string + expectedCalls func(iptables *iptablestest.MockInterfaceMockRecorder) + }{ + { + name: "encap,egress=true,multicastEnabled=true,proxyAll=true", + proxyAll: true, + multicastEnabled: true, + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeEncap, + TunnelType: ovsconfig.GeneveTunnel, + IPv4Enabled: true, + IPv6Enabled: true, + }, + nodeConfig: &config.NodeConfig{ + PodIPv4CIDR: ip.MustParseCIDR("172.16.10.0/24"), + PodIPv6CIDR: ip.MustParseCIDR("2001:ab03:cd04:55ef::/64"), + GatewayConfig: &config.GatewayConfig{ + Name: "antrea-gw0", + }, + }, + markToSNATIP: map[uint32]string{ + 1: "1.1.1.1", + 2: "fe80::e643:4bff:fe02", + }, + expectedCalls: func(mockIPTables *iptablestest.MockInterfaceMockRecorder) { + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.RawTable, antreaPreRoutingChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.RawTable, iptables.PreRoutingChain, []string{"-j", antreaPreRoutingChain, "-m", "comment", "--comment", "Antrea: jump to Antrea prerouting rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.RawTable, antreaOutputChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.RawTable, iptables.OutputChain, []string{"-j", antreaOutputChain, "-m", "comment", "--comment", "Antrea: jump to Antrea output rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.FilterTable, antreaForwardChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.FilterTable, iptables.ForwardChain, []string{"-j", antreaForwardChain, "-m", "comment", "--comment", "Antrea: jump to Antrea forwarding rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.NATTable, antreaPostRoutingChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.NATTable, iptables.PostRoutingChain, []string{"-j", antreaPostRoutingChain, "-m", "comment", "--comment", "Antrea: jump to Antrea postrouting rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.MangleTable, antreaMangleChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.MangleTable, iptables.PreRoutingChain, []string{"-j", antreaMangleChain, "-m", "comment", "--comment", "Antrea: jump to Antrea mangle rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.MangleTable, antreaOutputChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.MangleTable, iptables.OutputChain, []string{"-j", antreaOutputChain, "-m", "comment", "--comment", "Antrea: jump to Antrea output rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.NATTable, antreaPreRoutingChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.NATTable, iptables.PreRoutingChain, []string{"-j", antreaPreRoutingChain, "-m", "comment", "--comment", "Antrea: jump to Antrea prerouting rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.NATTable, antreaOutputChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.NATTable, iptables.OutputChain, []string{"-j", antreaOutputChain, "-m", "comment", "--comment", "Antrea: jump to Antrea output rules"}) + mockIPTables.Restore(`*raw +:ANTREA-PREROUTING - [0:0] +:ANTREA-OUTPUT - [0:0] +-A ANTREA-PREROUTING -m comment --comment "Antrea: do not track incoming encapsulation packets" -m udp -p udp --dport 6081 -m addrtype --dst-type LOCAL -j NOTRACK +-A ANTREA-OUTPUT -m comment --comment "Antrea: do not track outgoing encapsulation packets" -m udp -p udp --dport 6081 -m addrtype --src-type LOCAL -j NOTRACK +-A ANTREA-PREROUTING -m comment --comment "Antrea: drop Pod multicast traffic forwarded via underlay network" -m set --match-set CLUSTER-NODE-IP src -d 224.0.0.0/4 -j DROP +COMMIT +*mangle +:ANTREA-MANGLE - [0:0] +:ANTREA-OUTPUT - [0:0] +-A ANTREA-OUTPUT -m comment --comment "Antrea: mark LOCAL output packets" -m addrtype --src-type LOCAL -o antrea-gw0 -j MARK --or-mark 0x80000000 +COMMIT +*filter +:ANTREA-FORWARD - [0:0] +-A ANTREA-FORWARD -m comment --comment "Antrea: accept packets from local Pods" -i antrea-gw0 -j ACCEPT +-A ANTREA-FORWARD -m comment --comment "Antrea: accept packets to local Pods" -o antrea-gw0 -j ACCEPT +COMMIT +*nat +:ANTREA-PREROUTING - [0:0] +-A ANTREA-PREROUTING -m comment --comment "Antrea: DNAT external to NodePort packets" -m set --match-set ANTREA-NODEPORT-IP dst,dst -j DNAT --to-destination 169.254.0.252 +:ANTREA-OUTPUT - [0:0] +-A ANTREA-OUTPUT -m comment --comment "Antrea: DNAT local to NodePort packets" -m set --match-set ANTREA-NODEPORT-IP dst,dst -j DNAT --to-destination 169.254.0.252 +:ANTREA-POSTROUTING - [0:0] +-A ANTREA-POSTROUTING -m comment --comment "Antrea: SNAT Pod to external packets" ! -o antrea-gw0 -m mark --mark 0x00000001/0x000000ff -j SNAT --to 1.1.1.1 +-A ANTREA-POSTROUTING -m comment --comment "Antrea: masquerade Pod to external packets" -s 172.16.10.0/24 -m set ! --match-set ANTREA-POD-IP dst ! -o antrea-gw0 -j MASQUERADE +-A ANTREA-POSTROUTING -m comment --comment "Antrea: masquerade LOCAL traffic" -o antrea-gw0 -m addrtype ! --src-type LOCAL --limit-iface-out -m addrtype --src-type LOCAL -j MASQUERADE --random-fully +-A ANTREA-POSTROUTING -m comment --comment "Antrea: masquerade OVS virtual source IP" -s 169.254.0.253 -j MASQUERADE +COMMIT +`, false, false) + mockIPTables.Restore(`*raw +:ANTREA-PREROUTING - [0:0] +:ANTREA-OUTPUT - [0:0] +-A ANTREA-PREROUTING -m comment --comment "Antrea: do not track incoming encapsulation packets" -m udp -p udp --dport 6081 -m addrtype --dst-type LOCAL -j NOTRACK +-A ANTREA-OUTPUT -m comment --comment "Antrea: do not track outgoing encapsulation packets" -m udp -p udp --dport 6081 -m addrtype --src-type LOCAL -j NOTRACK +-A ANTREA-PREROUTING -m comment --comment "Antrea: drop Pod multicast traffic forwarded via underlay network" -m set --match-set CLUSTER-NODE-IP6 src -d 224.0.0.0/4 -j DROP +COMMIT +*mangle +:ANTREA-MANGLE - [0:0] +:ANTREA-OUTPUT - [0:0] +-A ANTREA-OUTPUT -m comment --comment "Antrea: mark LOCAL output packets" -m addrtype --src-type LOCAL -o antrea-gw0 -j MARK --or-mark 0x80000000 +COMMIT +*filter +:ANTREA-FORWARD - [0:0] +-A ANTREA-FORWARD -m comment --comment "Antrea: accept packets from local Pods" -i antrea-gw0 -j ACCEPT +-A ANTREA-FORWARD -m comment --comment "Antrea: accept packets to local Pods" -o antrea-gw0 -j ACCEPT +COMMIT +*nat +:ANTREA-PREROUTING - [0:0] +-A ANTREA-PREROUTING -m comment --comment "Antrea: DNAT external to NodePort packets" -m set --match-set ANTREA-NODEPORT-IP6 dst,dst -j DNAT --to-destination fc01::aabb:ccdd:eefe +:ANTREA-OUTPUT - [0:0] +-A ANTREA-OUTPUT -m comment --comment "Antrea: DNAT local to NodePort packets" -m set --match-set ANTREA-NODEPORT-IP6 dst,dst -j DNAT --to-destination fc01::aabb:ccdd:eefe +:ANTREA-POSTROUTING - [0:0] +-A ANTREA-POSTROUTING -m comment --comment "Antrea: SNAT Pod to external packets" ! -o antrea-gw0 -m mark --mark 0x00000002/0x000000ff -j SNAT --to fe80::e643:4bff:fe02 +-A ANTREA-POSTROUTING -m comment --comment "Antrea: masquerade Pod to external packets" -s 2001:ab03:cd04:55ef::/64 -m set ! --match-set ANTREA-POD-IP6 dst ! -o antrea-gw0 -j MASQUERADE +-A ANTREA-POSTROUTING -m comment --comment "Antrea: masquerade LOCAL traffic" -o antrea-gw0 -m addrtype ! --src-type LOCAL --limit-iface-out -m addrtype --src-type LOCAL -j MASQUERADE --random-fully +-A ANTREA-POSTROUTING -m comment --comment "Antrea: masquerade OVS virtual source IP" -s fc01::aabb:ccdd:eeff -j MASQUERADE +COMMIT +`, false, true) + }, + }, + { + name: "encap,eks", + isCloudEKS: true, + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeEncap, + TunnelType: ovsconfig.GeneveTunnel, + IPv4Enabled: true, + IPv6Enabled: true, + }, + nodeConfig: &config.NodeConfig{ + PodIPv4CIDR: ip.MustParseCIDR("172.16.10.0/24"), + PodIPv6CIDR: ip.MustParseCIDR("2001:ab03:cd04:55ef::/64"), + GatewayConfig: &config.GatewayConfig{ + Name: "antrea-gw0", + }, + }, + expectedCalls: func(mockIPTables *iptablestest.MockInterfaceMockRecorder) { + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.RawTable, antreaPreRoutingChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.RawTable, iptables.PreRoutingChain, []string{"-j", antreaPreRoutingChain, "-m", "comment", "--comment", "Antrea: jump to Antrea prerouting rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.RawTable, antreaOutputChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.RawTable, iptables.OutputChain, []string{"-j", antreaOutputChain, "-m", "comment", "--comment", "Antrea: jump to Antrea output rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.FilterTable, antreaForwardChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.FilterTable, iptables.ForwardChain, []string{"-j", antreaForwardChain, "-m", "comment", "--comment", "Antrea: jump to Antrea forwarding rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.NATTable, antreaPostRoutingChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.NATTable, iptables.PostRoutingChain, []string{"-j", antreaPostRoutingChain, "-m", "comment", "--comment", "Antrea: jump to Antrea postrouting rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.MangleTable, antreaMangleChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.MangleTable, iptables.PreRoutingChain, []string{"-j", antreaMangleChain, "-m", "comment", "--comment", "Antrea: jump to Antrea mangle rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.MangleTable, antreaOutputChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.MangleTable, iptables.OutputChain, []string{"-j", antreaOutputChain, "-m", "comment", "--comment", "Antrea: jump to Antrea output rules"}) + mockIPTables.EnsureChain(iptables.ProtocolDual, iptables.NATTable, antreaPreRoutingChain) + mockIPTables.AppendRule(iptables.ProtocolDual, iptables.NATTable, iptables.PreRoutingChain, []string{"-j", antreaPreRoutingChain, "-m", "comment", "--comment", "Antrea: jump to Antrea prerouting rules"}) + mockIPTables.Restore(`*raw +:ANTREA-PREROUTING - [0:0] +:ANTREA-OUTPUT - [0:0] +-A ANTREA-PREROUTING -m comment --comment "Antrea: do not track incoming encapsulation packets" -m udp -p udp --dport 6081 -m addrtype --dst-type LOCAL -j NOTRACK +-A ANTREA-OUTPUT -m comment --comment "Antrea: do not track outgoing encapsulation packets" -m udp -p udp --dport 6081 -m addrtype --src-type LOCAL -j NOTRACK +COMMIT +*mangle +:ANTREA-MANGLE - [0:0] +:ANTREA-OUTPUT - [0:0] +-A ANTREA-MANGLE -m comment --comment "Antrea: AWS, primary ENI" -i antrea-gw0 -j CONNMARK --restore-mark --nfmask 0x80 --ctmask 0x80 +-A ANTREA-OUTPUT -m comment --comment "Antrea: mark LOCAL output packets" -m addrtype --src-type LOCAL -o antrea-gw0 -j MARK --or-mark 0x80000000 +COMMIT +*filter +:ANTREA-FORWARD - [0:0] +-A ANTREA-FORWARD -m comment --comment "Antrea: accept packets from local Pods" -i antrea-gw0 -j ACCEPT +-A ANTREA-FORWARD -m comment --comment "Antrea: accept packets to local Pods" -o antrea-gw0 -j ACCEPT +COMMIT +*nat +:ANTREA-PREROUTING - [0:0] +:ANTREA-POSTROUTING - [0:0] +-A ANTREA-POSTROUTING -m comment --comment "Antrea: masquerade Pod to external packets" -s 172.16.10.0/24 -m set ! --match-set ANTREA-POD-IP dst ! -o antrea-gw0 -j MASQUERADE +-A ANTREA-POSTROUTING -m comment --comment "Antrea: masquerade LOCAL traffic" -o antrea-gw0 -m addrtype ! --src-type LOCAL --limit-iface-out -m addrtype --src-type LOCAL -j MASQUERADE --random-fully +-A ANTREA-PREROUTING -i antrea-gw0 -m comment --comment "Antrea: AWS, outbound connections" -j AWS-CONNMARK-CHAIN-0 +-A ANTREA-PREROUTING -m comment --comment "Antrea: AWS, CONNMARK (first packet)" -j CONNMARK --restore-mark --nfmask 0x80 --ctmask 0x80 +COMMIT +`, false, false) + mockIPTables.Restore(`*raw +:ANTREA-PREROUTING - [0:0] +:ANTREA-OUTPUT - [0:0] +-A ANTREA-PREROUTING -m comment --comment "Antrea: do not track incoming encapsulation packets" -m udp -p udp --dport 6081 -m addrtype --dst-type LOCAL -j NOTRACK +-A ANTREA-OUTPUT -m comment --comment "Antrea: do not track outgoing encapsulation packets" -m udp -p udp --dport 6081 -m addrtype --src-type LOCAL -j NOTRACK +COMMIT +*mangle +:ANTREA-MANGLE - [0:0] +:ANTREA-OUTPUT - [0:0] +-A ANTREA-OUTPUT -m comment --comment "Antrea: mark LOCAL output packets" -m addrtype --src-type LOCAL -o antrea-gw0 -j MARK --or-mark 0x80000000 +COMMIT +*filter +:ANTREA-FORWARD - [0:0] +-A ANTREA-FORWARD -m comment --comment "Antrea: accept packets from local Pods" -i antrea-gw0 -j ACCEPT +-A ANTREA-FORWARD -m comment --comment "Antrea: accept packets to local Pods" -o antrea-gw0 -j ACCEPT +COMMIT +*nat +:ANTREA-PREROUTING - [0:0] +:ANTREA-POSTROUTING - [0:0] +-A ANTREA-POSTROUTING -m comment --comment "Antrea: masquerade Pod to external packets" -s 2001:ab03:cd04:55ef::/64 -m set ! --match-set ANTREA-POD-IP6 dst ! -o antrea-gw0 -j MASQUERADE +-A ANTREA-POSTROUTING -m comment --comment "Antrea: masquerade LOCAL traffic" -o antrea-gw0 -m addrtype ! --src-type LOCAL --limit-iface-out -m addrtype --src-type LOCAL -j MASQUERADE --random-fully +COMMIT +`, false, true) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockIPTables := iptablestest.NewMockInterface(ctrl) + c := &Client{ipt: mockIPTables, + networkConfig: tt.networkConfig, + nodeConfig: tt.nodeConfig, + proxyAll: tt.proxyAll, + isCloudEKS: tt.isCloudEKS, + multicastEnabled: tt.multicastEnabled, + connectUplinkToBridge: tt.connectUplinkToBridge, + markToSNATIP: sync.Map{}, + } + for mark, snatIP := range tt.markToSNATIP { + c.markToSNATIP.Store(mark, net.ParseIP(snatIP)) + } + tt.expectedCalls(mockIPTables.EXPECT()) + assert.NoError(t, c.syncIPTables()) + }) + } +} + +func TestInitIPRoutes(t *testing.T) { + ipv4, nodeTransPortIPv4Addr, _ := net.ParseCIDR("172.16.10.2/24") + nodeTransPortIPv4Addr.IP = ipv4 + ipv6, nodeTransPortIPv6Addr, _ := net.ParseCIDR("fe80::e643:4bff:fe44:ee/64") + nodeTransPortIPv6Addr.IP = ipv6 + + tests := []struct { + name string + networkConfig *config.NetworkConfig + nodeConfig *config.NodeConfig + expectedCalls func(mockNetlink *netlinktest.MockInterfaceMockRecorder) + }{ + { + name: "networkPolicyOnly", + networkConfig: &config.NetworkConfig{TrafficEncapMode: config.TrafficEncapModeNetworkPolicyOnly}, + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{Name: "antrea-gw0"}, + NodeTransportIPv4Addr: nodeTransPortIPv4Addr, + NodeTransportIPv6Addr: nodeTransPortIPv6Addr, + }, + expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.LinkByName("antrea-gw0") + _, ipv4, _ := net.ParseCIDR("172.16.10.2/32") + mockNetlink.AddrReplace(gomock.Any(), &netlink.Addr{IPNet: ipv4}) + _, ipv6, _ := net.ParseCIDR("fe80::e643:4bff:fe44:ee/128") + mockNetlink.AddrReplace(gomock.Any(), &netlink.Addr{IPNet: ipv6}) + }, + }, + { + name: "encap", + networkConfig: &config.NetworkConfig{TrafficEncapMode: config.TrafficEncapModeEncap}, + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{Name: "antrea-gw0"}, + NodeTransportIPv4Addr: nodeTransPortIPv4Addr, + NodeTransportIPv6Addr: nodeTransPortIPv6Addr, + }, + expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) {}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockNetlink := netlinktest.NewMockInterface(ctrl) + c := &Client{netlink: mockNetlink, + networkConfig: tt.networkConfig, + nodeConfig: tt.nodeConfig, + } + tt.expectedCalls(mockNetlink.EXPECT()) + assert.NoError(t, c.initIPRoutes()) + }) + } +} + +func TestInitServiceIPRoutes(t *testing.T) { + tests := []struct { + name string + networkConfig *config.NetworkConfig + nodeConfig *config.NodeConfig + expectedCalls func(mockNetlink *netlinktest.MockInterfaceMockRecorder) + }{ + { + name: "encap", + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeEncap, + IPv4Enabled: true, + IPv6Enabled: true, + }, + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{Name: "antrea-gw0", LinkIndex: 10}, + }, + expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.NeighSet(&netlink.Neigh{ + LinkIndex: 10, + Family: netlink.FAMILY_V4, + State: netlink.NUD_PERMANENT, + IP: config.VirtualServiceIPv4, + HardwareAddr: globalVMAC, + }) + mockNetlink.RouteReplace(&netlink.Route{ + Dst: &net.IPNet{ + IP: config.VirtualServiceIPv4, + Mask: net.CIDRMask(32, 32), + }, + Scope: netlink.SCOPE_LINK, + LinkIndex: 10, + }) + mockNetlink.RouteReplace(&netlink.Route{ + Dst: &net.IPNet{ + IP: config.VirtualNodePortDNATIPv4, + Mask: net.CIDRMask(32, 32), + }, + Gw: config.VirtualServiceIPv4, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: 10, + }) + mockNetlink.NeighSet(&netlink.Neigh{ + LinkIndex: 10, + Family: netlink.FAMILY_V6, + State: netlink.NUD_PERMANENT, + IP: config.VirtualServiceIPv6, + HardwareAddr: globalVMAC, + }) + mockNetlink.RouteReplace(&netlink.Route{ + Dst: &net.IPNet{ + IP: config.VirtualServiceIPv6, + Mask: net.CIDRMask(128, 128), + }, + Scope: netlink.SCOPE_LINK, + LinkIndex: 10, + }) + mockNetlink.RouteReplace(&netlink.Route{ + Dst: &net.IPNet{ + IP: config.VirtualNodePortDNATIPv6, + Mask: net.CIDRMask(128, 128), + }, + Gw: config.VirtualServiceIPv6, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: 10, + }) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockNetlink := netlinktest.NewMockInterface(ctrl) + c := &Client{netlink: mockNetlink, + networkConfig: tt.networkConfig, + nodeConfig: tt.nodeConfig, + } + tt.expectedCalls(mockNetlink.EXPECT()) + assert.NoError(t, c.initServiceIPRoutes()) + }) + } +} + +func TestReconcile(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockNetlink := netlinktest.NewMockInterface(ctrl) + mockIPSet := ipsettest.NewMockInterface(ctrl) + c := &Client{netlink: mockNetlink, + ipset: mockIPSet, + proxyAll: true, + networkConfig: &config.NetworkConfig{}, + nodeConfig: &config.NodeConfig{ + PodIPv4CIDR: ip.MustParseCIDR("192.168.10.0/24"), + PodIPv6CIDR: ip.MustParseCIDR("2001:ab03:cd04:55ee:100a::/80"), + GatewayConfig: &config.GatewayConfig{LinkIndex: 10}, + }, + } + podCIDRs := []string{"192.168.0.0/24", "192.168.1.0/24", "2001:ab03:cd04:55ee:1001::/80", "2001:ab03:cd04:55ee:1002::/80"} + + mockIPSet.EXPECT().ListEntries(antreaPodIPSet).Return([]string{ + "192.168.0.0/24", // existing podCIDR, should not be deleted. + "192.168.2.0/24", // non-existing podCIDR, should be deleted. + }, nil) + mockIPSet.EXPECT().ListEntries(antreaPodIP6Set).Return([]string{ + "2001:ab03:cd04:55ee:1001::/80", // existing podCIDR, should not be deleted. + "2001:ab03:cd04:55ee:1003::/80", // non-existing podCIDR, should be deleted. + }, nil) + mockIPSet.EXPECT().DelEntry(antreaPodIPSet, "192.168.2.0/24") + mockIPSet.EXPECT().DelEntry(antreaPodIP6Set, "2001:ab03:cd04:55ee:1003::/80") + mockNetlink.EXPECT().RouteDel(&netlink.Route{Dst: ip.MustParseCIDR("192.168.2.0/24")}) + mockNetlink.EXPECT().RouteDel(&netlink.Route{Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:1003::/80")}) + + mockNetlink.EXPECT().RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{LinkIndex: 10}, netlink.RT_FILTER_OIF).Return([]netlink.Route{ + {Dst: ip.MustParseCIDR("192.168.10.0/24")}, // local podCIDR, should not be deleted. + {Dst: ip.MustParseCIDR("192.168.1.0/24")}, // existing podCIDR, should not be deleted. + {Dst: ip.MustParseCIDR("169.254.0.253/32")}, // service route, should not be deleted. + {Dst: ip.MustParseCIDR("192.168.11.0/24")}, // non-existing podCIDR, should be deleted. + }, nil) + mockNetlink.EXPECT().RouteListFiltered(netlink.FAMILY_V6, &netlink.Route{LinkIndex: 10}, netlink.RT_FILTER_OIF).Return([]netlink.Route{ + {Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:100a::/80")}, // local podCIDR, should not be deleted. + {Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:1001::1/128")}, // existing podCIDR, should not be deleted. + {Dst: ip.MustParseCIDR("fc01::aabb:ccdd:eeff/128")}, // service route, should not be deleted. + {Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:100b::/80")}, // non-existing podCIDR, should be deleted. + }, nil) + mockNetlink.EXPECT().RouteDel(&netlink.Route{Dst: ip.MustParseCIDR("192.168.11.0/24")}) + mockNetlink.EXPECT().RouteDel(&netlink.Route{Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:100b::/80")}) + + mockNetlink.EXPECT().NeighList(10, netlink.FAMILY_V6).Return([]netlink.Neigh{ + {IP: net.ParseIP("2001:ab03:cd04:55ee:1001::1")}, // existing podCIDR, should not be deleted. + {IP: net.ParseIP("fc01::aabb:ccdd:eeff")}, // virtual service IP, should not be deleted. + {IP: net.ParseIP("2001:ab03:cd04:55ee:100b::1")}, // non-existing podCIDR, should be deleted. + }, nil) + mockNetlink.EXPECT().NeighDel(&netlink.Neigh{IP: net.ParseIP("2001:ab03:cd04:55ee:100b::1")}) + assert.NoError(t, c.Reconcile(podCIDRs)) +} + +func TestAddRoutes(t *testing.T) { + ipv4, nodeTransPortIPv4Addr, _ := net.ParseCIDR("172.16.10.2/24") + nodeTransPortIPv4Addr.IP = ipv4 + ipv6, nodeTransPortIPv6Addr, _ := net.ParseCIDR("fe80::e643:4bff:fe44:ee/64") + nodeTransPortIPv6Addr.IP = ipv6 + + tests := []struct { + name string + networkConfig *config.NetworkConfig + nodeConfig *config.NodeConfig + podCIDR *net.IPNet + nodeName string + nodeIP net.IP + nodeGwIP net.IP + expectedIPSetCalls func(mockNetlink *ipsettest.MockInterfaceMockRecorder) + expectedNetlinkCalls func(mockNetlink *netlinktest.MockInterfaceMockRecorder) + }{ + { + name: "wireGuard IPv4", + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeEncap, + TrafficEncryptionMode: config.TrafficEncryptionModeWireGuard, + IPv4Enabled: true, + }, + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{ + Name: "antrea-gw0", + IPv4: net.ParseIP("1.1.1.1"), + LinkIndex: 10, + }, + WireGuardConfig: &config.WireGuardConfig{LinkIndex: 11}, + NodeTransportIPv4Addr: nodeTransPortIPv4Addr, + }, + podCIDR: ip.MustParseCIDR("192.168.10.0/24"), + nodeName: "node0", + nodeIP: net.ParseIP("1.1.1.10"), + nodeGwIP: net.ParseIP("192.168.10.1"), + expectedIPSetCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.AddEntry(antreaPodIPSet, "192.168.10.0/24") + }, + expectedNetlinkCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteReplace(&netlink.Route{ + Src: net.ParseIP("1.1.1.1"), + Dst: ip.MustParseCIDR("192.168.10.0/24"), + Scope: netlink.SCOPE_LINK, + LinkIndex: 11, + }) + }, + }, + { + name: "wireGuard IPv6", + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeEncap, + TrafficEncryptionMode: config.TrafficEncryptionModeWireGuard, + IPv6Enabled: true, + }, + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{ + Name: "antrea-gw0", + IPv6: net.ParseIP("fe80::e643:4bff:fe44:1"), + LinkIndex: 10, + }, + WireGuardConfig: &config.WireGuardConfig{LinkIndex: 11}, + NodeTransportIPv6Addr: nodeTransPortIPv6Addr, + }, + podCIDR: ip.MustParseCIDR("2001:ab03:cd04:55ee:1001::/80"), + nodeName: "node0", + nodeIP: net.ParseIP("fe80::e643:4bff:fe44:2"), + nodeGwIP: net.ParseIP("2001:ab03:cd04:55ee:1001::1"), + expectedIPSetCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.AddEntry(antreaPodIP6Set, "2001:ab03:cd04:55ee:1001::/80") + }, + expectedNetlinkCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteReplace(&netlink.Route{ + Src: net.ParseIP("fe80::e643:4bff:fe44:1"), + Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:1001::/80"), + Scope: netlink.SCOPE_LINK, + LinkIndex: 11, + }) + mockNetlink.RouteDel(&netlink.Route{ + Dst: &net.IPNet{IP: net.ParseIP("2001:ab03:cd04:55ee:1001::1"), Mask: net.CIDRMask(128, 128)}, + }) + mockNetlink.NeighDel(&netlink.Neigh{ + LinkIndex: 10, + Family: netlink.FAMILY_V6, + IP: net.ParseIP("2001:ab03:cd04:55ee:1001::1"), + }) + }, + }, + { + name: "encap IPv4", + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeEncap, + IPv4Enabled: true, + }, + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{ + Name: "antrea-gw0", + IPv4: net.ParseIP("1.1.1.1"), + LinkIndex: 10, + }, + NodeTransportIPv4Addr: nodeTransPortIPv4Addr, + }, + podCIDR: ip.MustParseCIDR("192.168.10.0/24"), + nodeName: "node0", + nodeIP: net.ParseIP("1.1.1.10"), + nodeGwIP: net.ParseIP("192.168.10.1"), + expectedIPSetCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.AddEntry(antreaPodIPSet, "192.168.10.0/24") + }, + expectedNetlinkCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteReplace(&netlink.Route{ + Gw: net.ParseIP("192.168.10.1"), + Dst: ip.MustParseCIDR("192.168.10.0/24"), + Flags: int(netlink.FLAG_ONLINK), + LinkIndex: 10, + }) + }, + }, + { + name: "encap IPv6", + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeEncap, + IPv6Enabled: true, + }, + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{ + Name: "antrea-gw0", + IPv6: net.ParseIP("fe80::e643:4bff:fe44:1"), + LinkIndex: 10, + }, + NodeTransportIPv6Addr: nodeTransPortIPv6Addr, + }, + podCIDR: ip.MustParseCIDR("2001:ab03:cd04:55ee:1001::/80"), + nodeName: "node0", + nodeIP: net.ParseIP("fe80::e643:4bff:fe44:2"), + nodeGwIP: net.ParseIP("2001:ab03:cd04:55ee:1001::1"), + expectedIPSetCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.AddEntry(antreaPodIP6Set, "2001:ab03:cd04:55ee:1001::/80") + }, + expectedNetlinkCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteReplace(&netlink.Route{ + Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:1001::1/128"), + LinkIndex: 10, + }) + mockNetlink.RouteReplace(&netlink.Route{ + Gw: net.ParseIP("2001:ab03:cd04:55ee:1001::1"), + Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:1001::/80"), + LinkIndex: 10, + }) + mockNetlink.NeighSet(&netlink.Neigh{ + LinkIndex: 10, + Family: netlink.FAMILY_V6, + State: netlink.NUD_PERMANENT, + IP: net.ParseIP("2001:ab03:cd04:55ee:1001::1"), + HardwareAddr: globalVMAC, + }) + }, + }, + { + name: "noencap IPv4, direct routing", + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeNoEncap, + IPv4Enabled: true, + }, + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{ + Name: "antrea-gw0", + IPv4: net.ParseIP("192.168.1.1"), + LinkIndex: 10, + }, + NodeTransportIPv4Addr: nodeTransPortIPv4Addr, + }, + podCIDR: ip.MustParseCIDR("192.168.10.0/24"), + nodeName: "node0", + nodeIP: net.ParseIP("172.16.10.3"), // In the same subnet as local Node IP. + nodeGwIP: net.ParseIP("192.168.10.1"), + expectedIPSetCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.AddEntry(antreaPodIPSet, "192.168.10.0/24") + }, + expectedNetlinkCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteReplace(&netlink.Route{ + Gw: net.ParseIP("172.16.10.3"), + Dst: ip.MustParseCIDR("192.168.10.0/24"), + }) + }, + }, + { + name: "noencap IPv4, no direct routing", + networkConfig: &config.NetworkConfig{ + TrafficEncapMode: config.TrafficEncapModeNoEncap, + IPv4Enabled: true, + }, + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{ + Name: "antrea-gw0", + IPv4: net.ParseIP("192.168.1.1"), + LinkIndex: 10, + }, + NodeTransportIPv4Addr: nodeTransPortIPv4Addr, + }, + podCIDR: ip.MustParseCIDR("192.168.10.0/24"), + nodeName: "node0", + nodeIP: net.ParseIP("172.16.11.3"), // In different subnet from local Node IP. + nodeGwIP: net.ParseIP("192.168.10.1"), + expectedIPSetCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.AddEntry(antreaPodIPSet, "192.168.10.0/24") + }, + expectedNetlinkCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) {}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockNetlink := netlinktest.NewMockInterface(ctrl) + mockIPSet := ipsettest.NewMockInterface(ctrl) + c := &Client{netlink: mockNetlink, + ipset: mockIPSet, + networkConfig: tt.networkConfig, + nodeConfig: tt.nodeConfig, + } + tt.expectedIPSetCalls(mockIPSet.EXPECT()) + tt.expectedNetlinkCalls(mockNetlink.EXPECT()) + assert.NoError(t, c.AddRoutes(tt.podCIDR, tt.nodeName, tt.nodeIP, tt.nodeGwIP)) + }) + } +} + +func TestDeleteRoutes(t *testing.T) { + tests := []struct { + name string + networkConfig *config.NetworkConfig + nodeConfig *config.NodeConfig + podCIDR *net.IPNet + existingNodeRoutes map[string][]*netlink.Route + existingNodeNeighbors map[string]*netlink.Neigh + nodeName string + expectedIPSetCalls func(mockNetlink *ipsettest.MockInterfaceMockRecorder) + expectedNetlinkCalls func(mockNetlink *netlinktest.MockInterfaceMockRecorder) + }{ + { + name: "IPv4", + podCIDR: ip.MustParseCIDR("192.168.10.0/24"), + existingNodeRoutes: map[string][]*netlink.Route{ + "192.168.10.0/24": {{Gw: net.ParseIP("172.16.10.3"), Dst: ip.MustParseCIDR("192.168.10.0/24")}}, + "192.168.11.0/24": {{Gw: net.ParseIP("172.16.10.4"), Dst: ip.MustParseCIDR("192.168.11.0/24")}}, + }, + expectedIPSetCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.DelEntry(antreaPodIPSet, "192.168.10.0/24") + }, + expectedNetlinkCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteDel(&netlink.Route{Gw: net.ParseIP("172.16.10.3"), Dst: ip.MustParseCIDR("192.168.10.0/24")}) + }, + }, + { + name: "IPv6", + podCIDR: ip.MustParseCIDR("2001:ab03:cd04:55ee:1001::/80"), + existingNodeRoutes: map[string][]*netlink.Route{ + "2001:ab03:cd04:55ee:1001::/80": {{Gw: net.ParseIP("fe80::e643:4bff:fe44:1"), Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:1001::/80")}}, + "2001:ab03:cd04:55ee:1002::/80": {{Gw: net.ParseIP("fe80::e643:4bff:fe44:2"), Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:1002::/80")}}, + }, + existingNodeNeighbors: map[string]*netlink.Neigh{}, + expectedIPSetCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.DelEntry(antreaPodIP6Set, "2001:ab03:cd04:55ee:1001::/80") + }, + expectedNetlinkCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteDel(&netlink.Route{Gw: net.ParseIP("fe80::e643:4bff:fe44:1"), Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:1001::/80")}) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockNetlink := netlinktest.NewMockInterface(ctrl) + mockIPSet := ipsettest.NewMockInterface(ctrl) + c := &Client{netlink: mockNetlink, + ipset: mockIPSet, + networkConfig: tt.networkConfig, + nodeConfig: tt.nodeConfig, + nodeRoutes: sync.Map{}, + nodeNeighbors: sync.Map{}, + } + for podCIDR, nodeRoute := range tt.existingNodeRoutes { + c.nodeRoutes.Store(podCIDR, nodeRoute) + } + for podCIDR, nodeNeighbor := range tt.existingNodeNeighbors { + c.nodeNeighbors.Store(podCIDR, nodeNeighbor) + } + tt.expectedIPSetCalls(mockIPSet.EXPECT()) + tt.expectedNetlinkCalls(mockNetlink.EXPECT()) + assert.NoError(t, c.DeleteRoutes(tt.podCIDR)) + }) + } +} + +func TestMigrateRoutesToGw(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockNetlink := netlinktest.NewMockInterface(ctrl) + mockIPSet := ipsettest.NewMockInterface(ctrl) + + gwLinkName := "antrea-gw0" + gwLink := &netlink.Device{LinkAttrs: netlink.LinkAttrs{Index: 11}} + linkName := "eth0" + link := &netlink.Device{LinkAttrs: netlink.LinkAttrs{Index: 10}} + linkAddr1, _ := netlink.ParseAddr("192.168.10.1/32") + linkAddr2, _ := netlink.ParseAddr("169.254.0.2/32") // LinkLocalUnicast address should not be migrated. + linkAddr3, _ := netlink.ParseAddr("2001:ab03:cd04:55ee:1001::1/80") + linkAddr4, _ := netlink.ParseAddr("fe80:ab03:cd04:55ee:1001::1/80") // LinkLocalUnicast address should not be migrated. + + mockNetlink.EXPECT().LinkByName(gwLinkName).Return(gwLink, nil) + mockNetlink.EXPECT().LinkByName(linkName).Return(link, nil) + mockNetlink.EXPECT().RouteList(link, netlink.FAMILY_V4).Return([]netlink.Route{ + {Gw: net.ParseIP("172.16.1.10"), Dst: ip.MustParseCIDR("192.168.10.0/24"), LinkIndex: 10}, + }, nil) + mockNetlink.EXPECT().RouteList(link, netlink.FAMILY_V6).Return([]netlink.Route{ + {Gw: net.ParseIP("fe80::e643:4bff:fe44:1"), Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:1001::/80"), LinkIndex: 10}, + }, nil) + mockNetlink.EXPECT().RouteReplace(&netlink.Route{Gw: net.ParseIP("172.16.1.10"), Dst: ip.MustParseCIDR("192.168.10.0/24"), LinkIndex: 11}) + mockNetlink.EXPECT().RouteReplace(&netlink.Route{Gw: net.ParseIP("fe80::e643:4bff:fe44:1"), Dst: ip.MustParseCIDR("2001:ab03:cd04:55ee:1001::/80"), LinkIndex: 11}) + mockNetlink.EXPECT().AddrList(link, netlink.FAMILY_V4).Return([]netlink.Addr{*linkAddr1, *linkAddr2}, nil) + mockNetlink.EXPECT().AddrList(link, netlink.FAMILY_V6).Return([]netlink.Addr{*linkAddr3, *linkAddr4}, nil) + mockNetlink.EXPECT().AddrDel(link, linkAddr1) + mockNetlink.EXPECT().AddrReplace(gwLink, linkAddr1) + mockNetlink.EXPECT().AddrDel(link, linkAddr3) + mockNetlink.EXPECT().AddrReplace(gwLink, linkAddr3) + + c := &Client{ + netlink: mockNetlink, + ipset: mockIPSet, + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{Name: gwLinkName}, + }, + } + c.MigrateRoutesToGw(linkName) +} + +func TestUnMigrateRoutesToGw(t *testing.T) { + gwLink := &netlink.Device{LinkAttrs: netlink.LinkAttrs{Index: 11}} + link := &netlink.Device{LinkAttrs: netlink.LinkAttrs{Index: 10}} + tests := []struct { + name string + nodeConfig *config.NodeConfig + route string + link string + expectedCalls func(mockNetlink *netlinktest.MockInterfaceMockRecorder) + }{ + { + name: "link provided", + route: "192.168.10.0/24", + link: "eth0", + nodeConfig: &config.NodeConfig{GatewayConfig: &config.GatewayConfig{Name: "antrea-gw0"}}, + expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.LinkByName("antrea-gw0").Return(gwLink, nil) + mockNetlink.LinkByName("eth0").Return(link, nil) + mockNetlink.RouteList(gwLink, netlink.FAMILY_V4).Return([]netlink.Route{ + {Gw: net.ParseIP("172.16.1.10"), Dst: ip.MustParseCIDR("192.168.10.0/24"), LinkIndex: 11}, + }, nil) + mockNetlink.RouteReplace(&netlink.Route{Gw: net.ParseIP("172.16.1.10"), Dst: ip.MustParseCIDR("192.168.10.0/24"), LinkIndex: 10}) + }, + }, + { + name: "link not provided", + route: "192.168.10.0/24", + link: "", + nodeConfig: &config.NodeConfig{GatewayConfig: &config.GatewayConfig{Name: "antrea-gw0"}}, + expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.LinkByName("antrea-gw0").Return(gwLink, nil) + mockNetlink.RouteList(gwLink, netlink.FAMILY_V4).Return([]netlink.Route{ + {Gw: net.ParseIP("172.16.1.10"), Dst: ip.MustParseCIDR("192.168.10.0/24"), LinkIndex: 11}, + }, nil) + mockNetlink.RouteDel(&netlink.Route{Gw: net.ParseIP("172.16.1.10"), Dst: ip.MustParseCIDR("192.168.10.0/24"), LinkIndex: 11}) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockNetlink := netlinktest.NewMockInterface(ctrl) + c := &Client{ + netlink: mockNetlink, + nodeConfig: tt.nodeConfig, + } + tt.expectedCalls(mockNetlink.EXPECT()) + c.UnMigrateRoutesFromGw(ip.MustParseCIDR(tt.route), tt.link) + }) + } +} + +func TestAddSNATRule(t *testing.T) { + tests := []struct { + name string + networkConfig *config.NetworkConfig + nodeConfig *config.NodeConfig + snatIP net.IP + mark uint32 + expectedCalls func(mockIPTables *iptablestest.MockInterfaceMockRecorder) + }{ + { + name: "IPv4", + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{ + Name: "antrea-gw0", + }, + }, + snatIP: net.ParseIP("1.1.1.1"), + mark: 10, + expectedCalls: func(mockIPTables *iptablestest.MockInterfaceMockRecorder) { + mockIPTables.InsertRule(iptables.ProtocolIPv4, iptables.NATTable, antreaPostRoutingChain, []string{ + "-m", "comment", "--comment", "Antrea: SNAT Pod to external packets", + "!", "-o", "antrea-gw0", + "-m", "mark", "--mark", fmt.Sprintf("%#08x/%#08x", 10, types.SNATIPMarkMask), + "-j", iptables.SNATTarget, "--to", "1.1.1.1", + }) + }, + }, + { + name: "IPv6", + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{ + Name: "antrea-gw0", + }, + }, + snatIP: net.ParseIP("fe80::e643:4bff:fe44:1"), + mark: 11, + expectedCalls: func(mockIPTables *iptablestest.MockInterfaceMockRecorder) { + mockIPTables.InsertRule(iptables.ProtocolIPv6, iptables.NATTable, antreaPostRoutingChain, []string{ + "-m", "comment", "--comment", "Antrea: SNAT Pod to external packets", + "!", "-o", "antrea-gw0", + "-m", "mark", "--mark", fmt.Sprintf("%#08x/%#08x", 11, types.SNATIPMarkMask), + "-j", iptables.SNATTarget, "--to", "fe80::e643:4bff:fe44:1", + }) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockIPTables := iptablestest.NewMockInterface(ctrl) + c := &Client{ipt: mockIPTables, + nodeConfig: tt.nodeConfig, + } + tt.expectedCalls(mockIPTables.EXPECT()) + assert.NoError(t, c.AddSNATRule(tt.snatIP, tt.mark)) + }) + } +} + +func TestDeleteSNATRule(t *testing.T) { + tests := []struct { + name string + networkConfig *config.NetworkConfig + markToSNATIP map[uint32]net.IP + nodeConfig *config.NodeConfig + mark uint32 + expectedCalls func(mockIPTables *iptablestest.MockInterfaceMockRecorder) + }{ + { + name: "IPv4", + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{ + Name: "antrea-gw0", + }, + }, + markToSNATIP: map[uint32]net.IP{ + 10: net.ParseIP("1.1.1.1"), + 11: net.ParseIP("1.1.1.2"), + }, + mark: 10, + expectedCalls: func(mockIPTables *iptablestest.MockInterfaceMockRecorder) { + mockIPTables.DeleteRule(iptables.ProtocolIPv4, iptables.NATTable, antreaPostRoutingChain, []string{ + "-m", "comment", "--comment", "Antrea: SNAT Pod to external packets", + "!", "-o", "antrea-gw0", + "-m", "mark", "--mark", fmt.Sprintf("%#08x/%#08x", 10, types.SNATIPMarkMask), + "-j", iptables.SNATTarget, "--to", "1.1.1.1", + }) + }, + }, + { + name: "IPv6", + nodeConfig: &config.NodeConfig{ + GatewayConfig: &config.GatewayConfig{ + Name: "antrea-gw0", + }, + }, + markToSNATIP: map[uint32]net.IP{ + 10: net.ParseIP("fe80::e643:4bff:fe44:1"), + 11: net.ParseIP("fe80::e643:4bff:fe44:2"), + }, + mark: 11, + expectedCalls: func(mockIPTables *iptablestest.MockInterfaceMockRecorder) { + mockIPTables.DeleteRule(iptables.ProtocolIPv6, iptables.NATTable, antreaPostRoutingChain, []string{ + "-m", "comment", "--comment", "Antrea: SNAT Pod to external packets", + "!", "-o", "antrea-gw0", + "-m", "mark", "--mark", fmt.Sprintf("%#08x/%#08x", 11, types.SNATIPMarkMask), + "-j", iptables.SNATTarget, "--to", "fe80::e643:4bff:fe44:2", + }) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockIPTables := iptablestest.NewMockInterface(ctrl) + c := &Client{ + ipt: mockIPTables, + nodeConfig: tt.nodeConfig, + markToSNATIP: sync.Map{}, + } + for mark, snatIP := range tt.markToSNATIP { + c.markToSNATIP.Store(mark, snatIP) + } + tt.expectedCalls(mockIPTables.EXPECT()) + assert.NoError(t, c.DeleteSNATRule(tt.mark)) + }) + } +} + +func TestAddNodePort(t *testing.T) { + tests := []struct { + name string + nodePortAddresses []net.IP + port uint16 + protocol openflow.Protocol + expectedCalls func(ipset *ipsettest.MockInterfaceMockRecorder) + }{ + { + name: "ipv4 tcp", + nodePortAddresses: []net.IP{ + net.ParseIP("1.1.1.1"), + net.ParseIP("1.1.2.2"), + }, + port: 30000, + protocol: openflow.ProtocolTCP, + expectedCalls: func(ipset *ipsettest.MockInterfaceMockRecorder) { + ipset.AddEntry(antreaNodePortIPSet, "1.1.1.1,tcp:30000") + ipset.AddEntry(antreaNodePortIPSet, "1.1.2.2,tcp:30000") + }, + }, + { + name: "ipv6 udp", + nodePortAddresses: []net.IP{ + net.ParseIP("fd00:1234:5678:dead:beaf::1"), + net.ParseIP("fd00:1234:5678:dead:beaf::2"), + }, + port: 30001, + protocol: openflow.ProtocolUDPv6, + expectedCalls: func(ipset *ipsettest.MockInterfaceMockRecorder) { + ipset.AddEntry(antreaNodePortIP6Set, "fd00:1234:5678:dead:beaf::1,udp:30001") + ipset.AddEntry(antreaNodePortIP6Set, "fd00:1234:5678:dead:beaf::2,udp:30001") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + ipset := ipsettest.NewMockInterface(ctrl) + c := &Client{ipset: ipset} + tt.expectedCalls(ipset.EXPECT()) + assert.NoError(t, c.AddNodePort(tt.nodePortAddresses, tt.port, tt.protocol)) + }) + } +} + +func TestDeleteNodePort(t *testing.T) { + tests := []struct { + name string + nodePortAddresses []net.IP + port uint16 + protocol openflow.Protocol + expectedCalls func(ipset *ipsettest.MockInterfaceMockRecorder) + }{ + { + name: "ipv4 tcp", + nodePortAddresses: []net.IP{ + net.ParseIP("1.1.1.1"), + net.ParseIP("1.1.2.2"), + }, + port: 30000, + protocol: openflow.ProtocolTCP, + expectedCalls: func(ipset *ipsettest.MockInterfaceMockRecorder) { + ipset.DelEntry(antreaNodePortIPSet, "1.1.1.1,tcp:30000") + ipset.DelEntry(antreaNodePortIPSet, "1.1.2.2,tcp:30000") + }, + }, + { + name: "ipv6 udp", + nodePortAddresses: []net.IP{ + net.ParseIP("fd00:1234:5678:dead:beaf::1"), + net.ParseIP("fd00:1234:5678:dead:beaf::2"), + }, + port: 30001, + protocol: openflow.ProtocolUDPv6, + expectedCalls: func(ipset *ipsettest.MockInterfaceMockRecorder) { + ipset.DelEntry(antreaNodePortIP6Set, "fd00:1234:5678:dead:beaf::1,udp:30001") + ipset.DelEntry(antreaNodePortIP6Set, "fd00:1234:5678:dead:beaf::2,udp:30001") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + ipset := ipsettest.NewMockInterface(ctrl) + c := &Client{ipset: ipset} + tt.expectedCalls(ipset.EXPECT()) + assert.NoError(t, c.DeleteNodePort(tt.nodePortAddresses, tt.port, tt.protocol)) + }) + } +} + +func TestAddClusterIPRoute(t *testing.T) { + nodeConfig := &config.NodeConfig{GatewayConfig: &config.GatewayConfig{LinkIndex: 10}} + tests := []struct { + name string + clusterIPs []string + expectedCalls func(mockNetlink *netlinktest.MockInterfaceMockRecorder) + }{ + { + name: "IPv4", + clusterIPs: []string{"10.96.0.1", "10.96.0.10"}, + expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteReplace(&netlink.Route{ + Dst: &net.IPNet{IP: net.ParseIP("10.96.0.1"), Mask: net.CIDRMask(32, 32)}, + Gw: config.VirtualServiceIPv4, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: 10, + }) + mockNetlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{LinkIndex: 10}, netlink.RT_FILTER_OIF).Return([]netlink.Route{ + {Dst: ip.MustParseCIDR("10.96.0.0/24"), Gw: config.VirtualServiceIPv4}, + }, nil) + mockNetlink.RouteListFiltered(netlink.FAMILY_V6, &netlink.Route{LinkIndex: 10}, netlink.RT_FILTER_OIF).Return([]netlink.Route{}, nil) + mockNetlink.RouteDel(&netlink.Route{ + Dst: ip.MustParseCIDR("10.96.0.0/24"), Gw: config.VirtualServiceIPv4, + }) + mockNetlink.RouteReplace(&netlink.Route{ + Dst: &net.IPNet{IP: net.ParseIP("10.96.0.0").To4(), Mask: net.CIDRMask(28, 32)}, + Gw: config.VirtualServiceIPv4, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: 10, + }) + mockNetlink.RouteDel(&netlink.Route{ + Dst: &net.IPNet{IP: net.ParseIP("10.96.0.1"), Mask: net.CIDRMask(32, 32)}, + Gw: config.VirtualServiceIPv4, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: 10, + }) + }, + }, + { + name: "IPv6", + clusterIPs: []string{"fd00:1234:5678:dead:beaf::1", "fd00:1234:5678:dead:beaf::a"}, + expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteReplace(&netlink.Route{ + Dst: &net.IPNet{IP: net.ParseIP("fd00:1234:5678:dead:beaf::1"), Mask: net.CIDRMask(128, 128)}, + Gw: config.VirtualServiceIPv6, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: 10, + }) + mockNetlink.RouteListFiltered(netlink.FAMILY_V4, &netlink.Route{LinkIndex: 10}, netlink.RT_FILTER_OIF).Return([]netlink.Route{}, nil) + mockNetlink.RouteListFiltered(netlink.FAMILY_V6, &netlink.Route{LinkIndex: 10}, netlink.RT_FILTER_OIF).Return([]netlink.Route{ + {Dst: ip.MustParseCIDR("fd00:1234:5678:dead:beaf::/80"), Gw: config.VirtualServiceIPv6}, + }, nil) + mockNetlink.RouteDel(&netlink.Route{ + Dst: ip.MustParseCIDR("fd00:1234:5678:dead:beaf::/80"), Gw: config.VirtualServiceIPv6, + }) + mockNetlink.RouteReplace(&netlink.Route{ + Dst: &net.IPNet{IP: net.ParseIP("fd00:1234:5678:dead::"), Mask: net.CIDRMask(64, 128)}, + Gw: config.VirtualServiceIPv6, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: 10, + }) + mockNetlink.RouteDel(&netlink.Route{ + Dst: &net.IPNet{IP: net.ParseIP("fd00:1234:5678:dead:beaf::1"), Mask: net.CIDRMask(128, 128)}, + Gw: config.VirtualServiceIPv6, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: 10, + }) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockNetlink := netlinktest.NewMockInterface(ctrl) + c := &Client{ + netlink: mockNetlink, + nodeConfig: nodeConfig, + } + tt.expectedCalls(mockNetlink.EXPECT()) + + for _, clusterIP := range tt.clusterIPs { + assert.NoError(t, c.AddClusterIPRoute(net.ParseIP(clusterIP))) + } + }) + } +} + +func TestAddLoadBalancer(t *testing.T) { + nodeConfig := &config.NodeConfig{GatewayConfig: &config.GatewayConfig{LinkIndex: 10}} + tests := []struct { + name string + externalIP string + expectedCalls func(mockNetlink *netlinktest.MockInterfaceMockRecorder) + }{ + { + name: "IPv4", + externalIP: "1.1.1.1", + expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteReplace(&netlink.Route{ + Dst: &net.IPNet{ + IP: net.ParseIP("1.1.1.1"), + Mask: net.CIDRMask(32, 32), + }, + Gw: config.VirtualServiceIPv4, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: 10, + }) + }, + }, + { + name: "IPv6", + externalIP: "fd00:1234:5678:dead:beaf::1", + expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteReplace(&netlink.Route{ + Dst: &net.IPNet{IP: net.ParseIP("fd00:1234:5678:dead:beaf::1"), Mask: net.CIDRMask(128, 128)}, + Gw: config.VirtualServiceIPv6, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: 10, + }) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockNetlink := netlinktest.NewMockInterface(ctrl) + c := &Client{ + netlink: mockNetlink, + nodeConfig: nodeConfig, + } + tt.expectedCalls(mockNetlink.EXPECT()) + + assert.NoError(t, c.AddLoadBalancer(net.ParseIP(tt.externalIP))) + }) + } +} + +func TestDeleteLoadBalancer(t *testing.T) { + nodeConfig := &config.NodeConfig{GatewayConfig: &config.GatewayConfig{LinkIndex: 10}} + tests := []struct { + name string + externalIP string + expectedCalls func(mockNetlink *netlinktest.MockInterfaceMockRecorder) + }{ + { + name: "IPv4", + externalIP: "1.1.1.1", + expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteDel(&netlink.Route{ + Dst: &net.IPNet{ + IP: net.ParseIP("1.1.1.1"), + Mask: net.CIDRMask(32, 32), + }, + Gw: config.VirtualServiceIPv4, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: 10, + }) + }, + }, + { + name: "IPv6", + externalIP: "fd00:1234:5678:dead:beaf::1", + expectedCalls: func(mockNetlink *netlinktest.MockInterfaceMockRecorder) { + mockNetlink.RouteDel(&netlink.Route{ + Dst: &net.IPNet{IP: net.ParseIP("fd00:1234:5678:dead:beaf::1"), Mask: net.CIDRMask(128, 128)}, + Gw: config.VirtualServiceIPv6, + Scope: netlink.SCOPE_UNIVERSE, + LinkIndex: 10, + }) + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockNetlink := netlinktest.NewMockInterface(ctrl) + c := &Client{ + netlink: mockNetlink, + nodeConfig: nodeConfig, + } + tt.expectedCalls(mockNetlink.EXPECT()) + + assert.NoError(t, c.DeleteLoadBalancer(net.ParseIP(tt.externalIP))) + }) + } +} + +func TestAddLocalAntreaFlexibleIPAMPodRule(t *testing.T) { + tests := []struct { + name string + nodeConfig *config.NodeConfig + connectUplinkToBridge bool + podAddresses []net.IP + expectedCalls func(mockIPSet *ipsettest.MockInterfaceMockRecorder) + }{ + { + name: "connectUplinkToBridge=false", + nodeConfig: &config.NodeConfig{ + PodIPv4CIDR: ip.MustParseCIDR("1.1.1.0/24"), + PodIPv6CIDR: ip.MustParseCIDR("aabb::/64"), + }, + connectUplinkToBridge: false, + podAddresses: []net.IP{net.ParseIP("1.1.1.1"), net.ParseIP("aabb::1")}, + expectedCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) {}, + }, + { + name: "connectUplinkToBridge=true,nodeIPAMPod", + nodeConfig: &config.NodeConfig{ + PodIPv4CIDR: ip.MustParseCIDR("1.1.1.0/24"), + PodIPv6CIDR: ip.MustParseCIDR("aabb::/64"), + }, + connectUplinkToBridge: false, + podAddresses: []net.IP{net.ParseIP("1.1.1.1"), net.ParseIP("aabb::1")}, + expectedCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) {}, + }, + { + name: "connectUplinkToBridge=true,antreaIPAMPod", + nodeConfig: &config.NodeConfig{ + PodIPv4CIDR: ip.MustParseCIDR("1.1.1.0/24"), + PodIPv6CIDR: ip.MustParseCIDR("aabb::/64"), + }, + connectUplinkToBridge: true, + podAddresses: []net.IP{net.ParseIP("1.1.2.1"), net.ParseIP("aabc::1")}, + expectedCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.AddEntry(localAntreaFlexibleIPAMPodIPSet, "1.1.2.1") + mockIPSet.AddEntry(localAntreaFlexibleIPAMPodIP6Set, "aabc::1") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockIPSet := ipsettest.NewMockInterface(ctrl) + c := &Client{ + ipset: mockIPSet, + nodeConfig: tt.nodeConfig, + connectUplinkToBridge: tt.connectUplinkToBridge, + } + tt.expectedCalls(mockIPSet.EXPECT()) + + assert.NoError(t, c.AddLocalAntreaFlexibleIPAMPodRule(tt.podAddresses)) + }) + } +} + +func TestDeleteLocalAntreaFlexibleIPAMPodRule(t *testing.T) { + nodeConfig := &config.NodeConfig{GatewayConfig: &config.GatewayConfig{LinkIndex: 10}} + tests := []struct { + name string + connectUplinkToBridge bool + podAddresses []net.IP + expectedCalls func(mockIPSet *ipsettest.MockInterfaceMockRecorder) + }{ + { + name: "connectUplinkToBridge=false", + connectUplinkToBridge: false, + podAddresses: []net.IP{net.ParseIP("1.1.1.1"), net.ParseIP("aabb::1")}, + expectedCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) {}, + }, + { + name: "connectUplinkToBridge=true", + connectUplinkToBridge: true, + podAddresses: []net.IP{net.ParseIP("1.1.1.1"), net.ParseIP("aabb::1")}, + expectedCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.DelEntry(localAntreaFlexibleIPAMPodIPSet, "1.1.1.1") + mockIPSet.DelEntry(localAntreaFlexibleIPAMPodIP6Set, "aabb::1") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockIPSet := ipsettest.NewMockInterface(ctrl) + c := &Client{ + ipset: mockIPSet, + nodeConfig: nodeConfig, + connectUplinkToBridge: tt.connectUplinkToBridge, + } + tt.expectedCalls(mockIPSet.EXPECT()) + + assert.NoError(t, c.DeleteLocalAntreaFlexibleIPAMPodRule(tt.podAddresses)) + }) + } +} + +func TestAddAndDeleteNodeIP(t *testing.T) { + tests := []struct { + name string + multicastEnabled bool + networkConfig *config.NetworkConfig + podCIDR *net.IPNet + nodeIP net.IP + expectedCalls func(mockIPSet *ipsettest.MockInterfaceMockRecorder) + }{ + { + name: "IPv4", + multicastEnabled: true, + networkConfig: &config.NetworkConfig{TrafficEncapMode: config.TrafficEncapModeEncap}, + podCIDR: ip.MustParseCIDR("192.168.0.0/24"), + nodeIP: net.ParseIP("1.1.1.1"), + expectedCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.AddEntry(clusterNodeIPSet, "1.1.1.1") + mockIPSet.DelEntry(clusterNodeIPSet, "1.1.1.1") + }, + }, + { + name: "IPv6", + multicastEnabled: true, + networkConfig: &config.NetworkConfig{TrafficEncapMode: config.TrafficEncapModeEncap}, + podCIDR: ip.MustParseCIDR("1122:3344::/80"), + nodeIP: net.ParseIP("aabb:ccdd::1"), + expectedCalls: func(mockIPSet *ipsettest.MockInterfaceMockRecorder) { + mockIPSet.AddEntry(clusterNodeIP6Set, "aabb:ccdd::1") + mockIPSet.DelEntry(clusterNodeIP6Set, "aabb:ccdd::1") + }, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + ctrl := gomock.NewController(t) + defer ctrl.Finish() + mockIPSet := ipsettest.NewMockInterface(ctrl) + c := &Client{ + ipset: mockIPSet, + networkConfig: tt.networkConfig, + multicastEnabled: tt.multicastEnabled, + } + tt.expectedCalls(mockIPSet.EXPECT()) + + ipv6 := tt.nodeIP.To4() == nil + assert.NoError(t, c.addNodeIP(tt.podCIDR, tt.nodeIP)) + var exists bool + if ipv6 { + _, exists = c.clusterNodeIP6s.Load(tt.podCIDR.String()) + } else { + _, exists = c.clusterNodeIPs.Load(tt.podCIDR.String()) + } + assert.True(t, exists) + + assert.NoError(t, c.deleteNodeIP(tt.podCIDR)) + if ipv6 { + _, exists = c.clusterNodeIP6s.Load(tt.podCIDR.String()) + } else { + _, exists = c.clusterNodeIPs.Load(tt.podCIDR.String()) + } + assert.False(t, exists) + }) + } +} diff --git a/pkg/agent/route/route_windows.go b/pkg/agent/route/route_windows.go index 3e58e1556bd..7722774f68c 100644 --- a/pkg/agent/route/route_windows.go +++ b/pkg/agent/route/route_windows.go @@ -122,7 +122,7 @@ func (c *Client) initServiceIPRoutes() error { // Reconcile removes the orphaned routes and related configuration based on the desired podCIDRs and Service IPs. Only // the route entries on the host gateway interface are stored in the cache. -func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error { +func (c *Client) Reconcile(podCIDRs []string) error { desiredPodCIDRs := sets.NewString(podCIDRs...) routes, err := c.listRoutes() if err != nil { @@ -133,8 +133,8 @@ func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error { c.hostRoutes.Store(dst, rt) continue } - if _, ok := svcIPs[dst]; ok { - c.hostRoutes.Store(dst, rt) + // Don't delete the routes which are added by AntreaProxy. + if c.isServiceRoute(rt) { continue } err := util.RemoveNetRoute(rt) @@ -260,8 +260,8 @@ func (c *Client) addVirtualServiceIPRoute(isIPv6 bool) error { // TODO: Follow the code style in Linux that maintains one Service CIDR. func (c *Client) addServiceRoute(svcIP net.IP) error { - obj, found := c.hostRoutes.Load(svcIP.String()) svcIPNet := util.NewIPNet(svcIP) + obj, found := c.hostRoutes.Load(svcIPNet.String()) // Route: Service IP -> VirtualServiceIPv4 (169.254.0.253) route := &util.Route{ @@ -288,7 +288,7 @@ func (c *Client) addServiceRoute(svcIP net.IP) error { return err } - c.hostRoutes.Store(route.DestinationSubnet.String(), route) + c.hostRoutes.Store(svcIPNet.String(), route) klog.V(2).InfoS("Added Service route", "ServiceIP", route.DestinationSubnet, "GatewayIP", route.GatewayAddress) return nil } @@ -305,7 +305,7 @@ func (c *Client) deleteServiceRoute(svcIP net.IP) error { if err := util.RemoveNetRoute(rt); err != nil { return err } - c.hostRoutes.Delete(svcIP.String()) + c.hostRoutes.Delete(svcIPNet.String()) klog.V(2).InfoS("Deleted Service route from host gateway", "DestinationIP", svcIP) return nil } @@ -332,6 +332,15 @@ func (c *Client) UnMigrateRoutesFromGw(route *net.IPNet, linkName string) error func (c *Client) Run(stopCh <-chan struct{}) { } +func (c *Client) isServiceRoute(route *util.Route) bool { + // If the destination IP or gateway IP is the virtual Service IP, then it is the Service route added by AntreaProxy. + if route.DestinationSubnet != nil && route.DestinationSubnet.IP.Equal(config.VirtualServiceIPv4) || + route.GatewayAddress != nil && route.GatewayAddress.Equal(config.VirtualServiceIPv4) { + return true + } + return false +} + func (c *Client) listRoutes() (map[string]*util.Route, error) { routes, err := util.GetNetRoutesAll() if err != nil { @@ -394,22 +403,12 @@ func (c *Client) DeleteNodePort(nodePortAddresses []net.IP, port uint16, protoco return util.RemoveNetNatStaticMapping(antreaNatNodePort, "0.0.0.0", port, string(protocol)) } -func (c *Client) AddLoadBalancer(externalIPs []string) error { - for _, svcIPStr := range externalIPs { - if err := c.addServiceRoute(net.ParseIP(svcIPStr)); err != nil { - return err - } - } - return nil +func (c *Client) AddLoadBalancer(externalIP net.IP) error { + return c.addServiceRoute(externalIP) } -func (c *Client) DeleteLoadBalancer(externalIPs []string) error { - for _, svcIPStr := range externalIPs { - if err := c.deleteServiceRoute(net.ParseIP(svcIPStr)); err != nil { - return err - } - } - return nil +func (c *Client) DeleteLoadBalancer(externalIP net.IP) error { + return c.deleteServiceRoute(externalIP) } func (c *Client) AddLocalAntreaFlexibleIPAMPodRule(podAddresses []net.IP) error { diff --git a/pkg/agent/route/route_windows_test.go b/pkg/agent/route/route_windows_test.go index bfec5a310f7..b0e1c00020f 100644 --- a/pkg/agent/route/route_windows_test.go +++ b/pkg/agent/route/route_windows_test.go @@ -89,14 +89,20 @@ func TestRouteOperation(t *testing.T) { route3, err := util.GetNetRoutes(gwLink, svcIPNet1) require.Nil(t, err) assert.Equal(t, 1, len(route3)) + obj, found := client.hostRoutes.Load(svcIPNet1.String()) + assert.True(t, found) + assert.EqualValues(t, route3[0], *obj.(*util.Route)) err = client.AddClusterIPRoute(svcIP2) require.Nil(t, err) route4, err := util.GetNetRoutes(gwLink, svcIPNet2) require.Nil(t, err) assert.Equal(t, 1, len(route4)) + obj, found = client.hostRoutes.Load(svcIPNet2.String()) + assert.True(t, found) + assert.EqualValues(t, route4[0], *obj.(*util.Route)) - err = client.Reconcile([]string{dest2}, map[string]bool{svcIPNet1.String(): true}) + err = client.Reconcile([]string{dest2}) require.Nil(t, err) routes5, err := util.GetNetRoutes(gwLink, destCIDR1) @@ -105,7 +111,7 @@ func TestRouteOperation(t *testing.T) { routes6, err := util.GetNetRoutes(gwLink, svcIPNet2) require.Nil(t, err) - assert.Equal(t, 0, len(routes6)) + assert.Equal(t, 1, len(routes6)) err = client.DeleteRoutes(destCIDR2) require.Nil(t, err) @@ -118,4 +124,6 @@ func TestRouteOperation(t *testing.T) { routes8, err := util.GetNetRoutes(gwLink, svcIPNet1) require.Nil(t, err) assert.Equal(t, 0, len(routes8)) + _, found = client.hostRoutes.Load(svcIPNet1.String()) + assert.False(t, found) } diff --git a/pkg/agent/route/testing/mock_route.go b/pkg/agent/route/testing/mock_route.go index 67ee9270b63..ba8086898f5 100644 --- a/pkg/agent/route/testing/mock_route.go +++ b/pkg/agent/route/testing/mock_route.go @@ -1,4 +1,4 @@ -// Copyright 2021 Antrea Authors +// Copyright 2023 Antrea Authors // // Licensed under the Apache License, Version 2.0 (the "License"); // you may not use this file except in compliance with the License. @@ -65,7 +65,7 @@ func (mr *MockInterfaceMockRecorder) AddClusterIPRoute(arg0 interface{}) *gomock } // AddLoadBalancer mocks base method -func (m *MockInterface) AddLoadBalancer(arg0 []string) error { +func (m *MockInterface) AddLoadBalancer(arg0 net.IP) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "AddLoadBalancer", arg0) ret0, _ := ret[0].(error) @@ -149,7 +149,7 @@ func (mr *MockInterfaceMockRecorder) DeleteClusterIPRoute(arg0 interface{}) *gom } // DeleteLoadBalancer mocks base method -func (m *MockInterface) DeleteLoadBalancer(arg0 []string) error { +func (m *MockInterface) DeleteLoadBalancer(arg0 net.IP) error { m.ctrl.T.Helper() ret := m.ctrl.Call(m, "DeleteLoadBalancer", arg0) ret0, _ := ret[0].(error) @@ -247,17 +247,17 @@ func (mr *MockInterfaceMockRecorder) MigrateRoutesToGw(arg0 interface{}) *gomock } // Reconcile mocks base method -func (m *MockInterface) Reconcile(arg0 []string, arg1 map[string]bool) error { +func (m *MockInterface) Reconcile(arg0 []string) error { m.ctrl.T.Helper() - ret := m.ctrl.Call(m, "Reconcile", arg0, arg1) + ret := m.ctrl.Call(m, "Reconcile", arg0) ret0, _ := ret[0].(error) return ret0 } // Reconcile indicates an expected call of Reconcile -func (mr *MockInterfaceMockRecorder) Reconcile(arg0, arg1 interface{}) *gomock.Call { +func (mr *MockInterfaceMockRecorder) Reconcile(arg0 interface{}) *gomock.Call { mr.mock.ctrl.T.Helper() - return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reconcile", reflect.TypeOf((*MockInterface)(nil).Reconcile), arg0, arg1) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Reconcile", reflect.TypeOf((*MockInterface)(nil).Reconcile), arg0) } // Run mocks base method diff --git a/pkg/agent/util/ipset/ipset.go b/pkg/agent/util/ipset/ipset.go index 0f13104a111..a264fee6e1b 100644 --- a/pkg/agent/util/ipset/ipset.go +++ b/pkg/agent/util/ipset/ipset.go @@ -34,8 +34,26 @@ const ( // memberPattern is used to match the members part of ipset list result. var memberPattern = regexp.MustCompile("(?m)^(.*\n)*Members:\n") +type Interface interface { + CreateIPSet(name string, setType SetType, isIPv6 bool) error + + AddEntry(name string, entry string) error + + DelEntry(name string, entry string) error + + ListEntries(name string) ([]string, error) +} + +type Client struct{} + +var _ Interface = &Client{} + +func NewClient() *Client { + return &Client{} +} + // CreateIPSet creates a new set, it will ignore error when the set already exists. -func CreateIPSet(name string, setType SetType, isIPv6 bool) error { +func (c *Client) CreateIPSet(name string, setType SetType, isIPv6 bool) error { var cmd *exec.Cmd if isIPv6 { cmd = exec.Command("ipset", "create", name, string(setType), "family", "inet6", "-exist") @@ -49,7 +67,7 @@ func CreateIPSet(name string, setType SetType, isIPv6 bool) error { } // AddEntry adds a new entry to the set, it will ignore error when the entry already exists. -func AddEntry(name string, entry string) error { +func (c *Client) AddEntry(name string, entry string) error { cmd := exec.Command("ipset", "add", name, entry, "-exist") if err := cmd.Run(); err != nil { return fmt.Errorf("error adding entry %s to ipset %s: %v", entry, name, err) @@ -58,7 +76,7 @@ func AddEntry(name string, entry string) error { } // DelEntry deletes the entry from the set, it will ignore error when the entry doesn't exist. -func DelEntry(name string, entry string) error { +func (c *Client) DelEntry(name string, entry string) error { cmd := exec.Command("ipset", "del", name, entry, "-exist") if err := cmd.Run(); err != nil { return fmt.Errorf("error deleting entry %s from ipset %s: %v", entry, name, err) @@ -67,7 +85,7 @@ func DelEntry(name string, entry string) error { } // ListEntries lists all the entries of the set. -func ListEntries(name string) ([]string, error) { +func (c *Client) ListEntries(name string) ([]string, error) { cmd := exec.Command("ipset", "list", name) output, err := cmd.CombinedOutput() if err != nil { diff --git a/pkg/agent/util/ipset/testing/mock_ipset.go b/pkg/agent/util/ipset/testing/mock_ipset.go new file mode 100644 index 00000000000..c4cc4c5b55c --- /dev/null +++ b/pkg/agent/util/ipset/testing/mock_ipset.go @@ -0,0 +1,106 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by MockGen. DO NOT EDIT. +// Source: antrea.io/antrea/pkg/agent/util/ipset (interfaces: Interface) + +// Package testing is a generated GoMock package. +package testing + +import ( + ipset "antrea.io/antrea/pkg/agent/util/ipset" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockInterface is a mock of Interface interface +type MockInterface struct { + ctrl *gomock.Controller + recorder *MockInterfaceMockRecorder +} + +// MockInterfaceMockRecorder is the mock recorder for MockInterface +type MockInterfaceMockRecorder struct { + mock *MockInterface +} + +// NewMockInterface creates a new mock instance +func NewMockInterface(ctrl *gomock.Controller) *MockInterface { + mock := &MockInterface{ctrl: ctrl} + mock.recorder = &MockInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { + return m.recorder +} + +// AddEntry mocks base method +func (m *MockInterface) AddEntry(arg0, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddEntry", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddEntry indicates an expected call of AddEntry +func (mr *MockInterfaceMockRecorder) AddEntry(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddEntry", reflect.TypeOf((*MockInterface)(nil).AddEntry), arg0, arg1) +} + +// CreateIPSet mocks base method +func (m *MockInterface) CreateIPSet(arg0 string, arg1 ipset.SetType, arg2 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "CreateIPSet", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// CreateIPSet indicates an expected call of CreateIPSet +func (mr *MockInterfaceMockRecorder) CreateIPSet(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "CreateIPSet", reflect.TypeOf((*MockInterface)(nil).CreateIPSet), arg0, arg1, arg2) +} + +// DelEntry mocks base method +func (m *MockInterface) DelEntry(arg0, arg1 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DelEntry", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// DelEntry indicates an expected call of DelEntry +func (mr *MockInterfaceMockRecorder) DelEntry(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DelEntry", reflect.TypeOf((*MockInterface)(nil).DelEntry), arg0, arg1) +} + +// ListEntries mocks base method +func (m *MockInterface) ListEntries(arg0 string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListEntries", arg0) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListEntries indicates an expected call of ListEntries +func (mr *MockInterfaceMockRecorder) ListEntries(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListEntries", reflect.TypeOf((*MockInterface)(nil).ListEntries), arg0) +} diff --git a/pkg/agent/util/iptables/iptables.go b/pkg/agent/util/iptables/iptables.go index 780c8b5fa4c..9514c16008d 100644 --- a/pkg/agent/util/iptables/iptables.go +++ b/pkg/agent/util/iptables/iptables.go @@ -75,6 +75,26 @@ const ( // iptables-restore: support acquiring the lock. var restoreWaitSupportedMinVersion = semver.Version{Major: 1, Minor: 6, Patch: 2} +type Interface interface { + EnsureChain(protocol Protocol, table string, chain string) error + + ChainExists(protocol Protocol, table string, chain string) (bool, error) + + AppendRule(protocol Protocol, table string, chain string, ruleSpec []string) error + + InsertRule(protocol Protocol, table string, chain string, ruleSpec []string) error + + DeleteRule(protocol Protocol, table string, chain string, ruleSpec []string) error + + DeleteChain(protocol Protocol, table string, chain string) error + + ListRules(table string, chain string) ([]string, error) + + Restore(data string, flush bool, useIPv6 bool) error + + Save() ([]byte, error) +} + type Client struct { ipts map[Protocol]*iptables.IPTables // restoreWaitSupported indicates whether iptables-restore (or ip6tables-restore) supports --wait flag. @@ -271,7 +291,7 @@ func (c *Client) ListRules(table string, chain string) ([]string, error) { // Restore calls iptable-restore to restore iptables with the provided content. // If flush is true, all previous contents of the respective tables will be flushed. // Otherwise only involved chains will be flushed. Restore supports "ip6tables-restore" for IPv6. -func (c *Client) Restore(data []byte, flush bool, useIPv6 bool) error { +func (c *Client) Restore(data string, flush bool, useIPv6 bool) error { var args []string if !flush { args = append(args, "--noflush") @@ -281,7 +301,7 @@ func (c *Client) Restore(data []byte, flush bool, useIPv6 bool) error { iptablesCmd = "ip6tables-restore" } cmd := exec.Command(iptablesCmd, args...) - cmd.Stdin = bytes.NewBuffer(data) + cmd.Stdin = bytes.NewBuffer([]byte(data)) stderr := &bytes.Buffer{} cmd.Stderr = stderr // We acquire xtables lock for iptables-restore to prevent it from conflicting diff --git a/pkg/agent/util/iptables/testing/mock_iptables_linux.go b/pkg/agent/util/iptables/testing/mock_iptables_linux.go new file mode 100644 index 00000000000..732666c23ec --- /dev/null +++ b/pkg/agent/util/iptables/testing/mock_iptables_linux.go @@ -0,0 +1,178 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by MockGen. DO NOT EDIT. +// Source: antrea.io/antrea/pkg/agent/util/iptables (interfaces: Interface) + +// Package testing is a generated GoMock package. +package testing + +import ( + iptables "antrea.io/antrea/pkg/agent/util/iptables" + gomock "github.com/golang/mock/gomock" + reflect "reflect" +) + +// MockInterface is a mock of Interface interface +type MockInterface struct { + ctrl *gomock.Controller + recorder *MockInterfaceMockRecorder +} + +// MockInterfaceMockRecorder is the mock recorder for MockInterface +type MockInterfaceMockRecorder struct { + mock *MockInterface +} + +// NewMockInterface creates a new mock instance +func NewMockInterface(ctrl *gomock.Controller) *MockInterface { + mock := &MockInterface{ctrl: ctrl} + mock.recorder = &MockInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { + return m.recorder +} + +// AppendRule mocks base method +func (m *MockInterface) AppendRule(arg0 iptables.Protocol, arg1, arg2 string, arg3 []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AppendRule", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// AppendRule indicates an expected call of AppendRule +func (mr *MockInterfaceMockRecorder) AppendRule(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AppendRule", reflect.TypeOf((*MockInterface)(nil).AppendRule), arg0, arg1, arg2, arg3) +} + +// ChainExists mocks base method +func (m *MockInterface) ChainExists(arg0 iptables.Protocol, arg1, arg2 string) (bool, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ChainExists", arg0, arg1, arg2) + ret0, _ := ret[0].(bool) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ChainExists indicates an expected call of ChainExists +func (mr *MockInterfaceMockRecorder) ChainExists(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ChainExists", reflect.TypeOf((*MockInterface)(nil).ChainExists), arg0, arg1, arg2) +} + +// DeleteChain mocks base method +func (m *MockInterface) DeleteChain(arg0 iptables.Protocol, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteChain", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteChain indicates an expected call of DeleteChain +func (mr *MockInterfaceMockRecorder) DeleteChain(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteChain", reflect.TypeOf((*MockInterface)(nil).DeleteChain), arg0, arg1, arg2) +} + +// DeleteRule mocks base method +func (m *MockInterface) DeleteRule(arg0 iptables.Protocol, arg1, arg2 string, arg3 []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "DeleteRule", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// DeleteRule indicates an expected call of DeleteRule +func (mr *MockInterfaceMockRecorder) DeleteRule(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "DeleteRule", reflect.TypeOf((*MockInterface)(nil).DeleteRule), arg0, arg1, arg2, arg3) +} + +// EnsureChain mocks base method +func (m *MockInterface) EnsureChain(arg0 iptables.Protocol, arg1, arg2 string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "EnsureChain", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// EnsureChain indicates an expected call of EnsureChain +func (mr *MockInterfaceMockRecorder) EnsureChain(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "EnsureChain", reflect.TypeOf((*MockInterface)(nil).EnsureChain), arg0, arg1, arg2) +} + +// InsertRule mocks base method +func (m *MockInterface) InsertRule(arg0 iptables.Protocol, arg1, arg2 string, arg3 []string) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "InsertRule", arg0, arg1, arg2, arg3) + ret0, _ := ret[0].(error) + return ret0 +} + +// InsertRule indicates an expected call of InsertRule +func (mr *MockInterfaceMockRecorder) InsertRule(arg0, arg1, arg2, arg3 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "InsertRule", reflect.TypeOf((*MockInterface)(nil).InsertRule), arg0, arg1, arg2, arg3) +} + +// ListRules mocks base method +func (m *MockInterface) ListRules(arg0, arg1 string) ([]string, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "ListRules", arg0, arg1) + ret0, _ := ret[0].([]string) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// ListRules indicates an expected call of ListRules +func (mr *MockInterfaceMockRecorder) ListRules(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "ListRules", reflect.TypeOf((*MockInterface)(nil).ListRules), arg0, arg1) +} + +// Restore mocks base method +func (m *MockInterface) Restore(arg0 string, arg1, arg2 bool) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Restore", arg0, arg1, arg2) + ret0, _ := ret[0].(error) + return ret0 +} + +// Restore indicates an expected call of Restore +func (mr *MockInterfaceMockRecorder) Restore(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Restore", reflect.TypeOf((*MockInterface)(nil).Restore), arg0, arg1, arg2) +} + +// Save mocks base method +func (m *MockInterface) Save() ([]byte, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "Save") + ret0, _ := ret[0].([]byte) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// Save indicates an expected call of Save +func (mr *MockInterfaceMockRecorder) Save() *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "Save", reflect.TypeOf((*MockInterface)(nil).Save)) +} diff --git a/pkg/agent/util/net.go b/pkg/agent/util/net.go index dcdb3322f71..24e61532835 100644 --- a/pkg/agent/util/net.go +++ b/pkg/agent/util/net.go @@ -388,7 +388,7 @@ func GetAllNodeAddresses(excludeDevices []string) ([]net.IP, []net.IP, error) { // NewIPNet generates an IPNet from an ip address using a netmask of 32 or 128. func NewIPNet(ip net.IP) *net.IPNet { if ip.To4() != nil { - return &net.IPNet{IP: ip, Mask: net.CIDRMask(32, 32)} + return &net.IPNet{IP: ip.To4(), Mask: net.CIDRMask(32, 32)} } return &net.IPNet{IP: ip, Mask: net.CIDRMask(128, 128)} } diff --git a/pkg/agent/util/net_linux.go b/pkg/agent/util/net_linux.go index 94dc5ebe988..3a9a14111a8 100644 --- a/pkg/agent/util/net_linux.go +++ b/pkg/agent/util/net_linux.go @@ -33,16 +33,6 @@ import ( "k8s.io/klog/v2" ) -// GetNetLink returns dev link from name. -func GetNetLink(dev string) netlink.Link { - link, err := netlink.LinkByName(dev) - if err != nil { - klog.Errorf("Failed to find dev %s: %v", dev, err) - return nil - } - return link -} - // GetNSPeerDevBridge returns peer device and its attached bridge (if applicable) // for device dev in network space indicated by nsPath func GetNSPeerDevBridge(nsPath, dev string) (*net.Interface, string, error) { diff --git a/pkg/agent/util/netlink/netlink_linux.go b/pkg/agent/util/netlink/netlink_linux.go new file mode 100644 index 00000000000..f8648fb9a51 --- /dev/null +++ b/pkg/agent/util/netlink/netlink_linux.go @@ -0,0 +1,92 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package netlink + +import "github.com/vishvananda/netlink" + +// Interface is created to allow testing. +type Interface interface { + RouteReplace(route *netlink.Route) error + + RouteList(link netlink.Link, family int) ([]netlink.Route, error) + + RouteListFiltered(family int, filter *netlink.Route, filterMask uint64) ([]netlink.Route, error) + + RouteDel(route *netlink.Route) error + + AddrList(link netlink.Link, family int) ([]netlink.Addr, error) + + AddrReplace(link netlink.Link, addr *netlink.Addr) error + + AddrDel(link netlink.Link, addr *netlink.Addr) error + + NeighList(linkIndex, family int) ([]netlink.Neigh, error) + + NeighSet(neigh *netlink.Neigh) error + + NeighDel(neigh *netlink.Neigh) error + + LinkByName(name string) (netlink.Link, error) +} + +type Client struct{} + +func NewClient() *Client { + return &Client{} +} + +func (c *Client) RouteReplace(route *netlink.Route) error { + return netlink.RouteReplace(route) +} + +func (c *Client) RouteList(link netlink.Link, family int) ([]netlink.Route, error) { + return netlink.RouteList(link, family) +} + +func (c *Client) RouteListFiltered(family int, filter *netlink.Route, filterMask uint64) ([]netlink.Route, error) { + return netlink.RouteListFiltered(family, filter, filterMask) +} + +func (c *Client) RouteDel(route *netlink.Route) error { + return netlink.RouteDel(route) +} + +func (c *Client) AddrList(link netlink.Link, family int) ([]netlink.Addr, error) { + return netlink.AddrList(link, family) +} + +func (c *Client) AddrReplace(link netlink.Link, addr *netlink.Addr) error { + return netlink.AddrReplace(link, addr) +} + +func (c *Client) AddrDel(link netlink.Link, addr *netlink.Addr) error { + return netlink.AddrDel(link, addr) +} + +func (c *Client) NeighList(linkIndex, family int) ([]netlink.Neigh, error) { + return netlink.NeighList(linkIndex, family) +} + +func (c *Client) NeighSet(neigh *netlink.Neigh) error { + return netlink.NeighSet(neigh) +} + +func (c *Client) NeighDel(neigh *netlink.Neigh) error { + return netlink.NeighDel(neigh) +} + +func (c *Client) LinkByName(name string) (netlink.Link, error) { + return netlink.LinkByName(name) +} diff --git a/pkg/agent/util/netlink/testing/mock_netlink_linux.go b/pkg/agent/util/netlink/testing/mock_netlink_linux.go new file mode 100644 index 00000000000..51ffbed8ac8 --- /dev/null +++ b/pkg/agent/util/netlink/testing/mock_netlink_linux.go @@ -0,0 +1,208 @@ +// Copyright 2022 Antrea Authors +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. +// + +// Code generated by MockGen. DO NOT EDIT. +// Source: antrea.io/antrea/pkg/agent/util/netlink (interfaces: Interface) + +// Package testing is a generated GoMock package. +package testing + +import ( + gomock "github.com/golang/mock/gomock" + netlink "github.com/vishvananda/netlink" + reflect "reflect" +) + +// MockInterface is a mock of Interface interface +type MockInterface struct { + ctrl *gomock.Controller + recorder *MockInterfaceMockRecorder +} + +// MockInterfaceMockRecorder is the mock recorder for MockInterface +type MockInterfaceMockRecorder struct { + mock *MockInterface +} + +// NewMockInterface creates a new mock instance +func NewMockInterface(ctrl *gomock.Controller) *MockInterface { + mock := &MockInterface{ctrl: ctrl} + mock.recorder = &MockInterfaceMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use +func (m *MockInterface) EXPECT() *MockInterfaceMockRecorder { + return m.recorder +} + +// AddrDel mocks base method +func (m *MockInterface) AddrDel(arg0 netlink.Link, arg1 *netlink.Addr) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddrDel", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddrDel indicates an expected call of AddrDel +func (mr *MockInterfaceMockRecorder) AddrDel(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddrDel", reflect.TypeOf((*MockInterface)(nil).AddrDel), arg0, arg1) +} + +// AddrList mocks base method +func (m *MockInterface) AddrList(arg0 netlink.Link, arg1 int) ([]netlink.Addr, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddrList", arg0, arg1) + ret0, _ := ret[0].([]netlink.Addr) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// AddrList indicates an expected call of AddrList +func (mr *MockInterfaceMockRecorder) AddrList(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddrList", reflect.TypeOf((*MockInterface)(nil).AddrList), arg0, arg1) +} + +// AddrReplace mocks base method +func (m *MockInterface) AddrReplace(arg0 netlink.Link, arg1 *netlink.Addr) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "AddrReplace", arg0, arg1) + ret0, _ := ret[0].(error) + return ret0 +} + +// AddrReplace indicates an expected call of AddrReplace +func (mr *MockInterfaceMockRecorder) AddrReplace(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "AddrReplace", reflect.TypeOf((*MockInterface)(nil).AddrReplace), arg0, arg1) +} + +// LinkByName mocks base method +func (m *MockInterface) LinkByName(arg0 string) (netlink.Link, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "LinkByName", arg0) + ret0, _ := ret[0].(netlink.Link) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// LinkByName indicates an expected call of LinkByName +func (mr *MockInterfaceMockRecorder) LinkByName(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "LinkByName", reflect.TypeOf((*MockInterface)(nil).LinkByName), arg0) +} + +// NeighDel mocks base method +func (m *MockInterface) NeighDel(arg0 *netlink.Neigh) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NeighDel", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// NeighDel indicates an expected call of NeighDel +func (mr *MockInterfaceMockRecorder) NeighDel(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NeighDel", reflect.TypeOf((*MockInterface)(nil).NeighDel), arg0) +} + +// NeighList mocks base method +func (m *MockInterface) NeighList(arg0, arg1 int) ([]netlink.Neigh, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NeighList", arg0, arg1) + ret0, _ := ret[0].([]netlink.Neigh) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// NeighList indicates an expected call of NeighList +func (mr *MockInterfaceMockRecorder) NeighList(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NeighList", reflect.TypeOf((*MockInterface)(nil).NeighList), arg0, arg1) +} + +// NeighSet mocks base method +func (m *MockInterface) NeighSet(arg0 *netlink.Neigh) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "NeighSet", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// NeighSet indicates an expected call of NeighSet +func (mr *MockInterfaceMockRecorder) NeighSet(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "NeighSet", reflect.TypeOf((*MockInterface)(nil).NeighSet), arg0) +} + +// RouteDel mocks base method +func (m *MockInterface) RouteDel(arg0 *netlink.Route) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RouteDel", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// RouteDel indicates an expected call of RouteDel +func (mr *MockInterfaceMockRecorder) RouteDel(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RouteDel", reflect.TypeOf((*MockInterface)(nil).RouteDel), arg0) +} + +// RouteList mocks base method +func (m *MockInterface) RouteList(arg0 netlink.Link, arg1 int) ([]netlink.Route, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RouteList", arg0, arg1) + ret0, _ := ret[0].([]netlink.Route) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RouteList indicates an expected call of RouteList +func (mr *MockInterfaceMockRecorder) RouteList(arg0, arg1 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RouteList", reflect.TypeOf((*MockInterface)(nil).RouteList), arg0, arg1) +} + +// RouteListFiltered mocks base method +func (m *MockInterface) RouteListFiltered(arg0 int, arg1 *netlink.Route, arg2 uint64) ([]netlink.Route, error) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RouteListFiltered", arg0, arg1, arg2) + ret0, _ := ret[0].([]netlink.Route) + ret1, _ := ret[1].(error) + return ret0, ret1 +} + +// RouteListFiltered indicates an expected call of RouteListFiltered +func (mr *MockInterfaceMockRecorder) RouteListFiltered(arg0, arg1, arg2 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RouteListFiltered", reflect.TypeOf((*MockInterface)(nil).RouteListFiltered), arg0, arg1, arg2) +} + +// RouteReplace mocks base method +func (m *MockInterface) RouteReplace(arg0 *netlink.Route) error { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "RouteReplace", arg0) + ret0, _ := ret[0].(error) + return ret0 +} + +// RouteReplace indicates an expected call of RouteReplace +func (mr *MockInterfaceMockRecorder) RouteReplace(arg0 interface{}) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RouteReplace", reflect.TypeOf((*MockInterface)(nil).RouteReplace), arg0) +} diff --git a/test/integration/agent/route_test.go b/test/integration/agent/route_test.go index d74f23d60aa..1c901a30d88 100644 --- a/test/integration/agent/route_test.go +++ b/test/integration/agent/route_test.go @@ -187,6 +187,7 @@ func TestInitialize(t *testing.T) { case <-inited2: } + ipset := ipset.NewClient() // verify ipset err = exec.Command("ipset", "list", "ANTREA-POD-IP").Run() assert.NoError(t, err, "ipset not exist") @@ -382,6 +383,8 @@ func TestAddAndDeleteRoutes(t *testing.T) { } assert.Equal(t, expRouteStr, ipRoute, "route mismatch") + ipset := ipset.NewClient() + entries, err := ipset.ListEntries("ANTREA-POD-IP") assert.NoError(t, err, "list ipset entries failed") assert.Contains(t, entries, tc.peerCIDR, "entry should be in ipset") @@ -514,7 +517,6 @@ func TestReconcile(t *testing.T) { addedRoutes []peer desiredPeerCIDRs []string desiredNodeIPs []string - desiredServices map[string]bool // expectations expRoutes map[string]netlink.Link }{ @@ -527,7 +529,6 @@ func TestReconcile(t *testing.T) { }, desiredPeerCIDRs: []string{"10.10.20.0/24"}, desiredNodeIPs: []string{remotePeerIP.String()}, - desiredServices: map[string]bool{"200.200.10.10": true}, expRoutes: map[string]netlink.Link{"10.10.20.0/24": gwLink, "10.10.30.0/24": nil}, }, { @@ -539,7 +540,6 @@ func TestReconcile(t *testing.T) { }, desiredPeerCIDRs: []string{"10.10.20.0/24"}, desiredNodeIPs: []string{localPeerIP.String()}, - desiredServices: map[string]bool{"200.200.10.10": true}, expRoutes: map[string]netlink.Link{"10.10.20.0/24": nodeLink, "10.10.30.0/24": nil}, }, { @@ -553,7 +553,6 @@ func TestReconcile(t *testing.T) { }, desiredPeerCIDRs: []string{"10.10.20.0/24", "10.10.40.0/24"}, desiredNodeIPs: []string{localPeerIP.String(), remotePeerIP.String()}, - desiredServices: map[string]bool{"200.200.10.10": true}, expRoutes: map[string]netlink.Link{"10.10.20.0/24": nodeLink, "10.10.30.0/24": nil, "10.10.40.0/24": gwLink, "10.10.50.0/24": nil}, }, } @@ -571,7 +570,7 @@ func TestReconcile(t *testing.T) { assert.NoError(t, routeClient.AddRoutes(peerNet, tc.nodeName, route.peerIP, peerGwIP), "adding routes failed") } - assert.NoError(t, routeClient.Reconcile(tc.desiredPeerCIDRs, tc.desiredServices), "reconcile failed") + assert.NoError(t, routeClient.Reconcile(tc.desiredPeerCIDRs), "reconcile failed") for dst, uplink := range tc.expRoutes { expNum := 0 @@ -586,6 +585,7 @@ func TestReconcile(t *testing.T) { assert.Equal(t, fmt.Sprint(expNum), output, "mismatch number of routes to %s", dst) } + ipset := ipset.NewClient() entries, err := ipset.ListEntries("ANTREA-POD-IP") assert.NoError(t, err, "list ipset entries failed") assert.ElementsMatch(t, entries, tc.desiredPeerCIDRs, "mismatch ipset entries")