Fix that Service routes may get lost when starting on Windows

Fix antrea-io#4467

Signed-off-by: Hongliang Liu <[email protected]>

pkg/agent/controller/noderoute/node_route_controller.go

@@ -22,6 +22,7 @@ import (
 	"github.com/containernetworking/plugins/pkg/ip"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/apimachinery/pkg/util/wait"
 	"k8s.io/client-go/informers"
 	coreinformers "k8s.io/client-go/informers/core/v1"
@@ -203,16 +204,22 @@ func (c *Controller) removeStaleGatewayRoutes() error {
 		desiredPodCIDRs = append(desiredPodCIDRs, podCIDRs...)
 	}
 
-	// TODO: This is not the best place to keep the ClusterIP Service routes.
-	desiredClusterIPSvcIPs := map[string]bool{}
+	// TODO: This is not the best place to keep the Service routes.
+	desiredSvcIPNets := sets.NewString()
 	if c.proxyAll && runtime.IsWindowsPlatform() {
 		// The route for virtual IP -> antrea-gw0 should be always kept.
-		desiredClusterIPSvcIPs[config.VirtualServiceIPv4.String()] = true
+		vIPNet := util.NewIPNet(config.VirtualServiceIPv4).String()
+		desiredSvcIPNets.Insert(vIPNet)
 
 		svcs, err := c.svcLister.List(labels.Everything())
 		for _, svc := range svcs {
-			for _, ip := range svc.Spec.ClusterIPs {
-				desiredClusterIPSvcIPs[ip] = true
+			for _, clusterIP := range svc.Spec.ClusterIPs {
+				clusterIPNet := util.NewIPNet(net.ParseIP(clusterIP)).String()
+				desiredSvcIPNets.Insert(clusterIPNet)
+			}
+			for _, ingressIP := range svc.Status.LoadBalancer.Ingress {
+				ingressIPNet := util.NewIPNet(net.ParseIP(ingressIP.IP)).String()
+				desiredSvcIPNets.Insert(ingressIPNet)
 			}
 		}
 		if err != nil {
@@ -222,8 +229,8 @@ func (c *Controller) removeStaleGatewayRoutes() error {
 
 	// routeClient will remove orphaned routes whose destinations are not in desiredPodCIDRs.
 	// If proxyAll enabled, it will also remove routes that are for Windows ClusterIP Services
-	// which no longer exist.
-	if err := c.routeClient.Reconcile(desiredPodCIDRs, desiredClusterIPSvcIPs); err != nil {
+	// which no longer exists.
+	if err := c.routeClient.Reconcile(desiredPodCIDRs, desiredSvcIPNets); err != nil {
 		return err
 	}
 	return nil

pkg/agent/route/interfaces.go

@@ -17,6 +17,8 @@ package route
 import (
 	"net"
 
+	"k8s.io/apimachinery/pkg/util/sets"
+
 	"antrea.io/antrea/pkg/agent/config"
 	binding "antrea.io/antrea/pkg/ovs/openflow"
 )
@@ -29,7 +31,7 @@ type Interface interface {
 
 	// Reconcile should remove orphaned routes and related configuration based on the desired podCIDRs and Service IPs.
 	// If IPv6 is enabled in the cluster, Reconcile should also remove the orphaned IPv6 neighbors.
-	Reconcile(podCIDRs []string, svcIPs map[string]bool) error
+	Reconcile(podCIDRs []string, svcIPNets sets.String) error
 
 	// AddRoutes should add routes to the provided podCIDR.
 	// It should override the routes if they already exist, without error.

pkg/agent/route/route_linux.go

@@ -843,7 +843,7 @@ func (c *Client) initServiceIPRoutes() error {
 
 // Reconcile removes orphaned podCIDRs from ipset and removes routes to orphaned podCIDRs
 // based on the desired podCIDRs. svcIPs are used for Windows only.
-func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error {
+func (c *Client) Reconcile(podCIDRs []string, svcIPNets sets.String) error {
 	desiredPodCIDRs := sets.NewString(podCIDRs...)
 	// Get the peer IPv6 gateways from pod CIDRs
 	desiredIPv6GWs := getIPv6Gateways(podCIDRs)

pkg/agent/route/route_windows.go

@@ -122,7 +122,7 @@ func (c *Client) initServiceIPRoutes() error {
 
 // Reconcile removes the orphaned routes and related configuration based on the desired podCIDRs and Service IPs. Only
 // the route entries on the host gateway interface are stored in the cache.
-func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error {
+func (c *Client) Reconcile(podCIDRs []string, svcIPNets sets.String) error {
 	desiredPodCIDRs := sets.NewString(podCIDRs...)
 	routes, err := c.listRoutes()
 	if err != nil {
@@ -133,7 +133,7 @@ func (c *Client) Reconcile(podCIDRs []string, svcIPs map[string]bool) error {
 			c.hostRoutes.Store(dst, rt)
 			continue
 		}
-		if _, ok := svcIPs[dst]; ok {
+		if _, ok := svcIPNets[dst]; ok {
 			c.hostRoutes.Store(dst, rt)
 			continue
 		}

pkg/agent/route/route_windows_test.go

@@ -23,6 +23,7 @@ import (
 
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
+	"k8s.io/apimachinery/pkg/util/sets"
 	"k8s.io/klog/v2"
 
 	"antrea.io/antrea/pkg/agent/config"
@@ -96,7 +97,7 @@ func TestRouteOperation(t *testing.T) {
 	require.Nil(t, err)
 	assert.Equal(t, 1, len(route4))
 
-	err = client.Reconcile([]string{dest2}, map[string]bool{svcIPNet1.String(): true})
+	err = client.Reconcile([]string{dest2}, sets.NewString(svcIPNet1.String()))
 	require.Nil(t, err)
 
 	routes5, err := util.GetNetRoutes(gwLink, destCIDR1)

test/integration/agent/route_test.go

@@ -517,7 +517,7 @@ func TestReconcile(t *testing.T) {
 		addedRoutes      []peer
 		desiredPeerCIDRs []string
 		desiredNodeIPs   []string
-		desiredServices  map[string]bool
+		desiredServices  sets.String
 		// expectations
 		expRoutes map[string]netlink.Link
 	}{
@@ -530,7 +530,7 @@ func TestReconcile(t *testing.T) {
 			},
 			desiredPeerCIDRs: []string{"10.10.20.0/24"},
 			desiredNodeIPs:   []string{remotePeerIP.String()},
-			desiredServices:  map[string]bool{"200.200.10.10": true},
+			desiredServices:  sets.NewString("200.200.10.10/32"),
 			expRoutes:        map[string]netlink.Link{"10.10.20.0/24": gwLink, "10.10.30.0/24": nil},
 		},
 		{
@@ -542,7 +542,7 @@ func TestReconcile(t *testing.T) {
 			},
 			desiredPeerCIDRs: []string{"10.10.20.0/24"},
 			desiredNodeIPs:   []string{localPeerIP.String()},
-			desiredServices:  map[string]bool{"200.200.10.10": true},
+			desiredServices:  sets.NewString("200.200.10.10/32"),
 			expRoutes:        map[string]netlink.Link{"10.10.20.0/24": nodeLink, "10.10.30.0/24": nil},
 		},
 		{
@@ -556,7 +556,7 @@ func TestReconcile(t *testing.T) {
 			},
 			desiredPeerCIDRs: []string{"10.10.20.0/24", "10.10.40.0/24"},
 			desiredNodeIPs:   []string{localPeerIP.String(), remotePeerIP.String()},
-			desiredServices:  map[string]bool{"200.200.10.10": true},
+			desiredServices:  sets.NewString("200.200.10.10/32"),
 			expRoutes:        map[string]netlink.Link{"10.10.20.0/24": nodeLink, "10.10.30.0/24": nil, "10.10.40.0/24": gwLink, "10.10.50.0/24": nil},
 		},
 	}