Verify the status of required routes and IP configuration of gateway …

…periodically

Add checks to the routeClient. The required routes will be added back if they were
deleted unexpectedly. Add IP configuration check of the gateway to the agent.
An e2e test is added to verify that the route will be added back correctly.

Fixes #627

cmd/antrea-agent/agent.go

@@ -127,6 +127,7 @@ func run(o *Options) error {
 	// networkReadyCh is used to notify that the Node's network is ready.
 	// Functions that rely on the Node's network should wait for the channel to close.
 	networkReadyCh := make(chan struct{})
+	stopCh := signals.RegisterSignalHandlers()
 	// Initialize agent and node network.
 	agentInitializer := agent.NewInitializer(
 		k8sClient,
@@ -141,6 +142,7 @@ func run(o *Options) error {
 		serviceCIDRNetv6,
 		networkConfig,
 		networkReadyCh,
+		stopCh,
 		features.DefaultFeatureGate.Enabled(features.AntreaProxy))
 	err = agentInitializer.Initialize()
 	if err != nil {
@@ -259,7 +261,6 @@ func run(o *Options) error {
 	// set up signal capture: the first SIGTERM / SIGINT signal is handled gracefully and will
 	// cause the stopCh channel to be closed; if another signal is received before the program
 	// exits, we will force exit.
-	stopCh := signals.RegisterSignalHandlers()
 
 	// Start the NPL agent.
 	if features.DefaultFeatureGate.Enabled(features.NodePortLocal) {

pkg/agent/agent.go

@@ -71,6 +71,7 @@ type Initializer struct {
 	// networkReadyCh should be closed once the Node's network is ready.
 	// The CNI server will wait for it before handling any CNI Add requests.
 	networkReadyCh chan<- struct{}
+	stopCh         <-chan struct{}
 }
 
 func NewInitializer(
@@ -86,6 +87,7 @@ func NewInitializer(
 	serviceCIDRv6 *net.IPNet,
 	networkConfig *config.NetworkConfig,
 	networkReadyCh chan<- struct{},
+	stopCh <-chan struct{},
 	enableProxy bool) *Initializer {
 	return &Initializer{
 		ovsBridgeClient: ovsBridgeClient,
@@ -100,6 +102,7 @@ func NewInitializer(
 		serviceCIDRv6:   serviceCIDRv6,
 		networkConfig:   networkConfig,
 		networkReadyCh:  networkReadyCh,
+		stopCh:          stopCh,
 		enableProxy:     enableProxy,
 	}
 }
@@ -222,6 +225,31 @@ func (i *Initializer) Initialize() error {
 		return err
 	}
 
+	// Periodically check whether IP configuration of the gateway is correct.
+	// Terminated when stopCh is closed.
+	go wait.Until(func() {
+		_, _, err := util.SetLinkUp(i.hostGateway)
+		if err != nil {
+			klog.Errorf("Failed to set up gateway: %v", err)
+			return
+		}
+		podCidrs := []*net.IPNet{i.nodeConfig.PodIPv4CIDR, i.nodeConfig.PodIPv6CIDR}
+		gwIPs := []*net.IPNet{}
+		for _, podCidr := range podCidrs {
+			if podCidr == nil {
+				continue
+			}
+			subnetID := podCidr.IP.Mask(podCidr.Mask)
+			gwIP := &net.IPNet{IP: ip.NextIP(subnetID), Mask: podCidr.Mask}
+			gwIPs = append(gwIPs, gwIP)
+		}
+
+		if err := util.ConfigureLinkAddresses(i.nodeConfig.GatewayConfig.LinkIndex, gwIPs); err != nil {
+			klog.Errorf("Failed to check IP configuration of the gateway: %v", err)
+			return
+		}
+	}, 60*time.Second, i.stopCh)
+
 	wg.Add(1)
 	// routeClient.Initialize() should be after i.setupOVSBridge() which
 	// creates the host gateway interface.

pkg/agent/route/route_linux.go

@@ -152,7 +152,79 @@ func (c *Client) syncIPInfra() {
 		klog.Errorf("Failed to sync iptables: %v", err)
 		return
 	}
-	klog.V(3).Infof("Successfully synced node iptables")
+	if err := c.syncRoutes(); err != nil {
+		klog.Errorf("Failed to sync routes: %v", err)
+	}
+	klog.V(3).Infof("Successfully synced node iptables and routes")
+}
+
+func (c *Client) syncRoutes() error {
+	routeList, err := netlink.RouteList(nil, netlink.FAMILY_ALL)
+	if err != nil {
+		return err
+	}
+	routeMap := make(map[string]*netlink.Route)
+	for i := range routeList {
+		r := &routeList[i]
+		if r == nil || r.Dst == nil {
+			continue
+		}
+		routeMap[r.Dst.String()] = r
+	}
+	routes := []*netlink.Route{}
+	c.nodeRoutes.Range(func(_, v interface{}) bool {
+		for _, route := range v.([]*netlink.Route) {
+			routes = append(routes, route)
+		}
+		return true
+	})
+	for _, route := range routes {
+		r, ok := routeMap[route.Dst.String()]
+		if !ok {
+			if err := netlink.RouteAdd(route); err != nil {
+				return err
+			}
+			continue
+		}
+		if !routeEqual(route, r) {
+			if err := netlink.RouteAdd(route); err != nil {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+// func (c *Client) syncGwIp() error {
+// 	_, _, err := util.SetLinkUp(c.nodeConfig.GatewayConfig.Name)
+// 	if err != nil {
+// 		return err
+// 	}
+// 	podCidrs := []*net.IPNet{c.nodeConfig.PodIPv4CIDR, c.nodeConfig.PodIPv6CIDR}
+// 	gwIPs := []*net.IPNet{}
+// 	for _, podCidr := range podCidrs {
+// 		if podCidr == nil {
+// 			continue
+// 		}
+// 		subnetID := podCidr.IP.Mask(podCidr.Mask)
+// 		gwIP := &net.IPNet{IP: ip.NextIP(subnetID), Mask: podCidr.Mask}
+// 		gwIPs = append(gwIPs, gwIP)
+// 	}
+
+// 	if err := util.ConfigureLinkAddresses(c.nodeConfig.GatewayConfig.LinkIndex, gwIPs); err != nil {
+// 		return err
+// 	}
+// 	return nil
+// }
+
+func routeEqual(x, y *netlink.Route) bool {
+	if x == nil || y == nil {
+		return false
+	}
+	return x.LinkIndex == y.LinkIndex &&
+		x.Dst.IP.Equal(y.Dst.IP) &&
+		bytes.Equal(x.Dst.Mask, y.Dst.Mask) &&
+		x.Gw.Equal(y.Gw)
 }
 
 // syncIPSet ensures that the required ipset exists and it has the initial members.

pkg/agent/util/net_linux.go

@@ -38,7 +38,7 @@ func GetNetLink(dev string) netlink.Link {
 	return link
 }
 
-// GetPeerLinkBridge returns peer device and its attached bridge (if applicable)
+// GetNSPeerDevBridge returns peer device and its attached bridge (if applicable)
 // for device dev in network space indicated by nsPath
 func GetNSPeerDevBridge(nsPath, dev string) (*net.Interface, string, error) {
 	var peerIdx int