Skip to content

Commit

Permalink
[IPv6] Support no-encap mode in dual-stack setup
Browse files Browse the repository at this point in the history 
* Extend single IP address to IPv4 and IPv6 ones for NodeAddr and NodeTransportAddr in nodeConfig
* Refactor occurrences of Node IP address to adjust to 2 IP: agent initialization, node route
* Still use one IP for IPv6/dual-stack not supported cases: Windows, NPL
* Update tests

Signed-off-by: Zhecheng Li <[email protected]>

Related: #2426
  • Loading branch information
@lzhecheng
lzhecheng committed Aug 5, 2021
1 parent 1a84b66 commit 92118f6
Show file tree
Hide file tree
Showing 25 changed files with 323 additions and 189 deletions.
2 changes: 1 addition & 1 deletion cmd/antrea-agent/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -228,7 +228,7 @@ func run(o *Options) error {
var egressController *egress.EgressController
if features.DefaultFeatureGate.Enabled(features.Egress) {
egressController, err = egress.NewEgressController(
ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPAddr.IP,
ofClient, antreaClientProvider, crdClient, ifaceStore, routeClient, nodeConfig.Name, nodeConfig.NodeIPv4Addr.IP,
o.config.ClusterMembershipPort, egressInformer, nodeInformer, externalIPPoolInformer,
)
if err != nil {
Expand Down
57 changes: 35 additions & 22 deletions pkg/agent/agent.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -21,6 +21,7 @@ import (
"net"
"os"
"strconv"
"strings"
"sync"
"time"

Expand Down Expand Up @@ -533,15 +534,17 @@ func (i *Initializer) configureGatewayInterface(gatewayIface *interfacestore.Int

i.nodeConfig.GatewayConfig = &config.GatewayConfig{Name: i.hostGateway, MAC: gwMAC}
gatewayIface.MAC = gwMAC
gatewayIface.IPs = []net.IP{}
if i.networkConfig.TrafficEncapMode.IsNetworkPolicyOnly() {
// Assign IP to gw as required by SpoofGuard.
// NodeIPAddr can be either IPv4 or IPv6.
if i.nodeConfig.NodeIPAddr.IP.To4() != nil {
i.nodeConfig.GatewayConfig.IPv4 = i.nodeConfig.NodeTransportIPAddr.IP
} else {
i.nodeConfig.GatewayConfig.IPv6 = i.nodeConfig.NodeTransportIPAddr.IP
if i.nodeConfig.NodeIPv4Addr != nil {
i.nodeConfig.GatewayConfig.IPv4 = i.nodeConfig.NodeTransportIPv4Addr.IP
gatewayIface.IPs = append(gatewayIface.IPs, i.nodeConfig.NodeTransportIPv4Addr.IP)
}
if i.nodeConfig.NodeIPv6Addr != nil {
i.nodeConfig.GatewayConfig.IPv6 = i.nodeConfig.NodeTransportIPv6Addr.IP
gatewayIface.IPs = append(gatewayIface.IPs, i.nodeConfig.NodeTransportIPv6Addr.IP)
}
gatewayIface.IPs = []net.IP{i.nodeConfig.NodeTransportIPAddr.IP}
// No need to assign local CIDR to gw0 because local CIDR is not managed by Antrea
return nil
}
Expand Down Expand Up @@ -645,26 +648,34 @@ func (i *Initializer) initNodeLocalConfig() error {
return err
}

var nodeIPAddr, transportIPAddr *net.IPNet
var nodeIPv4Addr, nodeIPv6Addr, transportIPv4Addr, transportIPv6Addr *net.IPNet
var localIntf *net.Interface
// Find the interface configured with Node IP and use it for Pod traffic.
ipAddr, err := k8s.GetNodeAddr(node)
ipAddrs, err := k8s.GetNodeAddrs(node)
if err != nil {
return fmt.Errorf("failed to obtain local IP address from K8s: %w", err)
return fmt.Errorf("failed to obtain local IP addresses from K8s: %w", err)
}
nodeIPAddr, localIntf, err = getIPNetDeviceFromIP(ipAddr)
nodeIPv4Addr, nodeIPv6Addr, localIntf, err = getIPNetDeviceFromIP(ipAddrs)
if err != nil {
return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddr, err)
return fmt.Errorf("failed to get local IPNet device with IP %v: %v", ipAddrs, err)
}
transportIPAddr = nodeIPAddr
transportIPv4Addr = nodeIPv4Addr
transportIPv6Addr = nodeIPv6Addr
if i.networkConfig.TransportIface != "" {
// Find the configured transport interface, and update its IP address in Node's annotation.
transportIPAddr, localIntf, err = getTransportIPNetDeviceByName(i.networkConfig.TransportIface, i.ovsBridge)
transportIPv4Addr, transportIPv6Addr, localIntf, err = getTransportIPNetDeviceByName(i.networkConfig.TransportIface, i.ovsBridge)
if err != nil {
return fmt.Errorf("failed to get local IPNet device with transport interface %s: %v", i.networkConfig.TransportIface, err)
}
klog.InfoS("Updating Node transport address annotation")
if err := i.patchNodeAnnotations(nodeName, types.NodeTransportAddressAnnotationKey, transportIPAddr.IP.String()); err != nil {
klog.InfoS("Updating Node transport addresses annotation")
var ips []string
if transportIPv4Addr != nil {
ips = append(ips, transportIPv4Addr.IP.String())
}
if transportIPv6Addr != nil {
ips = append(ips, transportIPv6Addr.IP.String())
}
if err := i.patchNodeAnnotations(nodeName, types.NodeTransportAddressAnnotationKey, strings.Join(ips, ",")); err != nil {
return err
}
} else {
Expand All @@ -686,12 +697,14 @@ func (i *Initializer) initNodeLocalConfig() error {
}

i.nodeConfig = &config.NodeConfig{
Name: nodeName,
OVSBridge: i.ovsBridge,
DefaultTunName: defaultTunInterfaceName,
NodeIPAddr: nodeIPAddr,
NodeTransportIPAddr: transportIPAddr,
UplinkNetConfig: new(config.AdapterNetConfig)}
Name: nodeName,
OVSBridge: i.ovsBridge,
DefaultTunName: defaultTunInterfaceName,
NodeIPv4Addr: nodeIPv4Addr,
NodeIPv6Addr: nodeIPv6Addr,
NodeTransportIPv4Addr: transportIPv4Addr,
NodeTransportIPv6Addr: transportIPv6Addr,
UplinkNetConfig: new(config.AdapterNetConfig)}

mtu, err := i.getNodeMTU(localIntf)
if err != nil {
Expand Down Expand Up @@ -864,7 +877,7 @@ func (i *Initializer) getNodeMTU(localIntf *net.Interface) (int, error) {
} else if i.networkConfig.TunnelType == ovsconfig.GRETunnel {
mtu -= config.GREOverhead
}
if i.nodeConfig.NodeIPAddr.IP.To4() == nil {
if i.nodeConfig.NodeIPv6Addr != nil {
mtu -= config.IPv6ExtraOverhead
}
}
Expand Down
2 changes: 1 addition & 1 deletion pkg/agent/agent_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -43,6 +43,6 @@ func (i *Initializer) getTunnelPortLocalIP() net.IP {
return nil
}

func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.Interface, error) {
func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.IPNet, *net.Interface, error) {
return util.GetIPNetDeviceByName(ifaceName)
}
26 changes: 13 additions & 13 deletions pkg/agent/agent_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -278,14 +278,14 @@ func TestInitNodeLocalConfig(t *testing.T) {
client := fake.NewSimpleClientset(node)
ifaceStore := interfacestore.NewInterfaceStore()
expectedNodeConfig := config.NodeConfig{
Name: nodeName,
OVSBridge: ovsBridge,
DefaultTunName: defaultTunInterfaceName,
PodIPv4CIDR: podCIDR,
NodeIPAddr: nodeIPNet,
NodeTransportIPAddr: nodeIPNet,
NodeMTU: tt.expectedMTU,
UplinkNetConfig: new(config.AdapterNetConfig),
Name: nodeName,
OVSBridge: ovsBridge,
DefaultTunName: defaultTunInterfaceName,
PodIPv4CIDR: podCIDR,
NodeIPv4Addr: nodeIPNet,
NodeTransportIPv4Addr: nodeIPNet,
NodeMTU: tt.expectedMTU,
UplinkNetConfig: new(config.AdapterNetConfig),
}

initializer := &Initializer{
Expand All @@ -300,7 +300,7 @@ func TestInitNodeLocalConfig(t *testing.T) {
}
if tt.transportInterface != nil {
initializer.networkConfig.TransportIface = tt.transportInterface.iface.Name
expectedNodeConfig.NodeTransportIPAddr = tt.transportInterface.ipNet
expectedNodeConfig.NodeTransportIPv4Addr = tt.transportInterface.ipNet
defer mockGetTransportIPNetDeviceByName(tt.transportInterface.ipNet, tt.transportInterface.iface)()
}
defer mockGetIPNetDeviceFromIP(nodeIPNet, ipDevice)()
Expand All @@ -317,8 +317,8 @@ func TestInitNodeLocalConfig(t *testing.T) {

func mockGetIPNetDeviceFromIP(ipNet *net.IPNet, ipDevice *net.Interface) func() {
prevGetIPNetDeviceFromIP := getIPNetDeviceFromIP
getIPNetDeviceFromIP = func(localIP net.IP) (*net.IPNet, *net.Interface, error) {
return ipNet, ipDevice, nil
getIPNetDeviceFromIP = func(localIP []net.IP) (*net.IPNet, *net.IPNet, *net.Interface, error) {
return ipNet, nil, ipDevice, nil
}
return func() { getIPNetDeviceFromIP = prevGetIPNetDeviceFromIP }
}
Expand All @@ -330,8 +330,8 @@ func mockNodeNameEnv(name string) func() {

func mockGetTransportIPNetDeviceByName(ipNet *net.IPNet, ipDevice *net.Interface) func() {
prevGetIPNetDeviceByName := getTransportIPNetDeviceByName
getTransportIPNetDeviceByName = func(ifName, brName string) (*net.IPNet, *net.Interface, error) {
return ipNet, ipDevice, nil
getTransportIPNetDeviceByName = func(ifName, brName string) (*net.IPNet, *net.IPNet, *net.Interface, error) {
return ipNet, nil, ipDevice, nil
}
return func() { getTransportIPNetDeviceByName = prevGetIPNetDeviceByName }
}
18 changes: 9 additions & 9 deletions pkg/agent/agent_windows.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ func (i *Initializer) prepareHostNetwork() error {
// Get uplink network configuration. The uplink interface is the one used for transporting Pod traffic across Nodes.
// Use the interface specified with "transportInterface" in the configuration if configured, otherwise the interface
// configured with NodeIP is used as uplink.
_, adapter, err := util.GetIPNetDeviceFromIP(i.nodeConfig.NodeTransportIPAddr.IP)
_, _, adapter, err := util.GetIPNetDeviceFromIP([]net.IP{i.nodeConfig.NodeTransportIPv4Addr.IP})
if err != nil {
return err
}
Expand All @@ -68,7 +68,7 @@ func (i *Initializer) prepareHostNetwork() error {
}
i.nodeConfig.UplinkNetConfig.Name = adapter.Name
i.nodeConfig.UplinkNetConfig.MAC = adapter.HardwareAddr
i.nodeConfig.UplinkNetConfig.IP = i.nodeConfig.NodeTransportIPAddr
i.nodeConfig.UplinkNetConfig.IP = i.nodeConfig.NodeTransportIPv4Addr
i.nodeConfig.UplinkNetConfig.Index = adapter.Index
defaultGW, err := util.GetDefaultGatewayByInterfaceIndex(adapter.Index)
if err != nil {
Expand Down Expand Up @@ -96,7 +96,7 @@ func (i *Initializer) prepareHostNetwork() error {
if subnetCIDR == nil {
return fmt.Errorf("failed to find valid IPv4 PodCIDR")
}
return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeTransportIPAddr, adapter)
return util.PrepareHNSNetwork(subnetCIDR, i.nodeConfig.NodeTransportIPv4Addr, adapter)
}

// prepareOVSBridge adds local port and uplink to ovs bridge.
Expand Down Expand Up @@ -213,7 +213,7 @@ func (i *Initializer) initHostNetworkFlows() error {

// getTunnelLocalIP returns local_ip of tunnel port
func (i *Initializer) getTunnelPortLocalIP() net.IP {
return i.nodeConfig.NodeTransportIPAddr.IP
return i.nodeConfig.NodeTransportIPv4Addr.IP
}

// saveHostRoutes saves routes which are configured on uplink interface before
Expand Down Expand Up @@ -271,17 +271,17 @@ func (i *Initializer) restoreHostRoutes() error {
return nil
}

func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.Interface, error) {
func GetTransportIPNetDeviceByName(ifaceName string, ovsBridgeName string) (*net.IPNet, *net.IPNet, *net.Interface, error) {
// Find transport Interface in the order: ifaceName -> "vEthernet (ifaceName)" -> br-int. Return immediately if
// an interface using the specified name exists. Using "vEthernet (ifaceName)" or br-int is for restart agent case.
for _, name := range []string{ifaceName, fmt.Sprintf("vEthernet (%s)", ifaceName), ovsBridgeName} {
ipNet, link, err := util.GetIPNetDeviceByName(name)
ipNet, _, link, err := util.GetIPNetDeviceByName(name)
if err == nil {
return ipNet, link, nil
return ipNet, nil, link, nil
}
if !strings.Contains(err.Error(), "no such network interface") {
return nil, nil, err
return nil, nil, nil, err
}
}
return nil, nil, fmt.Errorf("unable to find local IP and device")
return nil, nil, nil, fmt.Errorf("unable to find local IP and device")
}
22 changes: 12 additions & 10 deletions pkg/agent/config/node_config.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -83,10 +83,14 @@ type NodeConfig struct {
// The CIDR block from where to allocate IPv6 address to Pod.
// It's nil for the net workPolicyOnly trafficEncapMode which doesn't do IPAM.
PodIPv6CIDR *net.IPNet
// The Node's IP used in Kubernetes. It has the network mask information.
NodeIPAddr *net.IPNet
// The IP on the Node's transport interface. It is used for tunneling or routing the Pod traffic across Nodes.
NodeTransportIPAddr *net.IPNet
// The Node's IPv4 address used in Kubernetes. It has the network mask information.
NodeIPv4Addr *net.IPNet
// The Node's IPv6 address used in Kubernetes. It has the network mask information.
NodeIPv6Addr *net.IPNet
// The IPv4 address on the Node's transport interface. It is used for tunneling or routing the Pod traffic across Nodes.
NodeTransportIPv4Addr *net.IPNet
// The IPv6 address on the Node's transport interface. It is used for tunneling or routing the Pod traffic across Nodes.
NodeTransportIPv6Addr *net.IPNet
// Set either via defaultMTU config in antrea.yaml or auto discovered.
// Auto discovery will use MTU value of the Node's primary interface.
// For Encap and Hybrid mode, Node MTU will be adjusted to account for encap header.
Expand All @@ -98,8 +102,8 @@ type NodeConfig struct {
}

func (n *NodeConfig) String() string {
return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIP: %s, TransportIP: %s, Gateway: %s",
n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPAddr, n.NodeTransportIPAddr, n.GatewayConfig)
return fmt.Sprintf("NodeName: %s, OVSBridge: %s, PodIPv4CIDR: %s, PodIPv6CIDR: %s, NodeIPv4: %s, NodeIPv6: %s, TransportIPv4: %s, TransportIPv6: %s, Gateway: %s",
n.Name, n.OVSBridge, n.PodIPv4CIDR, n.PodIPv6CIDR, n.NodeIPv4Addr, n.NodeIPv6Addr, n.NodeTransportIPv4Addr, n.NodeTransportIPv6Addr, n.GatewayConfig)
}

// User provided network configuration parameters.
Expand All @@ -112,15 +116,13 @@ type NetworkConfig struct {
}

// IsIPv4Enabled returns true if the cluster network supports IPv4.
// TODO: support dual-stack in networkPolicyOnly mode.
func IsIPv4Enabled(nodeConfig *NodeConfig, trafficEncapMode TrafficEncapModeType) bool {
return nodeConfig.PodIPv4CIDR != nil ||
(trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPAddr.IP.To4() != nil)
(trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPv4Addr != nil)
}

// IsIPv6Enabled returns true if the cluster network supports IPv6.
// TODO: support dual-stack in networkPolicyOnly mode.
func IsIPv6Enabled(nodeConfig *NodeConfig, trafficEncapMode TrafficEncapModeType) bool {
return nodeConfig.PodIPv6CIDR != nil ||
(trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPAddr.IP.To4() == nil)
(trafficEncapMode.IsNetworkPolicyOnly() && nodeConfig.NodeIPv6Addr != nil)
}
2 changes: 1 addition & 1 deletion pkg/agent/controller/egress/ipassigner/ip_assigner_linux.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -49,7 +49,7 @@ type ipAssigner struct {

// NewIPAssigner returns an *ipAssigner.
func NewIPAssigner(nodeIPAddr net.IP, dummyDeviceName string) (*ipAssigner, error) {
_, egressInterface, err := util.GetIPNetDeviceFromIP(nodeIPAddr)
_, _, egressInterface, err := util.GetIPNetDeviceFromIP([]net.IP{nodeIPAddr})
if err != nil {
return nil, fmt.Errorf("get IPNetDevice from ip %v error: %+v", nodeIPAddr, err)
}
Expand Down
Loading

0 comments on commit 92118f6

Please sign in to comment.