feature: support exchange link names of OVS bridge and provider nic in underlay networks

dist/images/install.sh

@@ -19,6 +19,9 @@ ENABLE_EXTERNAL_VPC=${ENABLE_EXTERNAL_VPC:-true}
 CNI_CONFIG_PRIORITY=${CNI_CONFIG_PRIORITY:-01}
 ENABLE_LB_SVC=${ENABLE_LB_SVC:-false}
 ENABLE_KEEP_VM_IP=${ENABLE_KEEP_VM_IP:-true}
+# exchange link names of OVS bridge and the provider nic
+# in the default provider-network
+EXCHANGE_LINK_NAME=${EXCHANGE_LINK_NAME:-false}
 # The nic to support container network can be a nic name or a group of regex
 # separated by comma, if empty will use the nic that the default route use
 IFACE=${IFACE:-}
@@ -1405,6 +1408,8 @@ spec:
                         type: array
                         items:
                           type: string
+                exchangeLinkName:
+                  type: boolean
                 excludeNodes:
                   type: array
                   items:
@@ -2795,6 +2800,7 @@ spec:
           - --service-cluster-ip-range=$SVC_CIDR
           - --network-type=$NETWORK_TYPE
           - --default-interface-name=$VLAN_INTERFACE_NAME
+          - --default-exchange-link-name=$EXCHANGE_LINK_NAME
           - --default-vlan-id=$VLAN_ID
           - --ls-dnat-mod-dl-dst=$LS_DNAT_MOD_DL_DST
           - --pod-nic-type=$POD_NIC_TYPE

dist/images/start-ovs.sh

@@ -63,6 +63,77 @@ else
   ovs-vsctl --no-wait set open_vswitch . other_config:hw-offload=false
 fi
 
+function exchange_link_names() {
+  mappings=($(ovs-vsctl --if-exists get open . external-ids:ovn-bridge-mappings | tr -d '"' | tr ',' ' '))
+  bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn external-ids:exchange-link-name=true))
+  for br in ${bridges[@]}; do
+    provider=""
+    for m in ${mappings[*]}; do
+      if echo $m | grep -q ":$br"'$'; then
+        provider=${m%:$br}
+        break
+      fi
+    done
+    if [ "x$provider" = "x" ]; then
+      echo "error: failed to get provider name for bridge $br"
+      continue
+    fi
+
+    port="br-$provider"
+    if ip link show $port 2>/dev/null; then
+      echo "link $port already exists"
+      continue
+    fi
+    if ! ip link show $br 2>/dev/null; then
+      echo "link $br does not exists"
+      continue
+    fi
+
+    echo "change link name from $br to $port"
+    ipv4_routes=($(ip -4 route show dev $br | tr ' ' '#'))
+    ipv6_routes=($(ip -6 route show dev $br | tr ' ' '#'))
+    ip link set $br down
+    ip link set $br name $port
+    ip link set $port up
+
+    # transfer IPv4 routes
+    default_ipv4_routes=()
+    for route in ${ipv4_routes[@]}; do
+      r=$(echo $route | tr '#' ' ')
+      if echo $r | grep -q -w 'scope link'; then
+        printf "add/replace IPv4 route $r to $port\n"
+        ip -4 route replace $r dev $port
+      else
+        default_ipv4_routes=(${default_ipv4_routes[@]} $route)
+      fi
+    done
+    for route in ${default_ipv4_routes[@]}; do
+      r=$(echo $route | tr '#' ' ')
+      printf "add/replace IPv4 route $r to $port\n"
+      ip -4 route replace $r dev $port
+    done
+
+    # transfer IPv6 routes
+    default_ipv6_routes=()
+    for route in ${ipv6_routes[@]}; do
+      r=$(echo $route | tr '#' ' ')
+      if echo $r | grep -q -w 'scope link'; then
+        printf "add/replace IPv6 route $r to $port\n"
+        ip -6 route replace $r dev $port
+      else
+        default_ipv6_routes=(${default_ipv6_routes[@]} $route)
+      fi
+    done
+    for route in ${default_ipv6_routes[@]}; do
+      r=$(echo $route | tr '#' ' ')
+      printf "add/replace IPv6 route $r to $port\n"
+      ip -6 route replace $r dev $port
+    done
+  done
+}
+
+exchange_link_names
+
 # Start vswitchd. restart will automatically set/unset flow-restore-wait which is not what we want
 /usr/share/openvswitch/scripts/ovs-ctl start --no-ovsdb-server --system-id=random
 /usr/share/openvswitch/scripts/ovs-ctl --protocol=udp --dport=6081 enable-protocol

go.mod

@@ -6,6 +6,7 @@ require (
 	github.com/Mellanox/sriovnet v1.1.0
 	github.com/Microsoft/go-winio v0.5.2
 	github.com/Microsoft/hcsshim v0.9.3
+	github.com/Wifx/gonetworkmanager v0.4.0
 	github.com/alauda/felix v3.6.6-0.20201207121355-187332daf314+incompatible
 	github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e
 	github.com/cenkalti/backoff/v4 v4.1.3
@@ -69,6 +70,7 @@ require (
 	github.com/go-openapi/jsonpointer v0.19.5 // indirect
 	github.com/go-openapi/jsonreference v0.20.0 // indirect
 	github.com/go-openapi/swag v0.21.1 // indirect
+	github.com/godbus/dbus/v5 v5.1.0 // indirect
 	github.com/gogo/protobuf v1.3.2 // indirect
 	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect

go.sum

@@ -127,6 +127,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko
 github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ=
 github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo=
 github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI=
+github.com/Wifx/gonetworkmanager v0.4.0 h1:iZ7o3z3YIqEEAa+bBOCwQkLLUQuxG02CdiE7NCe5y6A=
+github.com/Wifx/gonetworkmanager v0.4.0/go.mod h1:EdhHf2O00IZXfMv9LC6CS6SgTwcMTg/ZSDhGvch0cs8=
 github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw=
 github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28 h1:FW5M3SAwSGBdtTboeV5sI7kEY6zraApSZQxTUfZ7LQY=
 github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28/go.mod h1:dXpg+IAC2yp2IZQlEVmnmEc1rqEmSZzgNfu6+ai38J4=
@@ -565,9 +567,12 @@ github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7
 github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw=
 github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4=
+github.com/godbus/dbus/v5 v5.0.2/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
+github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk=
+github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA=
 github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM=
 github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU=
 github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c=

pkg/apis/kubeovn/v1/types.go

@@ -280,6 +280,7 @@ type ProviderNetworkSpec struct {
 	DefaultInterface string            `json:"defaultInterface,omitempty"`
 	CustomInterfaces []CustomInterface `json:"customInterfaces,omitempty"`
 	ExcludeNodes     []string          `json:"excludeNodes,omitempty"`
+	ExchangeLinkName bool              `json:"exchangeLinkName,omitempty"`
 }
 
 type ProviderNetworkStatus struct {

pkg/controller/config.go

@@ -65,12 +65,13 @@ type Configuration struct {
 	EnablePprof     bool
 	NodePgProbeTime int
 
-	NetworkType          string
-	DefaultProviderName  string
-	DefaultHostInterface string
-	DefaultVlanName      string
-	DefaultVlanID        int
-	LsDnatModDlDst       bool
+	NetworkType             string
+	DefaultProviderName     string
+	DefaultHostInterface    string
+	DefaultExchangeLinkName bool
+	DefaultVlanName         string
+	DefaultVlanID           int
+	LsDnatModDlDst          bool
 
 	EnableLb          bool
 	EnableNP          bool
@@ -122,20 +123,21 @@ func ParseFlags() (*Configuration, error) {
 		argPprofPort       = pflag.Int("pprof-port", 10660, "The port to get profiling data")
 		argNodePgProbeTime = pflag.Int("nodepg-probe-time", 1, "The probe interval for node port-group, the unit is minute")
 
-		argNetworkType          = pflag.String("network-type", util.NetworkTypeGeneve, "The ovn network type")
-		argDefaultProviderName  = pflag.String("default-provider-name", "provider", "The vlan or vxlan type default provider interface name")
-		argDefaultInterfaceName = pflag.String("default-interface-name", "", "The default host interface name in the vlan/vxlan type")
-		argDefaultVlanName      = pflag.String("default-vlan-name", "ovn-vlan", "The default vlan name")
-		argDefaultVlanID        = pflag.Int("default-vlan-id", 1, "The default vlan id")
-		argLsDnatModDlDst       = pflag.Bool("ls-dnat-mod-dl-dst", true, "Set ethernet destination address for DNAT on logical switch")
-		argPodNicType           = pflag.String("pod-nic-type", "veth-pair", "The default pod network nic implementation type")
-		argEnableLb             = pflag.Bool("enable-lb", true, "Enable load balancer")
-		argEnableNP             = pflag.Bool("enable-np", true, "Enable network policy support")
-		argEnableEipSnat        = pflag.Bool("enable-eip-snat", true, "Enable EIP and SNAT")
-		argEnableExternalVpc    = pflag.Bool("enable-external-vpc", true, "Enable external vpc support")
-		argEnableEcmp           = pflag.Bool("enable-ecmp", false, "Enable ecmp route for centralized subnet")
-		argKeepVmIP             = pflag.Bool("keep-vm-ip", false, "Whether to keep ip for kubevirt pod when pod is rebuild")
-		argEnableLbSvc          = pflag.Bool("enable-lb-svc", false, "Whether to support loadbalancer service")
+		argNetworkType             = pflag.String("network-type", util.NetworkTypeGeneve, "The ovn network type")
+		argDefaultProviderName     = pflag.String("default-provider-name", "provider", "The vlan or vxlan type default provider interface name")
+		argDefaultInterfaceName    = pflag.String("default-interface-name", "", "The default host interface name in the vlan/vxlan type")
+		argDefaultExchangeLinkName = pflag.Bool("default-exchange-link-name", false, "exchange link names of OVS bridge and the provider nic in the default provider-network")
+		argDefaultVlanName         = pflag.String("default-vlan-name", "ovn-vlan", "The default vlan name")
+		argDefaultVlanID           = pflag.Int("default-vlan-id", 1, "The default vlan id")
+		argLsDnatModDlDst          = pflag.Bool("ls-dnat-mod-dl-dst", true, "Set ethernet destination address for DNAT on logical switch")
+		argPodNicType              = pflag.String("pod-nic-type", "veth-pair", "The default pod network nic implementation type")
+		argEnableLb                = pflag.Bool("enable-lb", true, "Enable load balancer")
+		argEnableNP                = pflag.Bool("enable-np", true, "Enable network policy support")
+		argEnableEipSnat           = pflag.Bool("enable-eip-snat", true, "Enable EIP and SNAT")
+		argEnableExternalVpc       = pflag.Bool("enable-external-vpc", true, "Enable external vpc support")
+		argEnableEcmp              = pflag.Bool("enable-ecmp", false, "Enable ecmp route for centralized subnet")
+		argKeepVmIP                = pflag.Bool("keep-vm-ip", false, "Whether to keep ip for kubevirt pod when pod is rebuild")
+		argEnableLbSvc             = pflag.Bool("enable-lb-svc", false, "Whether to support loadbalancer service")
 
 		argExternalGatewayConfigNS = pflag.String("external-gateway-config-ns", "kube-system", "The namespace of configmap external-gateway-config, default: kube-system")
 		argExternalGatewayNet      = pflag.String("external-gateway-net", "external", "The name of the external network which mappings with an ovs bridge, default: external")
@@ -192,6 +194,7 @@ func ParseFlags() (*Configuration, error) {
 		LsDnatModDlDst:                *argLsDnatModDlDst,
 		DefaultProviderName:           *argDefaultProviderName,
 		DefaultHostInterface:          *argDefaultInterfaceName,
+		DefaultExchangeLinkName:       *argDefaultExchangeLinkName,
 		DefaultVlanName:               *argDefaultVlanName,
 		PodName:                       os.Getenv("POD_NAME"),
 		PodNamespace:                  os.Getenv("KUBE_NAMESPACE"),

pkg/controller/init.go

@@ -467,6 +467,7 @@ func (c *Controller) initDefaultProviderNetwork() error {
 		},
 		Spec: kubeovnv1.ProviderNetworkSpec{
 			DefaultInterface: c.config.DefaultHostInterface,
+			ExchangeLinkName: c.config.DefaultExchangeLinkName,
 		},
 	}
 

pkg/daemon/controller.go

@@ -264,7 +264,7 @@ func (c *Controller) initProviderNetwork(pn *kubeovnv1.ProviderNetwork, node *v1
 
 	var mtu int
 	var err error
-	if mtu, err = ovsInitProviderNetwork(pn.Name, nic); err != nil {
+	if mtu, err = ovsInitProviderNetwork(pn.Name, nic, pn.Spec.ExchangeLinkName); err != nil {
 		if oldLen := len(node.Labels); oldLen != 0 {
 			delete(node.Labels, fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name))
 			delete(node.Labels, fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name))

pkg/daemon/init.go

@@ -96,10 +96,21 @@ func InitMirror(config *Configuration) error {
 	return configureEmptyMirror(config.MirrorNic, config.MTU)
 }
 
-func ovsInitProviderNetwork(provider, nic string) (int, error) {
+func ovsInitProviderNetwork(provider, nic string, exchangeLinkName bool) (int, error) {
 	// create and configure external bridge
 	brName := util.ExternalBridgeName(provider)
-	if err := configExternalBridge(provider, brName, nic); err != nil {
+	if exchangeLinkName {
+		exchanged, err := changeProvideNicName(nic, brName)
+		if err != nil {
+			klog.Errorf("failed to change provider nic name from %s to %s: %v", nic, brName, err)
+			return 0, err
+		}
+		if exchanged {
+			nic, brName = brName, nic
+		}
+	}
+
+	if err := configExternalBridge(provider, brName, nic, exchangeLinkName); err != nil {
 		errMsg := fmt.Errorf("failed to create and configure external bridge %s: %v", brName, err)
 		klog.Error(errMsg)
 		return 0, errMsg
@@ -124,35 +135,52 @@ func ovsInitProviderNetwork(provider, nic string) (int, error) {
 }
 
 func ovsCleanProviderNetwork(provider string) error {
-	output, err := ovs.Exec("list-br")
+	output, err := ovs.Exec(ovs.IfExists, "get", "open", ".", "external-ids:ovn-bridge-mappings")
 	if err != nil {
+		return fmt.Errorf("failed to get ovn-bridge-mappings, %v: %q", err, output)
+	}
+
+	var idx int
+	var m, brName string
+	mappingPrefix := provider + ":"
+	brMappings := strings.Split(output, ",")
+	for idx, m = range brMappings {
+		if strings.HasPrefix(m, mappingPrefix) {
+			brName = m[len(mappingPrefix):]
+			break
+		}
+	}
+
+	if output, err = ovs.Exec("list-br"); err != nil {
 		return fmt.Errorf("failed to list OVS bridge %v: %q", err, output)
 	}
 
-	brName := util.ExternalBridgeName(provider)
 	if !util.ContainsString(strings.Split(output, "\n"), brName) {
 		return nil
 	}
 
-	if output, err = ovs.Exec(ovs.IfExists, "get", "open", ".", "external-ids:ovn-bridge-mappings"); err != nil {
-		return fmt.Errorf("failed to get ovn-bridge-mappings, %v: %q", err, output)
+	// get host nic
+	if output, err = ovs.Exec("list-ports", brName); err != nil {
+		return fmt.Errorf("failed to list ports of OVS bridge %s, %v: %q", brName, err, output)
 	}
 
-	mappings := strings.Split(output, ",")
-	brMap := fmt.Sprintf("%s:%s", provider, brName)
-
-	var idx int
-	for idx = range mappings {
-		if mappings[idx] == brMap {
-			break
+	// remove host nic from the external bridge
+	if output != "" {
+		for _, port := range strings.Split(output, "\n") {
+			if err = removeProviderNic(port, brName); err != nil {
+				errMsg := fmt.Errorf("failed to remove port %s from external bridge %s: %v", port, brName, err)
+				klog.Error(errMsg)
+				return errMsg
+			}
 		}
 	}
-	if idx != len(mappings) {
-		mappings = append(mappings[:idx], mappings[idx+1:]...)
-		if len(mappings) == 0 {
+
+	if idx != len(brMappings) {
+		brMappings = append(brMappings[:idx], brMappings[idx+1:]...)
+		if len(brMappings) == 0 {
 			output, err = ovs.Exec(ovs.IfExists, "remove", "open", ".", "external-ids", "ovn-bridge-mappings")
 		} else {
-			output, err = ovs.Exec("set", "open", ".", "external-ids:ovn-bridge-mappings="+strings.Join(mappings, ","))
+			output, err = ovs.Exec("set", "open", ".", "external-ids:ovn-bridge-mappings="+strings.Join(brMappings, ","))
 		}
 		if err != nil {
 			return fmt.Errorf("failed to set ovn-bridge-mappings, %v: %q", err, output)
@@ -164,7 +192,7 @@ func ovsCleanProviderNetwork(provider string) error {
 	}
 	macMappings := strings.Split(output, ",")
 	for _, macMap := range macMappings {
-		if len(macMap) == len(provider)+18 && strings.HasPrefix(macMap, provider) {
+		if strings.HasPrefix(macMap, mappingPrefix) {
 			macMappings = util.RemoveString(macMappings, macMap)
 			break
 		}
@@ -185,9 +213,9 @@ func ovsCleanProviderNetwork(provider string) error {
 
 	// remove host nic from the external bridge
 	if output != "" {
-		for _, nic := range strings.Split(output, "\n") {
-			if err = removeProviderNic(nic, brName); err != nil {
-				errMsg := fmt.Errorf("failed to remove nic %s from external bridge %s: %v", nic, brName, err)
+		for _, port := range strings.Split(output, "\n") {
+			if err = removeProviderNic(port, brName); err != nil {
+				errMsg := fmt.Errorf("failed to remove port %s from external bridge %s: %v", port, brName, err)
 				klog.Error(errMsg)
 				return errMsg
 			}
@@ -199,5 +227,12 @@ func ovsCleanProviderNetwork(provider string) error {
 		return fmt.Errorf("failed to remove OVS bridge %s, %v: %q", brName, err, output)
 	}
 
+	if br := util.ExternalBridgeName(provider); br != brName {
+		if _, err = changeProvideNicName(br, brName); err != nil {
+			klog.Errorf("failed to change provider nic name from %s to %s: %v", br, brName, err)
+			return err
+		}
+	}
+
 	return nil
 }