From 5f2ff4250d436289f08a959f642063145f36ccce Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Mon, 25 Jul 2022 13:42:16 +0800 Subject: [PATCH 1/5] feature: support exchange link names of OVS bridge and provider nic in underlay networks --- dist/images/install.sh | 2 + dist/images/start-ovs.sh | 71 ++++++++++++++++++++++++ go.mod | 2 + go.sum | 5 ++ pkg/apis/kubeovn/v1/types.go | 1 + pkg/daemon/controller.go | 2 +- pkg/daemon/init.go | 75 ++++++++++++++++++-------- pkg/daemon/init_linux.go | 101 +++++++++++++++++++++++++++++++++++ pkg/daemon/init_windows.go | 6 +++ pkg/daemon/ovs.go | 6 ++- pkg/daemon/ovs_linux.go | 8 +-- yamls/crd.yaml | 2 + 12 files changed, 250 insertions(+), 31 deletions(-) create mode 100644 pkg/daemon/init_linux.go create mode 100644 pkg/daemon/init_windows.go diff --git a/dist/images/install.sh b/dist/images/install.sh index 3225083cfb1..f4091402ea0 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -1405,6 +1405,8 @@ spec: type: array items: type: string + exchangeLinkName: + type: boolean excludeNodes: type: array items: diff --git a/dist/images/start-ovs.sh b/dist/images/start-ovs.sh index cf0d03d0f60..6d8c206ff59 100755 --- a/dist/images/start-ovs.sh +++ b/dist/images/start-ovs.sh @@ -63,6 +63,77 @@ else ovs-vsctl --no-wait set open_vswitch . other_config:hw-offload=false fi +function exchange_link_names() { + mappings=($(ovs-vsctl --if-exists get open . external-ids:ovn-bridge-mappings | tr -d '"' | tr ',' ' ')) + bridges=($(ovs-vsctl --no-heading --columns=name find bridge external-ids:vendor=kube-ovn external-ids:exchange-link-name=true)) + for br in ${bridges[@]}; do + provider="" + for m in ${mappings[*]}; do + if echo $m | grep -q ":$br"'$'; then + provider=${m%:$br} + break + fi + done + if [ "x$provider" = "x" ]; then + echo "error: failed to get provider name for bridge $br" + continue + fi + + port="br-$provider" + if ip link show $port 2>/dev/null; then + echo "link $port already exists" + continue + fi + if ! ip link show $br 2>/dev/null; then + echo "link $br does not exists" + continue + fi + + echo "change link name from $br to $port" + ipv4_routes=($(ip -4 route show dev $br | tr ' ' '#')) + ipv6_routes=($(ip -6 route show dev $br | tr ' ' '#')) + ip link set $br down + ip link set $br name $port + ip link set $port up + + # transfer IPv4 routes + default_ipv4_routes=() + for route in ${ipv4_routes[@]}; do + r=$(echo $route | tr '#' ' ') + if echo $r | grep -q -w 'scope link'; then + printf "add/replace IPv4 route $r to $port\n" + ip -4 route replace $r dev $port + else + default_ipv4_routes=(${default_ipv4_routes[@]} $route) + fi + done + for route in ${default_ipv4_routes[@]}; do + r=$(echo $route | tr '#' ' ') + printf "add/replace IPv4 route $r to $port\n" + ip -4 route replace $r dev $port + done + + # transfer IPv6 routes + default_ipv6_routes=() + for route in ${ipv6_routes[@]}; do + r=$(echo $route | tr '#' ' ') + if echo $r | grep -q -w 'scope link'; then + printf "add/replace IPv6 route $r to $port\n" + ip -6 route replace $r dev $port + else + default_ipv6_routes=(${default_ipv6_routes[@]} $route) + fi + done + for route in ${default_ipv6_routes[@]}; do + r=$(echo $route | tr '#' ' ') + printf "add/replace IPv6 route $r to $port\n" + ip -6 route replace $r dev $port + done + done +} + +exchange_link_names + # Start vswitchd. restart will automatically set/unset flow-restore-wait which is not what we want /usr/share/openvswitch/scripts/ovs-ctl start --no-ovsdb-server --system-id=random /usr/share/openvswitch/scripts/ovs-ctl --protocol=udp --dport=6081 enable-protocol diff --git a/go.mod b/go.mod index 4f0e4f61480..ed72cf48eb9 100644 --- a/go.mod +++ b/go.mod @@ -6,6 +6,7 @@ require ( github.com/Mellanox/sriovnet v1.1.0 github.com/Microsoft/go-winio v0.5.2 github.com/Microsoft/hcsshim v0.9.3 + github.com/Wifx/gonetworkmanager v0.4.0 github.com/alauda/felix v3.6.6-0.20201207121355-187332daf314+incompatible github.com/bhendo/go-powershell v0.0.0-20190719160123-219e7fb4e41e github.com/cenkalti/backoff/v4 v4.1.3 @@ -69,6 +70,7 @@ require ( github.com/go-openapi/jsonpointer v0.19.5 // indirect github.com/go-openapi/jsonreference v0.20.0 // indirect github.com/go-openapi/swag v0.21.1 // indirect + github.com/godbus/dbus/v5 v5.1.0 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/glog v1.0.0 // indirect github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect diff --git a/go.sum b/go.sum index 98afab7ec4f..edb9440496b 100644 --- a/go.sum +++ b/go.sum @@ -127,6 +127,8 @@ github.com/PuerkitoBio/urlesc v0.0.0-20170810143723-de5bf2ad4578/go.mod h1:uGdko github.com/Shopify/logrus-bugsnag v0.0.0-20171204204709-577dee27f20d/go.mod h1:HI8ITrYtUY+O+ZhtlqUnD8+KwNPOyugEhfP9fdUIaEQ= github.com/Shopify/sarama v1.19.0/go.mod h1:FVkBWblsNy7DGZRfXLU0O9RCGt5g3g3yEuWXgklEdEo= github.com/Shopify/toxiproxy v2.1.4+incompatible/go.mod h1:OXgGpZ6Cli1/URJOF1DMxUHB2q5Ap20/P/eIdh4G0pI= +github.com/Wifx/gonetworkmanager v0.4.0 h1:iZ7o3z3YIqEEAa+bBOCwQkLLUQuxG02CdiE7NCe5y6A= +github.com/Wifx/gonetworkmanager v0.4.0/go.mod h1:EdhHf2O00IZXfMv9LC6CS6SgTwcMTg/ZSDhGvch0cs8= github.com/ajstarks/svgo v0.0.0-20180226025133-644b8db467af/go.mod h1:K08gAheRH3/J6wwsYMMT4xOr94bZjxIelGM0+d/wbFw= github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28 h1:FW5M3SAwSGBdtTboeV5sI7kEY6zraApSZQxTUfZ7LQY= github.com/alauda/ovsdb v0.0.0-20210113100339-040cf3e76c28/go.mod h1:dXpg+IAC2yp2IZQlEVmnmEc1rqEmSZzgNfu6+ai38J4= @@ -565,9 +567,12 @@ github.com/gobuffalo/flect v0.1.5/go.mod h1:W3K3X9ksuZfir8f/LrfVtWmCDQFfayuylOJ7 github.com/godbus/dbus v0.0.0-20151105175453-c7fdd8b5cd55/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20180201030542-885f9cc04c9c/go.mod h1:/YcGZj5zSblfDWMMoOzV4fas9FZnQYTkDnsGvmh2Grw= github.com/godbus/dbus v0.0.0-20190422162347-ade71ed3457e/go.mod h1:bBOAhwG1umN6/6ZUMtDFBMQR8jRg9O75tm9K00oMsK4= +github.com/godbus/dbus/v5 v5.0.2/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.3/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.4/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/godbus/dbus/v5 v5.0.6/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= +github.com/godbus/dbus/v5 v5.1.0 h1:4KLkAxT3aOY8Li4FRJe/KvhoNFFxo0m6fNuFUO8QJUk= +github.com/godbus/dbus/v5 v5.1.0/go.mod h1:xhWf0FNVPg57R7Z0UbKHbJfkEywrmjJnf7w5xrFpKfA= github.com/gofrs/uuid v4.0.0+incompatible/go.mod h1:b2aQJv3Z4Fp6yNu3cdSllBxTCLRxnplIgP/c0N/04lM= github.com/gogo/googleapis v1.2.0/go.mod h1:Njal3psf3qN6dwBtQfUmBZh2ybovJ0tlu3o/AC7HYjU= github.com/gogo/googleapis v1.4.0/go.mod h1:5YRNX2z1oM5gXdAkurHa942MDgEJyk02w4OecKY87+c= diff --git a/pkg/apis/kubeovn/v1/types.go b/pkg/apis/kubeovn/v1/types.go index 010e87c683b..68b338de686 100644 --- a/pkg/apis/kubeovn/v1/types.go +++ b/pkg/apis/kubeovn/v1/types.go @@ -280,6 +280,7 @@ type ProviderNetworkSpec struct { DefaultInterface string `json:"defaultInterface,omitempty"` CustomInterfaces []CustomInterface `json:"customInterfaces,omitempty"` ExcludeNodes []string `json:"excludeNodes,omitempty"` + ExchangeLinkName bool `json:"exchangeLinkName,omitempty"` } type ProviderNetworkStatus struct { diff --git a/pkg/daemon/controller.go b/pkg/daemon/controller.go index c7188ec89db..7360f5c274b 100644 --- a/pkg/daemon/controller.go +++ b/pkg/daemon/controller.go @@ -264,7 +264,7 @@ func (c *Controller) initProviderNetwork(pn *kubeovnv1.ProviderNetwork, node *v1 var mtu int var err error - if mtu, err = ovsInitProviderNetwork(pn.Name, nic); err != nil { + if mtu, err = ovsInitProviderNetwork(pn.Name, nic, pn.Spec.ExchangeLinkName); err != nil { if oldLen := len(node.Labels); oldLen != 0 { delete(node.Labels, fmt.Sprintf(util.ProviderNetworkReadyTemplate, pn.Name)) delete(node.Labels, fmt.Sprintf(util.ProviderNetworkInterfaceTemplate, pn.Name)) diff --git a/pkg/daemon/init.go b/pkg/daemon/init.go index 39acba84795..aef237c8db4 100644 --- a/pkg/daemon/init.go +++ b/pkg/daemon/init.go @@ -96,10 +96,20 @@ func InitMirror(config *Configuration) error { return configureEmptyMirror(config.MirrorNic, config.MTU) } -func ovsInitProviderNetwork(provider, nic string) (int, error) { +func ovsInitProviderNetwork(provider, nic string, exchangeLinkName bool) (int, error) { // create and configure external bridge brName := util.ExternalBridgeName(provider) - if err := configExternalBridge(provider, brName, nic); err != nil { + if exchangeLinkName { + exchanged, err := changeProvideNicName(nic, brName) + if err != nil { + return 0, err + } + if exchanged { + nic, brName = brName, nic + } + } + + if err := configExternalBridge(provider, brName, nic, exchangeLinkName); err != nil { errMsg := fmt.Errorf("failed to create and configure external bridge %s: %v", brName, err) klog.Error(errMsg) return 0, errMsg @@ -124,35 +134,52 @@ func ovsInitProviderNetwork(provider, nic string) (int, error) { } func ovsCleanProviderNetwork(provider string) error { - output, err := ovs.Exec("list-br") + output, err := ovs.Exec(ovs.IfExists, "get", "open", ".", "external-ids:ovn-bridge-mappings") if err != nil { + return fmt.Errorf("failed to get ovn-bridge-mappings, %v: %q", err, output) + } + + var idx int + var m, brName string + mappingPrefix := provider + ":" + brMappings := strings.Split(output, ",") + for idx, m = range brMappings { + if strings.HasPrefix(m, mappingPrefix) { + brName = m[len(mappingPrefix):] + break + } + } + + if output, err = ovs.Exec("list-br"); err != nil { return fmt.Errorf("failed to list OVS bridge %v: %q", err, output) } - brName := util.ExternalBridgeName(provider) if !util.ContainsString(strings.Split(output, "\n"), brName) { return nil } - if output, err = ovs.Exec(ovs.IfExists, "get", "open", ".", "external-ids:ovn-bridge-mappings"); err != nil { - return fmt.Errorf("failed to get ovn-bridge-mappings, %v: %q", err, output) + // get host nic + if output, err = ovs.Exec("list-ports", brName); err != nil { + return fmt.Errorf("failed to list ports of OVS bridge %s, %v: %q", brName, err, output) } - mappings := strings.Split(output, ",") - brMap := fmt.Sprintf("%s:%s", provider, brName) - - var idx int - for idx = range mappings { - if mappings[idx] == brMap { - break + // remove host nic from the external bridge + if output != "" { + for _, port := range strings.Split(output, "\n") { + if err = removeProviderNic(port, brName); err != nil { + errMsg := fmt.Errorf("failed to remove port %s from external bridge %s: %v", port, brName, err) + klog.Error(errMsg) + return errMsg + } } } - if idx != len(mappings) { - mappings = append(mappings[:idx], mappings[idx+1:]...) - if len(mappings) == 0 { + + if idx != len(brMappings) { + brMappings = append(brMappings[:idx], brMappings[idx+1:]...) + if len(brMappings) == 0 { output, err = ovs.Exec(ovs.IfExists, "remove", "open", ".", "external-ids", "ovn-bridge-mappings") } else { - output, err = ovs.Exec("set", "open", ".", "external-ids:ovn-bridge-mappings="+strings.Join(mappings, ",")) + output, err = ovs.Exec("set", "open", ".", "external-ids:ovn-bridge-mappings="+strings.Join(brMappings, ",")) } if err != nil { return fmt.Errorf("failed to set ovn-bridge-mappings, %v: %q", err, output) @@ -164,7 +191,7 @@ func ovsCleanProviderNetwork(provider string) error { } macMappings := strings.Split(output, ",") for _, macMap := range macMappings { - if len(macMap) == len(provider)+18 && strings.HasPrefix(macMap, provider) { + if strings.HasPrefix(macMap, mappingPrefix) { macMappings = util.RemoveString(macMappings, macMap) break } @@ -185,9 +212,9 @@ func ovsCleanProviderNetwork(provider string) error { // remove host nic from the external bridge if output != "" { - for _, nic := range strings.Split(output, "\n") { - if err = removeProviderNic(nic, brName); err != nil { - errMsg := fmt.Errorf("failed to remove nic %s from external bridge %s: %v", nic, brName, err) + for _, port := range strings.Split(output, "\n") { + if err = removeProviderNic(port, brName); err != nil { + errMsg := fmt.Errorf("failed to remove port %s from external bridge %s: %v", port, brName, err) klog.Error(errMsg) return errMsg } @@ -199,5 +226,11 @@ func ovsCleanProviderNetwork(provider string) error { return fmt.Errorf("failed to remove OVS bridge %s, %v: %q", brName, err, output) } + if br := util.ExternalBridgeName(provider); br != brName { + if _, err = changeProvideNicName(br, brName); err != nil { + return err + } + } + return nil } diff --git a/pkg/daemon/init_linux.go b/pkg/daemon/init_linux.go new file mode 100644 index 00000000000..a4eef7611a7 --- /dev/null +++ b/pkg/daemon/init_linux.go @@ -0,0 +1,101 @@ +package daemon + +import ( + "k8s.io/klog/v2" + + "github.com/Wifx/gonetworkmanager" + "github.com/vishvananda/netlink" +) + +var routeScopeOrders = [...]netlink.Scope{ + netlink.SCOPE_HOST, + netlink.SCOPE_LINK, + netlink.SCOPE_SITE, + netlink.SCOPE_UNIVERSE, +} + +func nmSetManaged(device string, managed bool) error { + nm, err := gonetworkmanager.NewNetworkManager() + if err != nil { + klog.V(5).Infof("failed to connect to NetworkManager: %v", err) + return nil + } + + d, err := nm.GetDeviceByIpIface(device) + if err != nil { + return err + } + current, err := d.GetPropertyManaged() + if err != nil { + return err + } + if current == managed { + return nil + } + + return d.SetPropertyManaged(managed) +} + +func changeProvideNicName(current, target string) (bool, error) { + link, err := netlink.LinkByName(current) + if err != nil { + if _, ok := err.(netlink.LinkNotFoundError); ok { + klog.Infof("link %s not found, skip", current) + return false, nil + } + return false, err + } + if link.Type() == "openvswitch" { + klog.Infof("%s is an openvswitch interface, skip", current) + return false, nil + } + + // set link unmanaged by NetworkManager to avoid getting new IP by DHCP + if err = nmSetManaged(current, false); err != nil { + return false, err + } + + klog.Infof("change nic name from %s to %s", current, target) + addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL) + if err != nil { + return false, err + } + routes, err := netlink.RouteList(link, netlink.FAMILY_ALL) + if err != nil { + return false, err + } + + if err = netlink.LinkSetDown(link); err != nil { + return false, err + } + if err = netlink.LinkSetName(link, target); err != nil { + return false, err + } + if err = netlink.LinkSetUp(link); err != nil { + return false, err + } + + for _, addr := range addresses { + if addr.IP.IsLinkLocalUnicast() { + continue + } + if err = netlink.AddrReplace(link, &addr); err != nil { + return false, err + } + } + + for _, scope := range routeScopeOrders { + for _, route := range routes { + if route.Gw == nil && route.Dst != nil && route.Dst.IP.IsLinkLocalUnicast() { + continue + } + if route.Scope == scope { + if err = netlink.RouteReplace(&route); err != nil { + return false, err + } + } + } + } + + return true, nil +} diff --git a/pkg/daemon/init_windows.go b/pkg/daemon/init_windows.go new file mode 100644 index 00000000000..997150189e7 --- /dev/null +++ b/pkg/daemon/init_windows.go @@ -0,0 +1,6 @@ +package daemon + +func changeProvideNicName(nic, br string) (bool, error) { + // not supported on windows + return false, nil +} diff --git a/pkg/daemon/ovs.go b/pkg/daemon/ovs.go index 076cf9d8cf8..a6f04d8f02c 100644 --- a/pkg/daemon/ovs.go +++ b/pkg/daemon/ovs.go @@ -71,13 +71,15 @@ func configureEmptyMirror(portName string, mtu int) error { return configureMirrorLink(portName, mtu) } -func configExternalBridge(provider, bridge, nic string) error { +func configExternalBridge(provider, bridge, nic string, exchangeLinkName bool) error { brExists, err := ovs.BridgeExists(bridge) if err != nil { return fmt.Errorf("failed to check OVS bridge existence: %v", err) } output, err := ovs.Exec(ovs.MayExist, "add-br", bridge, - "--", "set", "bridge", bridge, "external_ids:vendor="+util.CniTypeName) + "--", "set", "bridge", bridge, "external_ids:vendor="+util.CniTypeName, + "--", "set", "bridge", bridge, fmt.Sprintf("external_ids:exchange-link-name=%v", exchangeLinkName), + ) if err != nil { return fmt.Errorf("failed to create OVS bridge %s, %v: %q", bridge, err, output) } diff --git a/pkg/daemon/ovs_linux.go b/pkg/daemon/ovs_linux.go index 15455e2dd0f..42c015d3d32 100644 --- a/pkg/daemon/ovs_linux.go +++ b/pkg/daemon/ovs_linux.go @@ -583,13 +583,7 @@ func configProviderNic(nicName, brName string) (int, error) { return 0, fmt.Errorf("failed to set OVS bridge %s up: %v", brName, err) } - scopeOrders := [...]netlink.Scope{ - netlink.SCOPE_HOST, - netlink.SCOPE_LINK, - netlink.SCOPE_SITE, - netlink.SCOPE_UNIVERSE, - } - for _, scope := range scopeOrders { + for _, scope := range routeScopeOrders { for _, route := range routes { if route.Gw == nil && route.Dst != nil && route.Dst.IP.IsLinkLocalUnicast() { // skip 169.254.0.0/16 and fe80::/10 diff --git a/yamls/crd.yaml b/yamls/crd.yaml index be7a4959fe6..5c7c457331b 100644 --- a/yamls/crd.yaml +++ b/yamls/crd.yaml @@ -816,6 +816,8 @@ spec: type: array items: type: string + exchangeLinkName: + type: boolean excludeNodes: type: array items: From 4dad754f210c98ae8661cce0653f5b4a9ee84e0d Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Wed, 27 Jul 2022 10:55:36 +0800 Subject: [PATCH 2/5] add e2e testing --- dist/images/install.sh | 4 ++++ pkg/controller/config.go | 43 +++++++++++++++++++---------------- pkg/controller/init.go | 1 + test/e2e/e2e_suite_test.go | 3 ++- test/e2e/underlay/underlay.go | 20 ++++++++++++---- 5 files changed, 46 insertions(+), 25 deletions(-) diff --git a/dist/images/install.sh b/dist/images/install.sh index f4091402ea0..de7e5938db4 100755 --- a/dist/images/install.sh +++ b/dist/images/install.sh @@ -19,6 +19,9 @@ ENABLE_EXTERNAL_VPC=${ENABLE_EXTERNAL_VPC:-true} CNI_CONFIG_PRIORITY=${CNI_CONFIG_PRIORITY:-01} ENABLE_LB_SVC=${ENABLE_LB_SVC:-false} ENABLE_KEEP_VM_IP=${ENABLE_KEEP_VM_IP:-true} +# exchange link names of OVS bridge and the provider nic +# in the default provider-network +EXCHANGE_LINK_NAME=${EXCHANGE_LINK_NAME:-false} # The nic to support container network can be a nic name or a group of regex # separated by comma, if empty will use the nic that the default route use IFACE=${IFACE:-} @@ -2797,6 +2800,7 @@ spec: - --service-cluster-ip-range=$SVC_CIDR - --network-type=$NETWORK_TYPE - --default-interface-name=$VLAN_INTERFACE_NAME + - --default-exchange-link-name=$EXCHANGE_LINK_NAME - --default-vlan-id=$VLAN_ID - --ls-dnat-mod-dl-dst=$LS_DNAT_MOD_DL_DST - --pod-nic-type=$POD_NIC_TYPE diff --git a/pkg/controller/config.go b/pkg/controller/config.go index 33e2f9e9d50..5c0dc0a8e88 100644 --- a/pkg/controller/config.go +++ b/pkg/controller/config.go @@ -65,12 +65,13 @@ type Configuration struct { EnablePprof bool NodePgProbeTime int - NetworkType string - DefaultProviderName string - DefaultHostInterface string - DefaultVlanName string - DefaultVlanID int - LsDnatModDlDst bool + NetworkType string + DefaultProviderName string + DefaultHostInterface string + DefaultExchangeLinkName bool + DefaultVlanName string + DefaultVlanID int + LsDnatModDlDst bool EnableLb bool EnableNP bool @@ -122,20 +123,21 @@ func ParseFlags() (*Configuration, error) { argPprofPort = pflag.Int("pprof-port", 10660, "The port to get profiling data") argNodePgProbeTime = pflag.Int("nodepg-probe-time", 1, "The probe interval for node port-group, the unit is minute") - argNetworkType = pflag.String("network-type", util.NetworkTypeGeneve, "The ovn network type") - argDefaultProviderName = pflag.String("default-provider-name", "provider", "The vlan or vxlan type default provider interface name") - argDefaultInterfaceName = pflag.String("default-interface-name", "", "The default host interface name in the vlan/vxlan type") - argDefaultVlanName = pflag.String("default-vlan-name", "ovn-vlan", "The default vlan name") - argDefaultVlanID = pflag.Int("default-vlan-id", 1, "The default vlan id") - argLsDnatModDlDst = pflag.Bool("ls-dnat-mod-dl-dst", true, "Set ethernet destination address for DNAT on logical switch") - argPodNicType = pflag.String("pod-nic-type", "veth-pair", "The default pod network nic implementation type") - argEnableLb = pflag.Bool("enable-lb", true, "Enable load balancer") - argEnableNP = pflag.Bool("enable-np", true, "Enable network policy support") - argEnableEipSnat = pflag.Bool("enable-eip-snat", true, "Enable EIP and SNAT") - argEnableExternalVpc = pflag.Bool("enable-external-vpc", true, "Enable external vpc support") - argEnableEcmp = pflag.Bool("enable-ecmp", false, "Enable ecmp route for centralized subnet") - argKeepVmIP = pflag.Bool("keep-vm-ip", false, "Whether to keep ip for kubevirt pod when pod is rebuild") - argEnableLbSvc = pflag.Bool("enable-lb-svc", false, "Whether to support loadbalancer service") + argNetworkType = pflag.String("network-type", util.NetworkTypeGeneve, "The ovn network type") + argDefaultProviderName = pflag.String("default-provider-name", "provider", "The vlan or vxlan type default provider interface name") + argDefaultInterfaceName = pflag.String("default-interface-name", "", "The default host interface name in the vlan/vxlan type") + argDefaultExchangeLinkName = pflag.Bool("default-exchange-link-name", false, "exchange link names of OVS bridge and the provider nic in the default provider-network") + argDefaultVlanName = pflag.String("default-vlan-name", "ovn-vlan", "The default vlan name") + argDefaultVlanID = pflag.Int("default-vlan-id", 1, "The default vlan id") + argLsDnatModDlDst = pflag.Bool("ls-dnat-mod-dl-dst", true, "Set ethernet destination address for DNAT on logical switch") + argPodNicType = pflag.String("pod-nic-type", "veth-pair", "The default pod network nic implementation type") + argEnableLb = pflag.Bool("enable-lb", true, "Enable load balancer") + argEnableNP = pflag.Bool("enable-np", true, "Enable network policy support") + argEnableEipSnat = pflag.Bool("enable-eip-snat", true, "Enable EIP and SNAT") + argEnableExternalVpc = pflag.Bool("enable-external-vpc", true, "Enable external vpc support") + argEnableEcmp = pflag.Bool("enable-ecmp", false, "Enable ecmp route for centralized subnet") + argKeepVmIP = pflag.Bool("keep-vm-ip", false, "Whether to keep ip for kubevirt pod when pod is rebuild") + argEnableLbSvc = pflag.Bool("enable-lb-svc", false, "Whether to support loadbalancer service") argExternalGatewayConfigNS = pflag.String("external-gateway-config-ns", "kube-system", "The namespace of configmap external-gateway-config, default: kube-system") argExternalGatewayNet = pflag.String("external-gateway-net", "external", "The name of the external network which mappings with an ovs bridge, default: external") @@ -192,6 +194,7 @@ func ParseFlags() (*Configuration, error) { LsDnatModDlDst: *argLsDnatModDlDst, DefaultProviderName: *argDefaultProviderName, DefaultHostInterface: *argDefaultInterfaceName, + DefaultExchangeLinkName: *argDefaultExchangeLinkName, DefaultVlanName: *argDefaultVlanName, PodName: os.Getenv("POD_NAME"), PodNamespace: os.Getenv("KUBE_NAMESPACE"), diff --git a/pkg/controller/init.go b/pkg/controller/init.go index dff2119f0bb..d45ddf8db47 100644 --- a/pkg/controller/init.go +++ b/pkg/controller/init.go @@ -467,6 +467,7 @@ func (c *Controller) initDefaultProviderNetwork() error { }, Spec: kubeovnv1.ProviderNetworkSpec{ DefaultInterface: c.config.DefaultHostInterface, + ExchangeLinkName: c.config.DefaultExchangeLinkName, }, } diff --git a/test/e2e/e2e_suite_test.go b/test/e2e/e2e_suite_test.go index dbf42c4e5aa..693dbef9caa 100644 --- a/test/e2e/e2e_suite_test.go +++ b/test/e2e/e2e_suite_test.go @@ -213,7 +213,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { providerInterface := underlay.UnderlayInterface if underlay.VlanID != "" { if vlanID, err = strconv.Atoi(underlay.VlanID); err != nil || vlanID <= 0 || vlanID > 4095 { - Fail(underlay.VlanID + " is not an invalid VLAN id") + Fail(underlay.VlanID + " is not a valid VLAN ID") } providerInterface = underlay.VlanInterface } @@ -291,6 +291,7 @@ var _ = SynchronizedBeforeSuite(func() []byte { }, Spec: kubeovn.ProviderNetworkSpec{ DefaultInterface: providerInterface, + ExchangeLinkName: underlay.ExchangeLinkName, }, } if _, err = f.OvnClientSet.KubeovnV1().ProviderNetworks().Create(context.Background(), pn, metav1.CreateOptions{}); err != nil { diff --git a/test/e2e/underlay/underlay.go b/test/e2e/underlay/underlay.go index 0034cdfbdd9..ff740791b0e 100644 --- a/test/e2e/underlay/underlay.go +++ b/test/e2e/underlay/underlay.go @@ -3,6 +3,7 @@ package underlay import ( "context" "fmt" + "math/rand" "os" "os/exec" "strings" @@ -34,6 +35,8 @@ const ( ) var ( + ExchangeLinkName bool + VlanID = os.Getenv("VLAN_ID") cidr string @@ -45,6 +48,11 @@ var ( nodeMTU = make(map[string]int) ) +func init() { + rand.Seed(time.Now().UnixNano()) + ExchangeLinkName = rand.Intn(2) != 0 +} + func SetCIDR(s string) { cidr = s } @@ -100,7 +108,11 @@ var _ = Describe("[Underlay]", func() { } Expect(ovsPod).NotTo(BeNil()) - stdout, _, err := f.ExecToPodThroughAPI("ip addr show "+providerInterface, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil) + nic, br := providerInterface, util.ExternalBridgeName(ProviderNetwork) + if ExchangeLinkName { + nic, br = br, nic + } + stdout, _, err := f.ExecToPodThroughAPI("ip addr show "+nic, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil) Expect(err).NotTo(HaveOccurred()) addrNotFound := make([]bool, len(nodeAddrs[node.Name])) @@ -117,19 +129,19 @@ var _ = Describe("[Underlay]", func() { Expect(found).To(BeTrue()) } - stdout, _, err = f.ExecToPodThroughAPI("ovs-vsctl list-ports "+util.ExternalBridgeName(ProviderNetwork), "openvswitch", ovsPod.Name, ovsPod.Namespace, nil) + stdout, _, err = f.ExecToPodThroughAPI("ovs-vsctl list-ports "+br, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil) Expect(err).NotTo(HaveOccurred()) var portFound bool for _, port := range strings.Split(stdout, "\n") { - if port == providerInterface { + if port == nic { portFound = true break } } Expect(portFound).To(BeTrue()) - stdout, _, err = f.ExecToPodThroughAPI("ip addr show "+util.ExternalBridgeName(ProviderNetwork), "openvswitch", ovsPod.Name, ovsPod.Namespace, nil) + stdout, _, err = f.ExecToPodThroughAPI("ip addr show "+br, "openvswitch", ovsPod.Name, ovsPod.Namespace, nil) Expect(err).NotTo(HaveOccurred()) var isUp bool From 4273f3a3ce1104e7026f018b363661d5f4a90b30 Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Wed, 27 Jul 2022 14:30:19 +0800 Subject: [PATCH 3/5] fix bug --- pkg/daemon/init_linux.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/daemon/init_linux.go b/pkg/daemon/init_linux.go index a4eef7611a7..a24b40524fc 100644 --- a/pkg/daemon/init_linux.go +++ b/pkg/daemon/init_linux.go @@ -47,7 +47,7 @@ func changeProvideNicName(current, target string) (bool, error) { } if link.Type() == "openvswitch" { klog.Infof("%s is an openvswitch interface, skip", current) - return false, nil + return true, nil } // set link unmanaged by NetworkManager to avoid getting new IP by DHCP From d197e8d8f7ed8d43c29b5e078d6166b240d51d56 Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Fri, 29 Jul 2022 09:45:23 +0800 Subject: [PATCH 4/5] add logging --- pkg/daemon/init_linux.go | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/pkg/daemon/init_linux.go b/pkg/daemon/init_linux.go index a24b40524fc..af072250a05 100644 --- a/pkg/daemon/init_linux.go +++ b/pkg/daemon/init_linux.go @@ -23,17 +23,24 @@ func nmSetManaged(device string, managed bool) error { d, err := nm.GetDeviceByIpIface(device) if err != nil { + klog.Errorf("failed to get device by IP iface %s: %v", device, err) return err } current, err := d.GetPropertyManaged() if err != nil { + klog.Errorf("failed to get device property managed: %v", err) return err } if current == managed { return nil } - return d.SetPropertyManaged(managed) + if err = d.SetPropertyManaged(managed); err != nil { + klog.Errorf("failed to set device property managed to %v: %v", managed, err) + return err + } + + return nil } func changeProvideNicName(current, target string) (bool, error) { From 1a24c7de36fc9a22af11b069164311c93bed063f Mon Sep 17 00:00:00 2001 From: zhangzujian Date: Fri, 29 Jul 2022 11:21:53 +0800 Subject: [PATCH 5/5] add more logging --- pkg/daemon/init.go | 2 ++ pkg/daemon/init_linux.go | 9 +++++++++ 2 files changed, 11 insertions(+) diff --git a/pkg/daemon/init.go b/pkg/daemon/init.go index aef237c8db4..42997f8aa5b 100644 --- a/pkg/daemon/init.go +++ b/pkg/daemon/init.go @@ -102,6 +102,7 @@ func ovsInitProviderNetwork(provider, nic string, exchangeLinkName bool) (int, e if exchangeLinkName { exchanged, err := changeProvideNicName(nic, brName) if err != nil { + klog.Errorf("failed to change provider nic name from %s to %s: %v", nic, brName, err) return 0, err } if exchanged { @@ -228,6 +229,7 @@ func ovsCleanProviderNetwork(provider string) error { if br := util.ExternalBridgeName(provider); br != brName { if _, err = changeProvideNicName(br, brName); err != nil { + klog.Errorf("failed to change provider nic name from %s to %s: %v", br, brName, err) return err } } diff --git a/pkg/daemon/init_linux.go b/pkg/daemon/init_linux.go index af072250a05..8ff51195af9 100644 --- a/pkg/daemon/init_linux.go +++ b/pkg/daemon/init_linux.go @@ -50,6 +50,7 @@ func changeProvideNicName(current, target string) (bool, error) { klog.Infof("link %s not found, skip", current) return false, nil } + klog.Errorf("failed to get link %s: %v", current, err) return false, err } if link.Type() == "openvswitch" { @@ -59,26 +60,32 @@ func changeProvideNicName(current, target string) (bool, error) { // set link unmanaged by NetworkManager to avoid getting new IP by DHCP if err = nmSetManaged(current, false); err != nil { + klog.Errorf("failed set device %s to unmanaged by NetworkManager: %v", current, err) return false, err } klog.Infof("change nic name from %s to %s", current, target) addresses, err := netlink.AddrList(link, netlink.FAMILY_ALL) if err != nil { + klog.Errorf("failed to list addresses of link %s: %v", current, err) return false, err } routes, err := netlink.RouteList(link, netlink.FAMILY_ALL) if err != nil { + klog.Errorf("failed to list routes of link %s: %v", current, err) return false, err } if err = netlink.LinkSetDown(link); err != nil { + klog.Errorf("failed to set link %s down: %v", current, err) return false, err } if err = netlink.LinkSetName(link, target); err != nil { + klog.Errorf("failed to set name of link %s to %s: %v", current, target, err) return false, err } if err = netlink.LinkSetUp(link); err != nil { + klog.Errorf("failed to set link %s up: %v", target, err) return false, err } @@ -87,6 +94,7 @@ func changeProvideNicName(current, target string) (bool, error) { continue } if err = netlink.AddrReplace(link, &addr); err != nil { + klog.Errorf("failed to replace address %s: %v", addr.String(), err) return false, err } } @@ -98,6 +106,7 @@ func changeProvideNicName(current, target string) (bool, error) { } if route.Scope == scope { if err = netlink.RouteReplace(&route); err != nil { + klog.Errorf("failed to replace route %s: %v", route.String(), err) return false, err } }