support specifying routes when providing IPAM for other CNI plugins (k…

…ubeovn#3904) Signed-off-by: zhangzujian <[email protected]>
zhangzujian · Apr 11, 2024 · b32f607 · b32f607
1 parent 96a5695
commit b32f607
Show file tree

Hide file tree

Showing 13 changed files with 723 additions and 67 deletions.
diff --git a/.github/workflows/build-x86-image.yaml b/.github/workflows/build-x86-image.yaml
@@ -1263,6 +1263,124 @@ jobs:
           name: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log
           path: kube-ovn-ic-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
 
+  multus-conformance-e2e:
+    name: Multus Conformance E2E
+    needs:
+      - build-kube-ovn
+      - build-e2e-binaries
+    runs-on: ubuntu-22.04
+    timeout-minutes: 10
+    strategy:
+      fail-fast: false
+      matrix:
+        ip-family:
+          - ipv4
+          - ipv6
+          - dual
+    steps:
+      - uses: jlumbroso/[email protected]
+        with:
+          android: true
+          dotnet: true
+          haskell: true
+          docker-images: false
+          large-packages: false
+          tool-cache: false
+          swap-storage: false
+
+      - uses: actions/checkout@v4
+
+      - name: Create the default branch directory
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        run: mkdir -p test/e2e/source
+
+      - name: Check out the default branch
+        if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
+        uses: actions/checkout@v4
+        with:
+          ref: ${{ github.event.repository.default_branch }}
+          fetch-depth: 1
+          path: test/e2e/source
+
+      - name: Export E2E directory
+        run: |
+          if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
+            echo "E2E_DIR=." >> "$GITHUB_ENV"
+          else
+            echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
+          fi
+
+      - uses: actions/setup-go@v5
+        with:
+          go-version: ${{ env.GO_VERSION || '' }}
+          go-version-file: ${{ env.E2E_DIR }}/go.mod
+          check-latest: true
+          cache: false
+
+      - name: Export Go full version
+        run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV"
+
+      - name: Go cache
+        uses: actions/cache/restore@v4
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
+          restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-
+
+      - name: Install kind
+        uses: helm/[email protected]
+        with:
+          version: ${{ env.KIND_VERSION }}
+          install_only: true
+
+      - name: Install ginkgo
+        working-directory: ${{ env.E2E_DIR }}
+        run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo
+
+      - name: Download kube-ovn image
+        uses: actions/download-artifact@v4
+        with:
+          name: kube-ovn
+
+      - name: Load images
+        run: docker load -i kube-ovn.tar
+
+      - name: Create kind cluster
+        run: |
+          sudo pip3 install j2cli
+          sudo pip3 install "j2cli[yaml]"
+          sudo PATH=~/.local/bin:$PATH make kind-init-${{ matrix.ip-family }}
+          sudo cp -r /root/.kube/ ~/.kube/
+          sudo chown -R $(id -un). ~/.kube/
+
+      - name: Install Kube-OVN
+        run: make kind-install-${{ matrix.ip-family }}
+
+      - name: Install Multus
+        run: make kind-install-multus
+
+      - name: Run E2E
+        working-directory: ${{ env.E2E_DIR }}
+        env:
+          E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
+          E2E_IP_FAMILY: ${{ matrix.ip-family }}
+        run: make kube-ovn-multus-conformance-e2e
+
+      - name: kubectl ko log
+        if: failure()
+        run: |
+          make kubectl-ko-log
+          mv kubectl-ko-log.tar.gz multus-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
+
+      - name: upload kubectl ko log
+        uses: actions/upload-artifact@v4
+        if: failure()
+        with:
+          name: multus-conformance-e2e-${{ matrix.ip-family }}-ko-log
+          path: multus-conformance-e2e-${{ matrix.ip-family }}-ko-log.tar.gz
+
   chart-test:
     name: Chart Installation/Uninstallation Test
     needs: build-kube-ovn
@@ -2377,6 +2495,7 @@ jobs:
       - cyclonus-netpol-e2e
       - kube-ovn-conformance-e2e
       - kube-ovn-ic-conformance-e2e
+      - multus-conformance-e2e
       - ovn-vpc-nat-gw-conformance-e2e
       - iptables-vpc-nat-gw-conformance-e2e
       - webhook-e2e

diff --git a/Makefile.e2e b/Makefile.e2e
@@ -60,6 +60,7 @@ e2e-build:
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/k8s-network
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/kube-ovn
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ovn-ic
+	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/multus
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/vip
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/iptables-vpc-nat-gw
@@ -130,6 +131,15 @@ kube-ovn-submariner-conformance-e2e:
 		--context kind-kube-ovn --tocontext kind-kube-ovn1 \
 		--verbose --disruptive-tests
 
+.PHONY: kube-ovn-multus-conformance-e2e
+kube-ovn-multus-conformance-e2e:
+	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/multus
+	E2E_BRANCH=$(E2E_BRANCH) \
+	E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
+	E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
+	ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \
+		--focus=CNI:Kube-OVN ./test/e2e/multus/multus.test -- $(TEST_BIN_ARGS)
+
 .PHONY: kube-ovn-lb-svc-conformance-e2e
 kube-ovn-lb-svc-conformance-e2e:
 	ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/lb-svc

diff --git a/cmd/cni/cni.go b/cmd/cni/cni.go
@@ -13,6 +13,7 @@ import (
 	"github.com/containernetworking/cni/pkg/version"
 
 	kubeovnv1 "github.com/kubeovn/kube-ovn/pkg/apis/kubeovn/v1"
+	"github.com/kubeovn/kube-ovn/pkg/netconf"
 	"github.com/kubeovn/kube-ovn/pkg/request"
 	"github.com/kubeovn/kube-ovn/pkg/util"
 	"github.com/kubeovn/kube-ovn/versions"
@@ -73,6 +74,7 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R
 	result := current.Result{
 		CNIVersion: current.ImplementedSpecVersion,
 		DNS:        cniResponse.DNS,
+		Routes:     parseRoutes(cniResponse.Routes),
 	}
 	_, mask, _ := net.ParseCIDR(cniResponse.CIDR)
 	podIface := current.Interface{
@@ -84,20 +86,21 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R
 	case kubeovnv1.ProtocolIPv4:
 		ip, route := assignV4Address(cniResponse.IPAddress, cniResponse.Gateway, mask)
 		result.IPs = []*current.IPConfig{ip}
-		if route != nil {
+		if len(result.Routes) == 0 && route != nil {
 			result.Routes = []*types.Route{route}
 		}
 		result.Interfaces = []*current.Interface{&podIface}
 	case kubeovnv1.ProtocolIPv6:
 		ip, route := assignV6Address(cniResponse.IPAddress, cniResponse.Gateway, mask)
 		result.IPs = []*current.IPConfig{ip}
-		if route != nil {
+		if len(result.Routes) == 0 && route != nil {
 			result.Routes = []*types.Route{route}
 		}
 		result.Interfaces = []*current.Interface{&podIface}
 	case kubeovnv1.ProtocolDual:
 		var netMask *net.IPNet
 		var gwStr string
+		addRoutes := len(result.Routes) == 0
 		for _, cidrBlock := range strings.Split(cniResponse.CIDR, ",") {
 			_, netMask, _ = net.ParseCIDR(cidrBlock)
 			gwStr = ""
@@ -109,7 +112,7 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R
 
 				ip, route := assignV4Address(ipStr, gwStr, netMask)
 				result.IPs = append(result.IPs, ip)
-				if route != nil {
+				if addRoutes && route != nil {
 					result.Routes = append(result.Routes, route)
 				}
 			} else if util.CheckProtocol(cidrBlock) == kubeovnv1.ProtocolIPv6 {
@@ -120,7 +123,7 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R
 
 				ip, route := assignV6Address(ipStr, gwStr, netMask)
 				result.IPs = append(result.IPs, ip)
-				if route != nil {
+				if addRoutes && route != nil {
 					result.Routes = append(result.Routes, route)
 				}
 			}
@@ -131,6 +134,24 @@ func generateCNIResult(cniResponse *request.CniResponse, netns string) current.R
 	return result
 }
 
+func parseRoutes(routes []request.Route) []*types.Route {
+	parsedRoutes := make([]*types.Route, len(routes))
+	for i, r := range routes {
+		if r.Destination == "" {
+			if util.CheckProtocol(r.Gateway) == kubeovnv1.ProtocolIPv4 {
+				r.Destination = "0.0.0.0/0"
+			} else {
+				r.Destination = "::/0"
+			}
+		}
+		parsedRoutes[i] = &types.Route{GW: net.ParseIP(r.Gateway)}
+		if _, cidr, err := net.ParseCIDR(r.Destination); err == nil {
+			parsedRoutes[i].Dst = *cidr
+		}
+	}
+	return parsedRoutes
+}
+
 func cmdDel(args *skel.CmdArgs) error {
 	netConf, _, err := loadNetConf(args.StdinData)
 	if err != nil {
@@ -167,20 +188,16 @@ func cmdDel(args *skel.CmdArgs) error {
 	return nil
 }
 
-type ipamConf struct {
-	ServerSocket string `json:"server_socket"`
-	Provider     string `json:"provider"`
-}
-
-func loadNetConf(bytes []byte) (*netConf, string, error) {
-	n := &netConf{}
+func loadNetConf(bytes []byte) (*netconf.NetConf, string, error) {
+	n := &netconf.NetConf{}
 	if err := json.Unmarshal(bytes, n); err != nil {
 		return nil, "", types.NewError(types.ErrDecodingFailure, "failed to load netconf", err.Error())
 	}
 
 	if n.Type != util.CniTypeName && n.IPAM != nil {
 		n.Provider = n.IPAM.Provider
 		n.ServerSocket = n.IPAM.ServerSocket
+		n.Routes = n.IPAM.Routes
 	}
 
 	if n.ServerSocket == "" {
@@ -191,7 +208,7 @@ func loadNetConf(bytes []byte) (*netConf, string, error) {
 		n.Provider = util.OvnProvider
 	}
 
-	n.postLoad()
+	n.PostLoad()
 	return n, n.CNIVersion, nil
 }
 

diff --git a/pkg/daemon/handler.go b/pkg/daemon/handler.go
@@ -4,6 +4,7 @@ import (
 	"context"
 	"encoding/json"
 	"fmt"
+	"net"
 	"net/http"
 	"strconv"
 	"strings"
@@ -211,6 +212,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 		return
 	}
 
+	routes = append(podRequest.Routes, routes...)
 	if strings.HasSuffix(podRequest.Provider, util.OvnProvider) && subnet != "" {
 		podSubnet, err := csh.Controller.subnetsLister.Get(subnet)
 		if err != nil {
@@ -284,17 +286,17 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 			}
 		}
 
-		routes = append(podRequest.Routes, routes...)
 		macAddr = pod.Annotations[fmt.Sprintf(util.MacAddressAnnotationTemplate, podRequest.Provider)]
 		klog.Infof("create container interface %s mac %s, ip %s, cidr %s, gw %s, custom routes %v", ifName, macAddr, ipAddr, cidr, gw, routes)
 		podNicName = ifName
 		switch nicType {
 		case util.InternalType:
-			podNicName, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP)
+			podNicName, routes, err = csh.configureNicWithInternalPort(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP)
 		case util.DpdkType:
 			err = csh.configureDpdkNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, ifName, macAddr, mtu, ipAddr, gw, ingress, egress, getShortSharedDir(pod.UID, podRequest.VhostUserSocketVolumeName), podRequest.VhostUserSocketName)
+			routes = nil
 		default:
-			err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP)
+			routes, err = csh.configureNic(podRequest.PodName, podRequest.PodNamespace, podRequest.Provider, podRequest.NetNs, podRequest.ContainerID, podRequest.VfDriver, ifName, macAddr, mtu, ipAddr, gw, isDefaultRoute, detectIPConflict, routes, podRequest.DNS.Nameservers, podRequest.DNS.Search, ingress, egress, podRequest.DeviceID, nicType, latency, limit, loss, jitter, gatewayCheckMode, u2oInterconnectionIP)
 		}
 		if err != nil {
 			errMsg := fmt.Errorf("configure nic failed %v", err)
@@ -319,6 +321,30 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 			}
 			return
 		}
+	} else if len(routes) != 0 {
+		hasDefaultRoute := make(map[string]bool, 2)
+		for _, r := range routes {
+			if r.Destination == "" {
+				hasDefaultRoute[util.CheckProtocol(r.Gateway)] = true
+				continue
+			}
+			if _, cidr, err := net.ParseCIDR(r.Destination); err == nil {
+				if ones, _ := cidr.Mask.Size(); ones == 0 {
+					hasDefaultRoute[util.CheckProtocol(r.Gateway)] = true
+				}
+			}
+		}
+		if len(hasDefaultRoute) != 0 {
+			// remove existing default route so other CNI plugins, such as macvlan, can add the new default route correctly
+			if err = csh.removeDefaultRoute(podRequest.NetNs, hasDefaultRoute[kubeovnv1.ProtocolIPv4], hasDefaultRoute[kubeovnv1.ProtocolIPv6]); err != nil {
+				errMsg := fmt.Errorf("failed to remove existing default route for interface %s of pod %s/%s: %v", podRequest.IfName, podRequest.PodNamespace, podRequest.PodName, err)
+				klog.Error(errMsg)
+				if err = resp.WriteHeaderAndEntity(http.StatusInternalServerError, request.CniResponse{Err: errMsg.Error()}); err != nil {
+					klog.Errorf("failed to write response: %v", err)
+				}
+				return
+			}
+		}
 	}
 
 	response := &request.CniResponse{
@@ -327,6 +353,7 @@ func (csh cniServerHandler) handleAdd(req *restful.Request, resp *restful.Respon
 		MacAddress: macAddr,
 		CIDR:       cidr,
 		PodNicName: podNicName,
+		Routes:     routes,
 	}
 	if isDefaultRoute {
 		response.Gateway = gw