add scheduled e2e testing (#2144)

kubeovn · Dec 13, 2022 · 57457bd · 57457bd
1 parent 5444126
commit 57457bd
Show file tree

Hide file tree

Showing 7 changed files with 652 additions and 2 deletions.
diff --git a/.github/workflows/scheduled-e2e.yaml b/.github/workflows/scheduled-e2e.yaml
@@ -0,0 +1,229 @@
+name: Scheduled E2E
+
+on:
+  workflow_dispatch:
+  schedule:
+    - cron: "40 16 * * *"
+
+concurrency:
+  group: "${{ github.workflow }}-${{ github.ref }}"
+  cancel-in-progress: true
+
+env:
+  GO_VERSION: '1.19'
+  KIND_VERSION: v0.17.0
+  HELM_VERSION: v3.10.1
+
+jobs:
+  k8s-conformance-e2e:
+    name: Kubernetes Conformance E2E
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        branch:
+          - master
+        ip-family:
+          - ipv4
+          - ipv6
+          - dual
+        mode:
+          - overlay
+          - underlay
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Remove DNS search domain
+        run: |
+          sudo sed -i '/^search/d' /etc/resolv.conf
+          sudo systemctl restart docker
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '${{ env.GO_VERSION }}'
+          check-latest: true
+        id: go
+
+      - name: Export Go full version
+        run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV"
+
+      - name: Go Cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-
+
+      - name: Create branch directory
+        run: mkdir -p test/k8s-network/branches/${{ matrix.branch }}
+
+      - name: Check out branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ matrix.branch }}
+          fetch-depth: 1
+          path: test/k8s-network/branches/${{ matrix.branch }}
+
+      - name: Install Kind
+        run: |
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+
+      - name: Create cluster
+        working-directory: test/k8s-network/branches/${{ matrix.branch }}
+        run: |
+          sudo pip3 install j2cli
+          sudo pip3 install "j2cli[yaml]"
+          sudo PATH=~/.local/bin:$PATH make kind-init-${{ matrix.ip-family }}
+
+      - name: Install Kube-OVN
+        working-directory: test/k8s-network/branches/${{ matrix.branch }}
+        run: |
+          docker pull kubeovn/kube-ovn:$(cat VERSION)
+          sudo make kind-install-${{ matrix.mode }}-${{ matrix.ip-family }}
+
+      - name: Run E2E
+        env:
+          E2E_BRANCH: ${{ matrix.branch }}
+          E2E_IP_FAMILY: ${{ matrix.ip-family }}
+          E2E_NETWORK_MODE: ${{ matrix.mode }}
+        run: |
+          sudo cp -r /root/.kube/ ~/.kube/
+          sudo chmod -R 777 ~/.kube/
+          KUBECONFIG=~/.kube/config make k8s-conformance-e2e
+
+  k8s-netpol-e2e:
+    name: Kubernetes Network Policy E2E
+    runs-on: ubuntu-22.04
+    timeout-minutes: 60
+    strategy:
+      fail-fast: false
+      matrix:
+        branch:
+          - master
+        ip-family:
+          - ipv4
+          - ipv6
+          - dual
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Remove DNS search domain
+        run: |
+          sudo sed -i '/^search/d' /etc/resolv.conf
+          sudo systemctl restart docker
+
+      - uses: actions/setup-go@v3
+        with:
+          go-version: '${{ env.GO_VERSION }}'
+          check-latest: true
+        id: go
+
+      - name: Export Go full version
+        run: echo "GO_FULL_VER=$(go version | awk '{print $3}')" >> "$GITHUB_ENV"
+
+      - name: Go Cache
+        uses: actions/cache@v3
+        with:
+          path: |
+            ~/.cache/go-build
+            ~/go/pkg/mod
+          key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles('**/go.sum') }}
+          restore-keys: |
+            ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-
+
+      - name: Create branch directory
+        run: mkdir -p test/k8s-network/branches/${{ matrix.branch }}
+
+      - name: Check out branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ matrix.branch }}
+          fetch-depth: 1
+          path: test/k8s-network/branches/${{ matrix.branch }}
+
+      - name: Install Kind
+        run: |
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+
+      - name: Create cluster
+        working-directory: test/k8s-network/branches/${{ matrix.branch }}
+        run: |
+          sudo pip3 install j2cli
+          sudo pip3 install "j2cli[yaml]"
+          sudo PATH=~/.local/bin:$PATH make kind-init-${{ matrix.ip-family }}
+
+      - name: Install Kube-OVN
+        working-directory: test/k8s-network/branches/${{ matrix.branch }}
+        run: |
+          docker pull kubeovn/kube-ovn:$(cat VERSION)
+          sudo make kind-install-${{ matrix.ip-family }}
+
+      - name: Run E2E
+        run: |
+          sudo cp -r /root/.kube/ ~/.kube/
+          sudo chmod -R 777 ~/.kube/
+          KUBECONFIG=~/.kube/config make k8s-netpol-e2e
+
+  cyclonus-netpol-e2e:
+    name: Cyclonus Network Policy E2E
+    runs-on: ubuntu-22.04
+    timeout-minutes: 30
+    strategy:
+      fail-fast: false
+      matrix:
+        branch:
+          - master
+        ip-family:
+          - ipv4
+          - ipv6
+          - dual
+    steps:
+      - uses: actions/checkout@v3
+
+      - name: Remove DNS search domain
+        run: |
+          sudo sed -i '/^search/d' /etc/resolv.conf
+          sudo systemctl restart docker
+
+      - name: Create branch directory
+        run: mkdir -p test/k8s-network/branches/${{ matrix.branch }}
+
+      - name: Check out branch
+        uses: actions/checkout@v3
+        with:
+          ref: ${{ matrix.branch }}
+          fetch-depth: 1
+          path: test/k8s-network/branches/${{ matrix.branch }}
+
+      - name: Install Kind
+        run: |
+          curl -Lo ./kind https://github.com/kubernetes-sigs/kind/releases/download/${KIND_VERSION}/kind-$(uname)-amd64
+          chmod +x ./kind
+          sudo mv kind /usr/local/bin
+
+      - name: Create cluster
+        working-directory: test/k8s-network/branches/${{ matrix.branch }}
+        run: |
+          sudo pip3 install j2cli
+          sudo pip3 install "j2cli[yaml]"
+          sudo PATH=~/.local/bin:$PATH make kind-init-${{ matrix.ip-family }}
+
+      - name: Install Kube-OVN
+        working-directory: test/k8s-network/branches/${{ matrix.branch }}
+        run: |
+          docker pull kubeovn/kube-ovn:$(cat VERSION)
+          sudo make kind-install-${{ matrix.ip-family }}
+
+      - name: Run E2E
+        run: |
+          sudo cp -r /root/.kube/ ~/.kube/
+          sudo chmod -R 777 ~/.kube/
+          make cyclonus-netpol-e2e
diff --git a/Makefile b/Makefile
@@ -1,5 +1,6 @@
-GO_VERSION = 1.19
-SHELL=/bin/bash
+SHELL = /bin/bash
+
+include Makefile.e2e
 
 REGISTRY = kubeovn
 DEV_TAG = dev
@@ -235,6 +236,7 @@ kind-init-ipv4: kind-clean
 
 .PHONY: kind-init-ovn-ic
 kind-init-ovn-ic: kind-clean-ovn-ic kind-init-single
+	@single=true $(MAKE) kind-generate-config
 	$(call kind_create_cluster,yamls/kind.yaml,kube-ovn1)
 
 .PHONY: kind-init-iptables

diff --git a/Makefile.e2e b/Makefile.e2e
@@ -0,0 +1,50 @@
+E2E_BRANCH := $(shell echo $${E2E_BRANCH:-master})
+E2E_IP_FAMILY := $(shell echo $${E2E_IP_FAMILY:-ipv4})
+E2E_NETWORK_MODE := $(shell echo $${E2E_NETWORK_MODE:-overlay})
+
+K8S_CONFORMANCE_E2E_FOCUS = "sig-network.*Conformance" "sig-network.*Feature:NoSNAT"
+K8S_CONFORMANCE_E2E_SKIP = "sig-network.*Services.*session affinity"
+K8S_NETPOL_E2E_FOCUS = "sig-network.*Feature:NetworkPolicy"
+K8S_NETPOL_E2E_SKIP = "sig-network.*NetworkPolicyLegacy"
+
+ifeq ($(shell test $(E2E_IP_FAMILY) != ipv6 && echo true),true)
+K8S_CONFORMANCE_E2E_FOCUS += \
+	"sig-network.*Feature:Networking-IPv4" \
+	"sig-network.*Feature:Feature:Networking-DNS"
+endif
+
+ifeq ($(shell test $(E2E_IP_FAMILY) = dual && echo true),true)
+K8S_CONFORMANCE_E2E_FOCUS += "sig-network.*Feature:IPv6DualStack"
+endif
+
+define ginkgo_option
+--ginkgo.$(1)=$(shell echo '$(2)' | sed -E 's/^[[:space:]]+//' | sed -E 's/"[[:space:]]+"/" --ginkgo.$(1)="/g')
+endef
+
+.PHONY: k8s-conformance-e2e
+k8s-conformance-e2e:
+	go test ./test/k8s-network -c -o test/k8s-network/e2e.test
+	./test/k8s-network/e2e.test --ginkgo.timeout=1h \
+		$(call ginkgo_option,focus,$(K8S_CONFORMANCE_E2E_FOCUS)) \
+		$(call ginkgo_option,skip,$(K8S_CONFORMANCE_E2E_SKIP))
+
+.PHONY: k8s-netpol-e2e
+k8s-netpol-e2e:
+	go test ./test/k8s-network -c -o test/k8s-network/e2e.test
+	./test/k8s-network/e2e.test --ginkgo.timeout=2h \
+		$(call ginkgo_option,focus,$(K8S_NETPOL_E2E_FOCUS)) \
+		$(call ginkgo_option,skip,$(K8S_NETPOL_E2E_SKIP))
+
+.PHONY: cyclonus-netpol-e2e
+cyclonus-netpol-e2e:
+	kubectl create ns netpol
+	kubectl create clusterrolebinding cyclonus --clusterrole=cluster-admin --serviceaccount=netpol:cyclonus
+	kubectl create sa cyclonus -n netpol
+	kubectl create -f test/cyclonus.yaml -n netpol
+	while ! kubectl wait pod --for=condition=Ready -l job-name=cyclonus -n netpol; do \
+		sleep 3; \
+	done
+	kubectl logs -f -l job-name=cyclonus -n netpol
+	kubectl -n netpol logs \
+		$$(kubectl -n netpol get pod -l job-name=cyclonus -o=jsonpath={.items[0].metadata.name}) | \
+		grep failed; test $$? -ne 0