Skip to content

Commit

Permalink
refactor ipsec function (#4334)
Browse files Browse the repository at this point in the history
refactor ovn ipsec function

Signed-off-by: clyi <[email protected]>
  • Loading branch information
changluyi authored Aug 12, 2024
1 parent b32eb6e commit 502a90e
Show file tree
Hide file tree
Showing 24 changed files with 1,313 additions and 107 deletions.
135 changes: 135 additions & 0 deletions .github/workflows/build-x86-image.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -2685,6 +2685,141 @@ jobs:
if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.vip-e2e.conclusion == 'failure' || steps.vpc-e2e.conclusion == 'failure')) }}
run: make check-kube-ovn-pod-restarts

kube-ovn-ipsec-e2e:
name: OVN IPSEC E2E
needs:
- build-kube-ovn
- build-e2e-binaries
runs-on: ubuntu-24.04
timeout-minutes: 15
steps:
- uses: jlumbroso/[email protected]
with:
android: true
dotnet: true
haskell: true
docker-images: false
large-packages: false
tool-cache: false
swap-storage: false

- uses: actions/checkout@v4

- name: Create the default branch directory
if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
run: mkdir -p test/e2e/source

- name: Check out the default branch
if: (github.base_ref || github.ref_name) != github.event.repository.default_branch
uses: actions/checkout@v4
with:
ref: ${{ github.event.repository.default_branch }}
fetch-depth: 1
path: test/e2e/source

- name: Export E2E directory
run: |
if [ '${{ github.base_ref || github.ref_name }}' = '${{ github.event.repository.default_branch }}' ]; then
echo "E2E_DIR=." >> "$GITHUB_ENV"
else
echo "E2E_DIR=test/e2e/source" >> "$GITHUB_ENV"
fi
- uses: actions/setup-go@v5
with:
go-version: ${{ env.GO_VERSION || '' }}
go-version-file: ${{ env.E2E_DIR }}/go.mod
check-latest: true
cache: false

- name: Export Go full version
run: echo "GO_FULL_VER=$(go env GOVERSION)" >> "$GITHUB_ENV"

- name: Go cache
uses: actions/cache/restore@v4
with:
path: |
~/.cache/go-build
~/go/pkg/mod
key: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-${{ hashFiles(format('{0}/**/go.sum', env.E2E_DIR)) }}
restore-keys: ${{ runner.os }}-e2e-${{ env.GO_FULL_VER }}-x86-

- name: Install kind
uses: helm/[email protected]
with:
version: ${{ env.KIND_VERSION }}
install_only: true

- name: Install ginkgo
working-directory: ${{ env.E2E_DIR }}
run: go install -v -mod=mod github.com/onsi/ginkgo/v2/ginkgo

- name: Download kube-ovn image
uses: actions/download-artifact@v4
with:
name: kube-ovn

- name: Load images
run: docker load -i kube-ovn.tar

- name: Create kind cluster
run: |
pipx install jinjanator
make kind-init
- name: Install Kube-OVN
id: install
run: make kind-install-ovn-ipsec

- name: Run Ovn IPSEC E2E
id: kube-ovn-ipsec-e2e
working-directory: ${{ env.E2E_DIR }}
env:
E2E_BRANCH: ${{ github.base_ref || github.ref_name }}
run: make kube-ovn-ipsec-e2e

- name: Collect k8s events
if: failure() && ( steps.ovn-ipsec-e2e.conclusion == 'failure')
run: |
kubectl get events -A -o yaml > kube-ovn-ipsec-e2e-events.yaml
tar zcf kube-ovn-ipsec-e2e-events.tar.gz kube-ovn-ipsec-e2e-events.yaml
- name: Upload k8s events
uses: actions/upload-artifact@v4
if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
with:
name: kube-ovn-ipsec-e2e-events
path: kube-ovn-ipsec-e2e-events.tar.gz

- name: Collect apiserver audit logs
if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
run: |
docker cp kube-ovn-control-plane:/var/log/kubernetes/kube-apiserver-audit.log .
tar zcf kube-ovn-ipsec-e2e-audit-log.tar.gz kube-apiserver-audit.log
- name: Upload apiserver audit logs
uses: actions/upload-artifact@v4
if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
with:
name: kube-ovn-ipsec-e2e-audit-log
path: kube-ovn-ipsec-e2e-audit-log.tar.gz

- name: kubectl ko log
if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
run: |
make kubectl-ko-log
mv kubectl-ko-log.tar.gz kube-ovn-ipsec-e2e-ko-log.tar.gz
- name: upload kubectl ko log
uses: actions/upload-artifact@v4
if: failure() && (steps.kube-ovn-ipsec-e2e.conclusion == 'failure')
with:
name: kube-ovn-ipsec-e2e-ko-log
path: kube-ovn-ipsec-e2e-ko-log.tar.gz

- name: Check kube ovn pod restarts
if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.kube-ovn-ipsec-e2e.conclusion == 'failure')) }}
run: make check-kube-ovn-pod-restarts
push:
name: Push Images
needs:
Expand Down
7 changes: 6 additions & 1 deletion Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -913,6 +913,11 @@ kind-install-kwok:
kubectl apply -f kwok-node.yaml; \
done

.PHONY: kind-install-ovn-ipsec
kind-install-ovn-ipsec: kind-load-image
kubectl config use-context kind-kube-ovn
@$(MAKE) ENABLE_OVN_IPSEC=true DEBUG_WRAPPER=true kind-install

.PHONY: kind-reload
kind-reload: kind-reload-ovs
kubectl delete pod -n kube-system -l app=kube-ovn-controller
Expand Down Expand Up @@ -1024,4 +1029,4 @@ changelog:
local-dev: build-go
docker buildx build --platform linux/amd64 -t $(REGISTRY)/kube-ovn:$(RELEASE_TAG) --build-arg VERSION=$(RELEASE_TAG) -o type=docker -f dist/images/Dockerfile dist/images/
docker buildx build --platform linux/amd64 -t $(REGISTRY)/vpc-nat-gateway:$(RELEASE_TAG) -o type=docker -f dist/images/vpcnatgateway/Dockerfile dist/images/vpcnatgateway
@$(MAKE) kind-init kind-install
@$(MAKE) kind-init kind-install
9 changes: 9 additions & 0 deletions Makefile.e2e
Original file line number Diff line number Diff line change
Expand Up @@ -220,3 +220,12 @@ kube-ovn-webhook-e2e:
E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \
--focus=CNI:Kube-OVN ./test/e2e/webhook/webhook.test -- $(TEST_BIN_ARGS)

.PHONY: kube-ovn-ipsec-e2e
kube-ovn-ipsec-e2e:
ginkgo build $(E2E_BUILD_FLAGS) ./test/e2e/ipsec
E2E_BRANCH=$(E2E_BRANCH) \
E2E_IP_FAMILY=$(E2E_IP_FAMILY) \
E2E_NETWORK_MODE=$(E2E_NETWORK_MODE) \
ginkgo $(GINKGO_OUTPUT_OPT) $(GINKGO_PARALLEL_OPT) --randomize-all -v \
--focus=CNI:Kube-OVN ./test/e2e/ipsec/ipsec.test -- $(TEST_BIN_ARGS)
1 change: 1 addition & 0 deletions charts/kube-ovn/templates/controller-deploy.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -135,6 +135,7 @@ spec:
- --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
- --node-local-dns-ip={{- .Values.networking.NODE_LOCAL_DNS_IP }}
- --secure-serving={{- .Values.func.SECURE_SERVING }}
- --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }}
securityContext:
runAsUser: 65534
privileged: false
Expand Down
47 changes: 47 additions & 0 deletions charts/kube-ovn/templates/ovn-CR.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -175,6 +175,37 @@ rules:
- subjectaccessreviews
verbs:
- create
- apiGroups:
- "certificates.k8s.io"
resources:
- "certificatesigningrequests"
verbs:
- "get"
- "list"
- "watch"
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests/status
- certificatesigningrequests/approval
verbs:
- update
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- create
- apiGroups:
- certificates.k8s.io
resourceNames:
- kubeovn.io/signer
resources:
- signers
verbs:
- approve
- sign
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
Expand Down Expand Up @@ -271,6 +302,22 @@ rules:
- subjectaccessreviews
verbs:
- create
- apiGroups:
- "certificates.k8s.io"
resources:
- "certificatesigningrequests"
verbs:
- "create"
- "get"
- "list"
- "watch"
- "delete"
- apiGroups:
- ""
resources:
- "secrets"
verbs:
- "get"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
Expand Down
1 change: 1 addition & 0 deletions charts/kube-ovn/templates/ovncni-ds.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -118,6 +118,7 @@ spec:
- --enable-tproxy={{ .Values.func.ENABLE_TPROXY }}
- --ovs-vsctl-concurrency={{ .Values.performance.OVS_VSCTL_CONCURRENCY }}
- --secure-serving={{- .Values.func.SECURE_SERVING }}
- --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }}
securityContext:
runAsUser: 65534
runAsGroup: 0
Expand Down
1 change: 1 addition & 0 deletions charts/kube-ovn/values.yaml
Original file line number Diff line number Diff line change
Expand Up @@ -73,6 +73,7 @@ func:
ENABLE_TPROXY: false
ENABLE_IC: false
ENABLE_NAT_GW: true
ENABLE_OVN_IPSEC: false

ipv4:
POD_CIDR: "10.16.0.0/16"
Expand Down
1 change: 1 addition & 0 deletions dist/images/cleanup.sh
Original file line number Diff line number Diff line change
Expand Up @@ -195,6 +195,7 @@ kubectl delete --ignore-not-found clusterrole system:ovn system:ovn-ovs system:k
kubectl delete --ignore-not-found clusterrolebinding ovn ovn ovn-ovs kube-ovn-cni kube-ovn-app

kubectl delete --ignore-not-found -n kube-system lease kube-ovn-controller
kubectl delete --ignore-not-found -n kube-system secret ovn-ipsec-ca

# Remove annotations in all pods of all namespaces
for ns in $(kubectl get ns -o name | awk -F/ '{print $2}'); do
Expand Down
55 changes: 55 additions & 0 deletions dist/images/install.sh
Original file line number Diff line number Diff line change
Expand Up @@ -39,6 +39,7 @@ ENABLE_TPROXY=${ENABLE_TPROXY:-false}
OVS_VSCTL_CONCURRENCY=${OVS_VSCTL_CONCURRENCY:-100}
ENABLE_COMPACT=${ENABLE_COMPACT:-false}
SECURE_SERVING=${SECURE_SERVING:-false}
ENABLE_OVN_IPSEC=${ENABLE_OVN_IPSEC:-false}

# debug
DEBUG_WRAPPER=${DEBUG_WRAPPER:-}
Expand Down Expand Up @@ -3143,6 +3144,37 @@ rules:
- subjectaccessreviews
verbs:
- create
- apiGroups:
- "certificates.k8s.io"
resources:
- "certificatesigningrequests"
verbs:
- "get"
- "list"
- "watch"
- apiGroups:
- certificates.k8s.io
resources:
- certificatesigningrequests/status
- certificatesigningrequests/approval
verbs:
- update
- apiGroups:
- ""
resources:
- secrets
verbs:
- get
- create
- apiGroups:
- certificates.k8s.io
resourceNames:
- kubeovn.io/signer
resources:
- signers
verbs:
- approve
- sign
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
Expand Down Expand Up @@ -3245,6 +3277,22 @@ rules:
- subjectaccessreviews
verbs:
- create
- apiGroups:
- "certificates.k8s.io"
resources:
- "certificatesigningrequests"
verbs:
- "create"
- "get"
- "list"
- "watch"
- "delete"
- apiGroups:
- ""
resources:
- "secrets"
verbs:
- "get"
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
Expand Down Expand Up @@ -4240,6 +4288,7 @@ spec:
- --enable-lb-svc=$ENABLE_LB_SVC
- --keep-vm-ip=$ENABLE_KEEP_VM_IP
- --node-local-dns-ip=$NODE_LOCAL_DNS_IP
- --enable-ovn-ipsec=$ENABLE_OVN_IPSEC
- --secure-serving=${SECURE_SERVING}
securityContext:
runAsUser: ${RUN_AS_USER}
Expand Down Expand Up @@ -4431,6 +4480,7 @@ spec:
- --enable-tproxy=$ENABLE_TPROXY
- --ovs-vsctl-concurrency=$OVS_VSCTL_CONCURRENCY
- --secure-serving=${SECURE_SERVING}
- --enable-ovn-ipsec=$ENABLE_OVN_IPSEC
securityContext:
runAsUser: ${RUN_AS_USER}
runAsGroup: 0
Expand Down Expand Up @@ -4484,6 +4534,8 @@ spec:
- mountPath: /etc/openvswitch
name: systemid
readOnly: true
- mountPath: /etc/ovs_ipsec_keys
name: ovs-ipsec-keys
- mountPath: /run/openvswitch
name: host-run-ovs
mountPropagation: HostToContainer
Expand Down Expand Up @@ -4544,6 +4596,9 @@ spec:
- name: systemid
hostPath:
path: /etc/origin/openvswitch
- name: ovs-ipsec-keys
hostPath:
path: /etc/origin/ovs_ipsec_keys
- name: host-run-ovs
hostPath:
path: /run/openvswitch
Expand Down
Loading

0 comments on commit 502a90e

Please sign in to comment.