ipsec: fix chart installation failure

Signed-off-by: zhangzujian <[email protected]>

.github/workflows/build-x86-image.yaml

@@ -2820,6 +2820,7 @@ jobs:
       - name: Check kube ovn pod restarts
         if: ${{ success() || (failure() && (steps.install.conclusion == 'failure' || steps.kube-ovn-ipsec-e2e.conclusion == 'failure')) }}
         run: make check-kube-ovn-pod-restarts
+
   push:
     name: Push Images
     needs:
@@ -2828,6 +2829,7 @@ jobs:
       - cyclonus-netpol-e2e
       - kube-ovn-conformance-e2e
       - kube-ovn-ic-conformance-e2e
+      - kube-ovn-ipsec-e2e
       - multus-conformance-e2e
       - ovn-vpc-nat-gw-conformance-e2e
       - iptables-vpc-nat-gw-conformance-e2e

Makefile

@@ -475,6 +475,7 @@ kind-install-chart: kind-load-image kind-untaint-control-plane
 		--set networking.NET_STACK=$(shell echo $${NET_STACK:-ipv4} | sed 's/^dual$$/dual_stack/') \
 		--set networking.ENABLE_SSL=$(shell echo $${ENABLE_SSL:-false}) \
 		--set func.ENABLE_BIND_LOCAL_IP=$(shell echo $${ENABLE_BIND_LOCAL_IP:-true}) \
+		--set func.ENABLE_OVN_IPSEC=$(shell echo $${ENABLE_OVN_IPSEC:-false}) \
 		--set func.ENABLE_IC=$(shell kubectl get node --show-labels | grep -qw "ovn.kubernetes.io/ic-gw" && echo true || echo false)
 
 .PHONY: kind-install-chart-ssl
@@ -485,6 +486,10 @@ kind-install-chart-ssl:
 kind-upgrade-chart: kind-load-image
 	helm upgrade kubeovn ./charts/kube-ovn --wait \
 		--set global.images.kubeovn.tag=$(VERSION) \
+		--set networking.NET_STACK=$(shell echo $${NET_STACK:-ipv4} | sed 's/^dual$$/dual_stack/') \
+		--set networking.ENABLE_SSL=$(shell echo $${ENABLE_SSL:-false}) \
+		--set func.ENABLE_BIND_LOCAL_IP=$(shell echo $${ENABLE_BIND_LOCAL_IP:-true}) \
+		--set func.ENABLE_OVN_IPSEC=$(shell echo $${ENABLE_OVN_IPSEC:-false}) \
 		--set func.ENABLE_IC=$(shell kubectl get node --show-labels | grep -qw "ovn.kubernetes.io/ic-gw" && echo true || echo false)
 	kubectl -n kube-system wait pod --for=condition=ready -l app=ovs --timeout=60s
 
@@ -914,9 +919,8 @@ kind-install-kwok:
 	done
 
 .PHONY: kind-install-ovn-ipsec
-kind-install-ovn-ipsec: kind-load-image
-	kubectl config use-context kind-kube-ovn
-	@$(MAKE) ENABLE_OVN_IPSEC=true DEBUG_WRAPPER=true kind-install
+kind-install-ovn-ipsec:
+	@$(MAKE) ENABLE_OVN_IPSEC=true kind-install
 
 .PHONY: kind-reload
 kind-reload: kind-reload-ovs

charts/kube-ovn/templates/_helpers.tpl

@@ -75,3 +75,11 @@ Number of master nodes
     {{- end -}}
   {{- end -}}
 {{- end -}}
+
+{{- define "kubeovn.runAsUser" -}}
+  {{- if $.Values.func.ENABLE_OVN_IPSEC -}}
+    0
+  {{- else -}}
+    65534
+  {{- end -}}
+{{- end -}}

charts/kube-ovn/templates/central-deploy.yaml

@@ -70,7 +70,7 @@ spec:
           - bash
           - /kube-ovn/start-db.sh
           securityContext:
-            runAsUser: 65534
+            runAsUser: {{ include "kubeovn.runAsUser" . }}
             privileged: false
             capabilities:
               add:

charts/kube-ovn/templates/controller-deploy.yaml

@@ -137,7 +137,7 @@ spec:
           - --secure-serving={{- .Values.func.SECURE_SERVING }}
           - --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }}
           securityContext:
-            runAsUser: 65534
+            runAsUser: {{ include "kubeovn.runAsUser" . }}
             privileged: false
             capabilities:
               add:

charts/kube-ovn/templates/ic-controller-deploy.yaml

@@ -74,7 +74,7 @@ spec:
           - --logtostderr=false
           - --alsologtostderr=true
           securityContext:
-            runAsUser: 65534
+            runAsUser: {{ include "kubeovn.runAsUser" . }}
             privileged: false
             capabilities:
               add:

charts/kube-ovn/templates/monitor-deploy.yaml

@@ -68,7 +68,7 @@ spec:
           - --alsologtostderr=true
           - --log_file_max_size=200
           securityContext:
-            runAsUser: 65534
+            runAsUser: {{ include "kubeovn.runAsUser" . }}
             privileged: false
             capabilities:
               add:

charts/kube-ovn/templates/ovncni-ds.yaml

@@ -120,7 +120,7 @@ spec:
           - --secure-serving={{- .Values.func.SECURE_SERVING }}
           - --enable-ovn-ipsec={{- .Values.func.ENABLE_OVN_IPSEC }}
         securityContext:
-          runAsUser: 65534
+          runAsUser: {{ include "kubeovn.runAsUser" . }}
           runAsGroup: 0
           privileged: false
           capabilities:

charts/kube-ovn/templates/ovsovn-ds.yaml

@@ -90,7 +90,7 @@ spec:
           command: ["/kube-ovn/start-ovs.sh"]
           {{- end }}
           securityContext:
-            runAsUser: 65534
+            runAsUser: {{ include "kubeovn.runAsUser" . }}
             privileged: false
             capabilities:
               add:

charts/kube-ovn/templates/pinger-ds.yaml

@@ -77,7 +77,7 @@ spec:
           - --enable-metrics={{- .Values.networking.ENABLE_METRICS }}
           imagePullPolicy: {{ .Values.image.pullPolicy }}
           securityContext:
-            runAsUser: 65534
+            runAsUser: {{ include "kubeovn.runAsUser" . }}
             privileged: false
             capabilities:
               add:

dist/images/install.sh

@@ -44,7 +44,7 @@ ENABLE_OVN_IPSEC=${ENABLE_OVN_IPSEC:-false}
 # debug
 DEBUG_WRAPPER=${DEBUG_WRAPPER:-}
 RUN_AS_USER=65534 # run as nobody
-if [ -n "$DEBUG_WRAPPER" ]; then
+if [ "$ENABLE_OVN_IPSEC" = "true" -o -n "$DEBUG_WRAPPER" ]; then
   RUN_AS_USER=0
 fi
 

pkg/controller/pki.go

@@ -25,22 +25,19 @@ func (c *Controller) InitDefaultOVNIPsecCA() error {
 		return err
 	}
 
-	cmd := exec.Command("ovs-pki", "init", "--force")
-	_, err = cmd.Output()
+	output, err := exec.Command("ovs-pki", "init", "--force").CombinedOutput()
 	if err != nil {
+		klog.Errorf("ovs-pki init failed: %s", string(output))
 		return err
 	}
 
-	_, err = os.Stat(util.DefaultOVSCACertPath)
-	if err != nil {
+	if _, err = os.Stat(util.DefaultOVSCACertPath); err != nil {
 		if os.IsNotExist(err) {
 			return fmt.Errorf("CA Cert not exist: %s", util.DefaultOVSCACertPath)
 		}
 		return err
 	}
-
-	_, err = os.Stat(util.DefaultOVSCACertKeyPath)
-	if err != nil {
+	if _, err = os.Stat(util.DefaultOVSCACertKeyPath); err != nil {
 		if os.IsNotExist(err) {
 			return fmt.Errorf("CA Cert Key not exist: %s", util.DefaultOVSCACertKeyPath)
 		}
@@ -67,8 +64,7 @@ func (c *Controller) InitDefaultOVNIPsecCA() error {
 		},
 	}
 
-	_, err = c.config.KubeClient.CoreV1().Secrets("kube-system").Create(context.TODO(), secret, metav1.CreateOptions{})
-	if err != nil {
+	if _, err = c.config.KubeClient.CoreV1().Secrets("kube-system").Create(context.TODO(), secret, metav1.CreateOptions{}); err != nil {
 		return err
 	}
 

pkg/controller/signer.go

@@ -276,15 +276,15 @@ func signCSR(template *x509.Certificate, requestKey c.PublicKey, issuer *x509.Ce
 		return nil, err
 	}
 	if len(certs) != 1 {
-		return nil, errors.New("Expected a single certificate")
+		return nil, errors.New("expected a single certificate")
 	}
 	return certs[0], nil
 }
 
 func decodeCertificateRequest(pemBytes []byte) (*x509.CertificateRequest, error) {
 	block, _ := pem.Decode(pemBytes)
 	if block == nil || block.Type != "CERTIFICATE REQUEST" {
-		err := errors.New("PEM block type must be CERTIFICATE_REQUEST")
+		err := errors.New("certificate PEM block type must be CERTIFICATE_REQUEST")
 		return nil, err
 	}
 
@@ -294,7 +294,7 @@ func decodeCertificateRequest(pemBytes []byte) (*x509.CertificateRequest, error)
 func decodeCertificate(pemBytes []byte) (*x509.Certificate, error) {
 	block, _ := pem.Decode(pemBytes)
 	if block == nil || block.Type != "CERTIFICATE" {
-		err := errors.New("PEM block type must be CERTIFICATE")
+		err := errors.New("certificate PEM block type must be CERTIFICATE")
 		return nil, err
 	}
 
@@ -305,7 +305,7 @@ func decodePrivateKey(pemBytes []byte) (*rsa.PrivateKey, error) {
 	block, _ := pem.Decode(pemBytes)
 	if block == nil || block.Type != "PRIVATE KEY" {
 		fmt.Println(block.Type)
-		err := errors.New("PEM block type must be PRIVATE KEY")
+		err := errors.New("certificate PEM block type must be PRIVATE KEY")
 		return nil, err
 	}
 
@@ -316,7 +316,7 @@ func decodePrivateKey(pemBytes []byte) (*rsa.PrivateKey, error) {
 
 	rsaKey, ok := key.(*rsa.PrivateKey)
 	if !ok {
-		err := errors.New("Failed to convert private key to RSA private key")
+		err := errors.New("failed to convert private key to RSA private key")
 		return nil, err
 	}
 

pkg/daemon/ipsec.go

@@ -177,7 +177,7 @@ func (c *Controller) createCSR(csrBytes []byte) error {
 		return err
 	}
 
-	klog.Infof("node %s' ipsec init successfully ", os.Getenv("HOSTNAME"))
+	klog.Infof("node %s' ipsec init successfully", os.Getenv("HOSTNAME"))
 	return nil
 }
 
@@ -217,10 +217,9 @@ func linkCACertToIPSecDir() error {
 }
 
 func clearCACertToIPSecDir() error {
-	// clear /etc/openvswitch/keys/ipsec-cacert.pem
-	cmd := exec.Command("rm", "-f", "/etc/openvswitch/keys/ipsec-cacert.pem")
-	if err := cmd.Run(); err != nil {
-		return err
+	if err := os.Remove("/etc/ipsec.d/cacerts/ipsec-cacert.pem"); err != nil && !os.IsNotExist(err) {
+		klog.Error(err)
+		return fmt.Errorf("failed to remove ipsec-cacert.pem: %w", err)
 	}
 	return nil
 }
@@ -249,8 +248,7 @@ func clearIPSecKeysDir() error {
 }
 
 func (c *Controller) ManageIPSecKeys() error {
-	_, err := os.Stat(ipsecCertPath)
-	if os.IsNotExist(err) {
+	if _, err := os.Stat(ipsecCertPath); os.IsNotExist(err) {
 		if err := c.CreateIPSecKeys(); err != nil {
 			klog.Errorf("create ipsec keys error: %v", err)
 			return err

test/e2e/ipsec/e2e_test.go

@@ -7,7 +7,6 @@ import (
 	"strings"
 	"testing"
 
-	"github.com/onsi/ginkgo/v2"
 	corev1 "k8s.io/api/core/v1"
 	clientset "k8s.io/client-go/kubernetes"
 	"k8s.io/klog/v2"
@@ -17,6 +16,8 @@ import (
 	e2enode "k8s.io/kubernetes/test/e2e/framework/node"
 	e2epodoutput "k8s.io/kubernetes/test/e2e/framework/pod/output"
 
+	"github.com/onsi/ginkgo/v2"
+
 	"github.com/kubeovn/kube-ovn/test/e2e/framework"
 )
 
@@ -34,6 +35,31 @@ func TestE2E(t *testing.T) {
 	e2e.RunE2ETests(t)
 }
 
+func checkPodXfrmState(pod corev1.Pod, node1IP, node2IP string) {
+	ginkgo.GinkgoHelper()
+
+	ginkgo.By("Checking ip xfrm state for pod " + pod.Name + " on node " + pod.Spec.NodeName + " from " + node1IP + " to " + node2IP)
+	output, err := e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, "ip xfrm state")
+	framework.ExpectNoError(err)
+
+	var count int
+	for _, line := range strings.Split(output, "\n") {
+		if line == fmt.Sprintf("src %s dst %s", node1IP, node2IP) {
+			count++
+		}
+	}
+	framework.ExpectEqual(count, 2)
+}
+
+func checkXfrmState(pods []corev1.Pod, node1IP, node2IP string) {
+	ginkgo.GinkgoHelper()
+
+	for _, pod := range pods {
+		checkPodXfrmState(pod, node1IP, node2IP)
+		checkPodXfrmState(pod, node2IP, node1IP)
+	}
+}
+
 var _ = framework.SerialDescribe("[group:ipsec]", func() {
 	f := framework.NewDefaultFramework("ipsec")
 
@@ -52,76 +78,48 @@ var _ = framework.SerialDescribe("[group:ipsec]", func() {
 	})
 
 	framework.ConformanceIt("Should support OVN IPSec", func() {
-		ginkgo.By("Checking ip xfrm state")
-
 		ginkgo.By("Getting nodes")
 		nodeList, err := e2enode.GetReadySchedulableNodes(context.Background(), cs)
 		framework.ExpectNoError(err)
 		framework.ExpectNotEmpty(nodeList.Items)
+		framework.ExpectTrue(len(nodeList.Items) >= 2)
 
 		ginkgo.By("Getting kube-ovn-cni pods")
 		daemonSetClient := f.DaemonSetClientNS(framework.KubeOvnNamespace)
 		ds := daemonSetClient.Get("kube-ovn-cni")
-		pods := make([]corev1.Pod, 0, len(nodeList.Items))
+		podList, err := daemonSetClient.GetPods(ds)
+		framework.ExpectNoError(err)
+		framework.ExpectHaveLen(podList.Items, len(nodeList.Items))
 		nodeIPs := make([]string, 0, len(nodeList.Items))
 		for _, node := range nodeList.Items {
-			pod, err := daemonSetClient.GetPodOnNode(ds, node.Name)
-			framework.ExpectNoError(err, "failed to get kube-ovn-cni pod running on node %s", node.Name)
-			pods = append(pods, *pod)
-			nodeIPs = append(nodeIPs, node.Status.Addresses[0].Address)
+			for _, addr := range node.Status.Addresses {
+				if addr.Type == corev1.NodeInternalIP {
+					nodeIPs = append(nodeIPs, node.Status.Addresses[0].Address)
+					break
+				}
+			}
 		}
+		framework.ExpectHaveLen(nodeIPs, len(nodeList.Items))
 
-		for _, pod := range pods {
-			cmd := fmt.Sprintf("ip xfrm state | grep \"src %s dst %s\" | wc -l ", nodeIPs[0], nodeIPs[1])
-			output, err := e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, cmd)
-			framework.ExpectNoError(err)
-			output = strings.TrimSpace(output)
-			framework.ExpectEqual(output, "2")
-			cmd = fmt.Sprintf("ip xfrm state | grep \"src %s dst %s\" | wc -l ", nodeIPs[1], nodeIPs[0])
-			output, err = e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, cmd)
-			framework.ExpectNoError(err)
-			output = strings.TrimSpace(output)
-			framework.ExpectEqual(output, "2")
-		}
+		ginkgo.By("Checking ip xfrm state")
+		checkXfrmState(podList.Items, nodeIPs[0], nodeIPs[1])
 
-		ginkgo.By("Restart ds kube-ovn-cni")
+		ginkgo.By("Restarting ds kube-ovn-cni")
 		daemonSetClient.RestartSync(ds)
 
-		pods = make([]corev1.Pod, 0, len(nodeList.Items))
 		ds = daemonSetClient.Get("kube-ovn-cni")
-		for _, node := range nodeList.Items {
-			pod, err := daemonSetClient.GetPodOnNode(ds, node.Name)
-			framework.ExpectNoError(err, "failed to get kube-ovn-cni pod running on node %s", node.Name)
-			pods = append(pods, *pod)
-		}
-		for _, pod := range pods {
-			cmd := fmt.Sprintf("ip xfrm state | grep \"src %s dst %s\" | wc -l ", nodeIPs[0], nodeIPs[1])
-			output, err := e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, cmd)
-			framework.ExpectNoError(err)
-			output = strings.TrimSpace(output)
-			framework.ExpectEqual(output, "2")
-			cmd = fmt.Sprintf("ip xfrm state | grep \"src %s dst %s\" | wc -l ", nodeIPs[1], nodeIPs[0])
-			output, err = e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, cmd)
-			framework.ExpectNoError(err)
-			output = strings.TrimSpace(output)
-			framework.ExpectEqual(output, "2")
-		}
+		podList, err = daemonSetClient.GetPods(ds)
+		framework.ExpectNoError(err)
+		framework.ExpectHaveLen(podList.Items, len(nodeList.Items))
+
+		ginkgo.By("Checking ip xfrm state")
+		checkXfrmState(podList.Items, nodeIPs[0], nodeIPs[1])
 
-		ginkgo.By("Restart ds ovs-ovn ")
+		ginkgo.By("Restarting ds ovs-ovn")
 		ds = daemonSetClient.Get("ovs-ovn")
 		daemonSetClient.RestartSync(ds)
 
-		for _, pod := range pods {
-			cmd := fmt.Sprintf("ip xfrm state | grep \"src %s dst %s\" | wc -l ", nodeIPs[0], nodeIPs[1])
-			output, err := e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, cmd)
-			framework.ExpectNoError(err)
-			output = strings.TrimSpace(output)
-			framework.ExpectEqual(output, "2")
-			cmd = fmt.Sprintf("ip xfrm state | grep \"src %s dst %s\" | wc -l ", nodeIPs[1], nodeIPs[0])
-			output, err = e2epodoutput.RunHostCmd(pod.Namespace, pod.Name, cmd)
-			framework.ExpectNoError(err)
-			output = strings.TrimSpace(output)
-			framework.ExpectEqual(output, "2")
-		}
+		ginkgo.By("Checking ip xfrm state")
+		checkXfrmState(podList.Items, nodeIPs[0], nodeIPs[1])
 	})
 })