Enable Windows OVS Container to run on pristine Host Environment

Support installation of unsigned ovs kernel driver from inside container For antrea-io#5249 Signed-off-by: Naman Agarwal <[email protected]>
NamanAg30 · Oct 22, 2023 · 50e74e2 · 50e74e2
1 parent dc0ca2d
commit 50e74e2
Show file tree

Hide file tree

Showing 4 changed files with 77 additions and 14 deletions.
diff --git a/build/yamls/antrea-windows-containerd-with-ovs.yml b/build/yamls/antrea-windows-containerd-with-ovs.yml
@@ -5,9 +5,30 @@ data:
     mkdir -force c:/var/log/antrea
     $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
     $mountPath =  ($mountPath.Replace('\', '/')).TrimEnd('/')
-    mkdir -force C:/var/run/secrets/kubernetes.io/serviceaccount
-    cp $mountPath/var/run/secrets/kubernetes.io/serviceaccount/ca.crt C:/var/run/secrets/kubernetes.io/serviceaccount
-    cp $mountPath/var/run/secrets/kubernetes.io/serviceaccount/token C:/var/run/secrets/kubernetes.io/serviceaccount
+    #From containerd version 1.7 onwards servcieaccount directory and the ca.cert and token files will automatically be created.
+    $serviceAccountPath = "C:\var\run\secrets\kubernetes.io\serviceaccount"
+    if (-Not $(Test-Path $serviceAccountPath)) {
+        mkdir -force $serviceAccountPath
+    }
+
+    $localTokenFile = "$serviceAccountPath/token"
+    $localCAFile="$serviceAccountPath/ca.crt"
+
+    $tokenPath = "$mountPath/var/run/secrets/kubernetes.io/serviceaccount/token"
+    $certPath = "$mountPath/var/run/secrets/kubernetes.io/serviceaccount/ca.crt"
+
+    # Check if the local token file is not present or its content is different
+    if (-Not (Test-Path $localTokenFile) -or (Get-Content -Raw $localTokenFile) -ne (Get-Content -Raw $tokenPath)) {
+        Copy-Item -Path $tokenPath -Destination $localTokenFile -Force
+    }
+
+    # Check if the local ca.crt file is not present or its content is different
+    if (-Not (Test-Path $localCAFile) -or (Get-Content -Raw $localCAFile) -ne (Get-Content -Raw $certPath)) {
+        Copy-Item -Path $certPath -Destination $localCAFile -Force
+    }
+    # mkdir -force C:/var/run/secrets/kubernetes.io/serviceaccount
+    # cp $mountPath/var/run/secrets/kubernetes.io/serviceaccount/ca.crt C:/var/run/secrets/kubernetes.io/serviceaccount
+    # cp $mountPath/var/run/secrets/kubernetes.io/serviceaccount/token C:/var/run/secrets/kubernetes.io/serviceaccount
     mkdir -force c:/opt/cni/bin/
     mkdir -force c:/etc/cni/net.d/
     cp $mountPath/k/antrea/cni/* c:/opt/cni/bin/
@@ -25,16 +46,31 @@ data:
     $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
     $mountPath = ($mountPath.Replace('\', '/')).TrimEnd('/')
     $env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/openvswitch/usr/bin;$mountPath/openvswitch/usr/sbin"
+    $OVSDriverDir="$mountPath\openvswitch\driver"
+
+    # Check if OVSExt driver is already installed
+    $driverStatus = netcfg -q ovsext
+    if ($driverStatus -like '*not installed*') {
+      # Install OVS Driver
+      netcfg -l $OVSDriverDir/ovsext.inf -c s -i OVSExt
+    }
+
+    #Configure OVS processes
     $OVS_DB_SCHEMA_PATH = "$mountPath/openvswitch/usr/share/openvswitch/vswitch.ovsschema"
     $OVS_DB_PATH = "C:\openvswitch\etc\openvswitch\conf.db"
     if ($(Test-Path $OVS_DB_SCHEMA_PATH) -and !$(Test-Path $OVS_DB_PATH)) {
+        mkdir -force C:\openvswitch\etc\openvswitch\
         ovsdb-tool create "$OVS_DB_PATH" "$OVS_DB_SCHEMA_PATH"
     }
+    $OVS_RUN_PATH = "C:\openvswitch\var\run\openvswitch"
+    if (!$(Test-Path $OVS_RUN_PATH)) {
+      mkdir -force $OVS_RUN_PATH
+    }
     ovsdb-server $OVS_DB_PATH -vfile:info --remote=punix:db.sock --log-file=/var/log/antrea/openvswitch/ovsdb-server.log --pidfile --detach
     ovs-vsctl --no-wait init
 
     # Set OVS version.
-    $OVS_VERSION=$(Get-Item $mountPath\openvswitch\driver\OVSExt.sys).VersionInfo.ProductVersion
+    $OVS_VERSION=$(Get-Item $OVSDriverDir/OVSExt.sys).VersionInfo.ProductVersion
     ovs-vsctl --no-wait set Open_vSwitch . ovs_version=$OVS_VERSION
 
     ovs-vswitchd --log-file=/var/log/antrea/openvswitch/ovs-vswitchd.log --pidfile -vfile:info --detach
@@ -73,7 +109,7 @@ kind: ConfigMap
 metadata:
   labels:
     app: antrea
-  name: antrea-agent-windows-47gftkh2c5
+  name: antrea-agent-windows-b78t745f44
   namespace: kube-system
 ---
 apiVersion: v1
@@ -354,7 +390,7 @@ spec:
         name: antrea-windows-config
       - configMap:
           defaultMode: 420
-          name: antrea-agent-windows-47gftkh2c5
+          name: antrea-agent-windows-b78t745f44
         name: antrea-agent-windows
       - hostPath:
           path: /var/log/antrea/

diff --git a/build/yamls/windows/containerd-with-ovs/conf/Run-AntreaOVS-Containerd.ps1 b/build/yamls/windows/containerd-with-ovs/conf/Run-AntreaOVS-Containerd.ps1
@@ -2,11 +2,27 @@ $ErrorActionPreference = "Stop"
 $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
 $mountPath = ($mountPath.Replace('\', '/')).TrimEnd('/')
 $env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/openvswitch/usr/bin;$mountPath/openvswitch/usr/sbin"
+$OVSDriverDir="$mountPath\openvswitch\driver"
+$WorkDir="$mountPath\openvswitch\"
+
+# Check if OVSExt driver is already installed
+$driverStatus = netcfg -q ovsext
+if ($driverStatus -like '*not installed*') {
+  # Install OVS Driver
+  netcfg -l $mountPath/openvswitch/driver/ovsext.inf -c s -i OVSExt
+}
+
+#Configure OVS processes
 $OVS_DB_SCHEMA_PATH = "$mountPath/openvswitch/usr/share/openvswitch/vswitch.ovsschema"
 $OVS_DB_PATH = "C:\openvswitch\etc\openvswitch\conf.db"
 if ($(Test-Path $OVS_DB_SCHEMA_PATH) -and !$(Test-Path $OVS_DB_PATH)) {
+    mkdir -force C:\openvswitch\etc\openvswitch\
     ovsdb-tool create "$OVS_DB_PATH" "$OVS_DB_SCHEMA_PATH"
 }
+$OVS_RUN_PATH = "C:\openvswitch\var\run\openvswitch"
+if (!$(Test-Path $OVS_RUN_PATH)) {
+  mkdir -force $OVS_RUN_PATH
+}
 ovsdb-server $OVS_DB_PATH -vfile:info --remote=punix:db.sock --log-file=/var/log/antrea/openvswitch/ovsdb-server.log --pidfile --detach
 ovs-vsctl --no-wait init
 

diff --git a/ci/jenkins/test.sh b/ci/jenkins/test.sh
@@ -152,7 +152,12 @@ export NO_PULL
 E2ETEST_PATH=${WORKDIR}/kubernetes/_output/dockerized/bin/linux/amd64/e2e.test
 
 function export_govc_env_var {
-    export GOVC_URL=$GOVC_URL
+    env_govc="${WORKDIR}/govc.env"
+    if [ -f "$env_govc" ]; then
+        source "$env_govc"
+    else
+        export GOVC_URL=$GOVC_URL
+    fi
     export GOVC_USERNAME=$GOVC_USERNAME
     export GOVC_PASSWORD=$GOVC_PASSWORD
     export GOVC_INSECURE=1
@@ -322,7 +327,11 @@ function prepare_env {
 function revert_snapshot_windows {
     WIN_NAME=$1
     echo "==== Reverting Windows VM ${WIN_NAME} ====="
-    govc snapshot.revert -vm ${WIN_NAME} win-initial
+    if [[ $WIN_NAME == *"jumper"* ]]; then
+        govc snapshot.revert -vm ${WIN_NAME} win-initial
+    else
+        govc snapshot.revert -vm ${WIN_NAME} pristine-win-initial
+    fi
     # If Windows VM fails to power on correctly in time, retry several times.
     winVMIPs=""
     for i in `seq 10`; do
@@ -516,26 +525,24 @@ function build_and_deliver_antrea_windows_and_linux_docker_images {
 function  build_and_deliver_antrea_windows_and_linux_containerd_images {
     echo "====== Cleanup Antrea Installation Before Delivering Antrea Windows and Antrea Linux containerd Images ====="
     clean_antrea
-    kubectl delete -f ${WORKDIR}/antrea-windows-containerd.yml --ignore-not-found=true || true
+    kubectl delete -f ${WORKDIR}/antrea-windows-containerd-with-ovs.yml --ignore-not-found=true || true
     kubectl delete -f ${WORKDIR}/kube-proxy-windows-containerd.yml --ignore-not-found=true || true
     kubectl delete daemonset antrea-agent -n kube-system --ignore-not-found=true || true
     kubectl delete -f ${WORKDIR}/antrea.yml --ignore-not-found=true || true
 
     prepare_env
-    ${CLEAN_STALE_IMAGES_CONTAINERD}
-    ${PRINT_CONTAINERD_STATUS}
     chmod -R g-w build/images/ovs
     chmod -R g-w build/images/base
     # Clean docker image to save disk space.
     ${CLEAN_STALE_IMAGES}
     ${PRINT_DOCKER_STATUS}
     export_govc_env_var
     # Enable verbose log for troubleshooting.
-    sed -i "s/--v=0/--v=4/g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd.yml
+    sed -i "s/--v=0/--v=4/g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd-with-ovs.yml
 
     echo "====== Updating yaml files to enable proxyAll ======"
     KUBE_API_SERVER=$(kubectl --kubeconfig=$KubeConfigFile config view -o jsonpath='{.clusters[0].cluster.server}')
-    sed -i "s|.*kubeAPIServerOverride: \"\"|    kubeAPIServerOverride: \"${KUBE_API_SERVER}\"|g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd.yml
+    sed -i "s|.*kubeAPIServerOverride: \"\"|    kubeAPIServerOverride: \"${KUBE_API_SERVER}\"|g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd-with-ovs.yml
 
     cp -f build/yamls/*.yml $WORKDIR
     set +e
@@ -1148,7 +1155,7 @@ fi
 trap clean_antrea EXIT
 if [[ ${TESTCASE} =~ "windows" ]]; then
     if [[ ${TESTCASE} =~ "containerd" ]]; then
-        WINDOWS_YAML_SUFFIX="windows-containerd"
+        WINDOWS_YAML_SUFFIX="windows-containerd-with-ovs"
         build_and_deliver_antrea_windows_and_linux_containerd_images
         if [[ ${TESTCASE} =~ "e2e" ]]; then
             run_e2e_windows

diff --git a/docs/windows.md b/docs/windows.md
@@ -218,6 +218,10 @@ sed "s|.*kubeAPIServerOverride: \"\"|    kubeAPIServerOverride: \"${KUBE_APISERV
 kubectl apply -f -
 ```
 
+> Note: The above commands also install the signed OVS driver.
+> In case of unsigned driver , the user needs to pre-install it on host 
+> before installing antrea.
+
 #### Join Windows worker Nodes
 
 ##### 1. (Optional) Install OVS (provided by Antrea or your own)