Skip to content

Commit

Permalink
Add support to run Windows OVS in container
Browse files Browse the repository at this point in the history
For antrea-io#4952

Signed-off-by: Shuyang Xin <[email protected]>
Signed-off-by: Naman Agarwal <[email protected]>
Signed-off-by: Kumar Atish <[email protected]>
  • Loading branch information
Atish-iaf committed Jul 19, 2023
1 parent c0b9027 commit a78af8e
Show file tree
Hide file tree
Showing 18 changed files with 671 additions and 45 deletions.
1 change: 1 addition & 0 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -383,6 +383,7 @@ manifest:
$(CURDIR)/hack/generate-standard-manifests.sh --mode dev --out build/yamls
$(CURDIR)/hack/generate-manifest-windows.sh --mode dev > build/yamls/antrea-windows.yml
$(CURDIR)/hack/generate-manifest-windows.sh --mode dev --containerd > build/yamls/antrea-windows-containerd.yml
$(CURDIR)/hack/generate-manifest-windows.sh --mode dev --containerd --ovs-run-mode pod > build/yamls/antrea-windows-ovs-containerd.yml
$(CURDIR)/hack/generate-manifest-flow-aggregator.sh --mode dev > build/yamls/flow-aggregator.yml

.PHONY: manifest-scale
Expand Down
361 changes: 361 additions & 0 deletions build/yamls/antrea-windows-ovs-containerd.yml

Large diffs are not rendered by default.

6 changes: 6 additions & 0 deletions build/yamls/windows/base-containerd/kustomization.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,6 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
configMapGenerator:
- files:
- conf/Install-WindowsCNI-Containerd.ps1
name: antrea-agent-windows
3 changes: 2 additions & 1 deletion build/yamls/windows/containerd/kustomization.yml
Original file line number Diff line number Diff line change
Expand Up @@ -3,11 +3,12 @@ kind: Kustomization
resources:
- agent-containerd.yml
- ../base
- ../base-containerd
namespace: kube-system
commonLabels:
app: antrea
configMapGenerator:
- files:
- conf/Run-AntreaAgent-Containerd.ps1
- conf/Install-WindowsCNI-Containerd.ps1
name: antrea-agent-windows
behavior: merge
Original file line number Diff line number Diff line change
@@ -0,0 +1,12 @@
$networkName = "antrea-hnsnetwork"
$networks = Get-HnsNetwork
$networkExists = $false
foreach ($network in $networks) {
if ($network.Name -eq $networkName) {
$networkExists = $true
break
}
}
if ($networkExists) {
Enable-VMSwitchExtension -Name "Open vSwitch Extension" -VMSwitchName "antrea-hnsnetwork"
}
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
$ErrorActionPreference = "Stop"
$mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
$mountPath = ($mountPath.Replace('\', '/')).TrimEnd('/')
$env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/openvswitch/usr/bin;$mountPath/openvswitch/usr/sbin"
& "$mountPath/k/antrea/bin/antrea-agent.exe" --config=$mountPath/etc/antrea/antrea-agent.conf --logtostderr=false --log_dir=c:/var/log/antrea --alsologtostderr --log_file_max_size=100 --log_file_max_num=4 --v=0
Original file line number Diff line number Diff line change
@@ -0,0 +1,31 @@
$ErrorActionPreference = "Stop"
$mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
$mountPath = ($mountPath.Replace('\', '/')).TrimEnd('/')
$env:PATH = $env:PATH + ";$mountPath/Windows/System32;$mountPath/openvswitch/usr/bin;$mountPath/openvswitch/usr/sbin"
$OVS_DB_SCHEMA_PATH = "$mountPath/openvswitch/usr/share/openvswitch/vswitch.ovsschema"
$OVS_DB_PATH = "C:\openvswitch\etc\openvswitch\conf.db"
if ($(Test-Path $OVS_DB_SCHEMA_PATH) -and !$(Test-Path $OVS_DB_PATH)) {
ovsdb-tool create "$OVS_DB_PATH" "$OVS_DB_SCHEMA_PATH"
}
ovsdb-server $OVS_DB_PATH -vfile:info --remote=punix:db.sock --log-file=/var/log/antrea/openvswitch/ovsdb-server.log --pidfile --detach
ovs-vsctl --no-wait init

# Set OVS version.
$OVS_VERSION=$(Get-Item $mountPath\openvswitch\driver\OVSExt.sys).VersionInfo.ProductVersion
ovs-vsctl --no-wait set Open_vSwitch . ovs_version=$OVS_VERSION

ovs-vswitchd --log-file=/var/log/antrea/openvswitch/ovs-vswitchd.log --pidfile -vfile:info --detach

$SleepInterval = 30
Write-Host "Started the loop that checks OVS status every $SleepInterval seconds"
while ($true) {
if ( !( Get-Process ovsdb-server ) ) {
Write-Host "ovsdb-server is not running, starting it again..."
ovsdb-server $OVS_DB_PATH -vfile:info --remote=punix:db.sock --log-file=/var/log/antrea/openvswitch/ovsdb-server.log --pidfile --detach
}
if ( !( Get-Process ovs-vswitchd ) ) {
Write-Host "ovs-vswitchd is not running, starting it again..."
ovs-vswitchd --log-file=/var/log/antrea/openvswitch/ovs-vswitchd.log --pidfile -vfile:info --detach
}
Start-Sleep -Seconds $SleepInterval
}
16 changes: 16 additions & 0 deletions build/yamls/windows/ovs-containerd/kustomization.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
apiVersion: kustomize.config.k8s.io/v1beta1
kind: Kustomization
resources:
- ovs-containerd.yml
- ../base
- ../base-containerd
namespace: kube-system
commonLabels:
app: antrea
configMapGenerator:
- files:
- conf/Run-AntreaOVS-Containerd.ps1
- conf/Run-AntreaAgent-OVS-Containerd.ps1
- conf/PostStart-AntreaAgent-Containerd.ps1
name: antrea-agent-windows
behavior: merge
110 changes: 110 additions & 0 deletions build/yamls/windows/ovs-containerd/ovs-containerd.yml
Original file line number Diff line number Diff line change
@@ -0,0 +1,110 @@
apiVersion: apps/v1
kind: DaemonSet
metadata:
labels:
component: antrea-agent
name: antrea-agent-windows
spec:
selector:
matchLabels:
component: antrea-agent
template:
metadata:
annotations:
"microsoft.com/hostprocess-inherit-user": "true"
labels:
component: antrea-agent
spec:
securityContext:
windowsOptions:
runAsUserName: "NT AUTHORITY\\SYSTEM"
hostProcess: true
containers:
- name: antrea-agent
image: katish9/antrea-windows:latest
imagePullPolicy: IfNotPresent
lifecycle:
preStop:
exec:
command: ["powershell", "Disable-VMSwitchExtension -Name 'Open vSwitch Extension' -VMSwitchName antrea-hnsnetwork"]
postStart:
exec:
command: ["powershell", "-file", "$env:CONTAINER_SANDBOX_MOUNT_POINT/var/lib/antrea-windows/PostStart-AntreaAgent-Containerd.ps1"]
args:
- -file
- $env:CONTAINER_SANDBOX_MOUNT_POINT/var/lib/antrea-windows/Run-AntreaAgent-OVS-Containerd.ps1
command:
- powershell
env:
- name: POD_NAME
valueFrom:
fieldRef:
fieldPath: metadata.name
- name: POD_NAMESPACE
valueFrom:
fieldRef:
fieldPath: metadata.namespace
- name: NODE_NAME
valueFrom:
fieldRef:
fieldPath: spec.nodeName
volumeMounts:
- mountPath: /etc/antrea
name: antrea-windows-config
- mountPath: /var/lib/antrea-windows
name: antrea-agent-windows
- mountPath: /var/log/antrea/
name: var-log-antrea
- name: antrea-ovs
image: antrea/windows-ovs:1809-3.0.5
imagePullPolicy: IfNotPresent
args:
- -file
- $env:CONTAINER_SANDBOX_MOUNT_POINT/var/lib/antrea-windows/Run-AntreaOVS-Containerd.ps1
command:
- powershell
volumeMounts:
- mountPath: /var/lib/antrea-windows
name: antrea-agent-windows
- mountPath: /var/log/openvswitch
name: var-log-antrea
subPath: openvswitch
hostNetwork: true
initContainers:
- args:
- -file
- $env:CONTAINER_SANDBOX_MOUNT_POINT/var/lib/antrea-windows/Install-WindowsCNI-Containerd.ps1
command:
- powershell
image: antrea-windows
imagePullPolicy: IfNotPresent
name: install-cni
volumeMounts:
- mountPath: /etc/antrea
name: antrea-windows-config
readOnly: true
- mountPath: /var/lib/antrea-windows
name: antrea-agent-windows
nodeSelector:
kubernetes.io/os: windows
priorityClassName: system-node-critical
serviceAccountName: antrea-agent
tolerations:
- key: CriticalAddonsOnly
operator: Exists
- effect: NoSchedule
operator: Exists
volumes:
- configMap:
name: antrea-windows-config
name: antrea-windows-config
- configMap:
defaultMode: 420
name: antrea-agent-windows
name: antrea-agent-windows
- hostPath:
path: /var/log/antrea/
type: DirectoryOrCreate
name: var-log-antrea
updateStrategy:
type: RollingUpdate
18 changes: 10 additions & 8 deletions ci/jenkins/test.sh
Original file line number Diff line number Diff line change
Expand Up @@ -321,8 +321,9 @@ function prepare_env {

function revert_snapshot_windows {
WIN_NAME=$1
echo "==== Reverting Windows VM ${WIN_NAME} ====="
govc snapshot.revert -vm ${WIN_NAME} win-initial
SNAPSHOT_NAME=$2
echo "==== Reverting Windows VM ${WIN_NAME} to ${SNAPSHOT_NAME} ====="
govc snapshot.revert -vm ${WIN_NAME} ${SNAPSHOT_NAME}
# If Windows VM fails to power on correctly in time, retry several times.
winVMIPs=""
for i in `seq 10`; do
Expand Down Expand Up @@ -411,7 +412,7 @@ function deliver_antrea_windows {
rm -f antrea-windows.tar.gz
sed -i 's/if (!(Test-Path $AntreaAgentConfigPath))/if ($true)/' hack/windows/Helper.psm1
kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 !~ role && $1 ~ /win/ {print $1}' | while read WORKER_NAME; do
revert_snapshot_windows ${WORKER_NAME}
revert_snapshot_windows ${WORKER_NAME} win-initial

# Use a script to run antrea agent in windows Network Policy cases
if [ "$TESTCASE" == "windows-networkpolicy-process" ]; then
Expand Down Expand Up @@ -474,6 +475,7 @@ function deliver_antrea_windows_containerd {
echo "====== Cleanup Antrea Installation Before Delivering Antrea Windows Containerd ======"
clean_antrea
kubectl delete -f ${WORKDIR}/antrea-windows-containerd.yml --ignore-not-found=true || true
kubectl delete -f ${WORKDIR}/antrea-windows-ovs-containerd.yml --ignore-not-found=true || true
kubectl delete -f ${WORKDIR}/kube-proxy-windows-containerd.yml --ignore-not-found=true || true
kubectl delete daemonset antrea-agent -n kube-system --ignore-not-found=true || true
kubectl delete -f ${WORKDIR}/antrea.yml --ignore-not-found=true || true
Expand All @@ -496,7 +498,7 @@ function deliver_antrea_windows_containerd {

echo "====== Updating yaml files to enable proxyAll ======"
KUBE_API_SERVER=$(kubectl --kubeconfig=$KubeConfigFile config view -o jsonpath='{.clusters[0].cluster.server}')
sed -i "s|.*kubeAPIServerOverride: \"\"| kubeAPIServerOverride: \"${KUBE_API_SERVER}\"|g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd.yml
sed -i "s|.*kubeAPIServerOverride: \"\"| kubeAPIServerOverride: \"${KUBE_API_SERVER}\"|g" build/yamls/antrea.yml build/yamls/antrea-windows-containerd.yml build/yamls/antrea-windows-ovs-containerd.yml

cp -f build/yamls/*.yml $WORKDIR
docker save -o antrea-ubuntu.tar antrea/antrea-ubuntu:latest
Expand Down Expand Up @@ -532,8 +534,7 @@ function deliver_antrea_windows_containerd {
done

echo "===== Build Antrea Windows on Windows Jumper Node ====="
echo "==== Reverting Windows VM ${WIN_IMAGE_NODE} ====="
revert_snapshot_windows ${WIN_IMAGE_NODE}
revert_snapshot_windows ${WIN_IMAGE_NODE} win-initial
rm -f antrea-windows.tar.gz
# Compress antrea repo and copy it to a Windows node
mkdir -p jenkins
Expand All @@ -550,7 +551,7 @@ function deliver_antrea_windows_containerd {
echo "===== Deliver Antrea Windows to Windows worker nodes and pull necessary images on Windows worker nodes ====="
sed -i 's/if (!(Test-Path $AntreaAgentConfigPath))/if ($true)/' hack/windows/Helper.psm1
kubectl get nodes -o wide --no-headers=true | awk -v role="$CONTROL_PLANE_NODE_ROLE" '$3 !~ role && $1 ~ /win/ {print $1}' | while read WORKER_NAME; do
revert_snapshot_windows ${WORKER_NAME}
revert_snapshot_windows ${WORKER_NAME} win-initial-userspaceovs
# Some tests need us.gcr.io/k8s-artifacts-prod/e2e-test-images/agnhost:2.13 image but it is not for windows/amd64 10.0.17763
# Use e2eteam/agnhost:2.13 instead
harbor_images=("sigwindowstools-kube-proxy:v1.18.0" "agnhost:2.13" "agnhost:2.13" "agnhost:2.29" "e2eteam-jessie-dnsutils:1.0" "e2eteam-pause:3.2")
Expand Down Expand Up @@ -1085,7 +1086,8 @@ fi
trap clean_antrea EXIT
if [[ ${TESTCASE} =~ "windows" ]]; then
if [[ ${TESTCASE} =~ "containerd" ]]; then
WINDOWS_YAML_SUFFIX="windows-containerd"
# WINDOWS_YAML_SUFFIX="windows-containerd"
WINDOWS_YAML_SUFFIX="windows-ovs-containerd"
deliver_antrea_windows_containerd
if [[ ${TESTCASE} =~ "e2e" ]]; then
run_e2e_windows
Expand Down
28 changes: 25 additions & 3 deletions docs/windows.md
Original file line number Diff line number Diff line change
Expand Up @@ -47,9 +47,10 @@ The following components should be configured and run on the Windows Node.

antrea-agent and kube-proxy run as processes on host and are managed by
management Pods. It is recommended to run OVS daemons as Windows services.
If you don't want to run antrea-agent and kube-proxy from the management Pods
Antrea also provides scripts which help install and run these two components
directly without Pod, please see [Manually run kube-proxy and antrea-agent on Windows worker Nodes](#Manually-run-kube-proxy-and-antrea-agent-on-Windows-worker-Nodes)
We also support running OVS processes inside a container. If you don't want to
run antrea-agent and kube-proxy from the management Pods Antrea also provides
scripts which help to install and run these two components directly without Pod.
Please see [Manually run kube-proxy and antrea-agent on Windows worker Nodes](#Manually-run-kube-proxy-and-antrea-agent-on-Windows-worker-Nodes)
section for details.

### Antrea Windows demo
Expand Down Expand Up @@ -205,6 +206,13 @@ sed "s|.*kubeAPIServerOverride: \"\"| kubeAPIServerOverride: \"${KUBE_APISERV
kubectl apply -f -
```

Since Antrea 1.13, you can deploy both antrea-agent and antrea-ovs Windows DaemonSets
with containerd runtime by applying file `antrea-windows-ovs-containerd.yml`.

```bash
kubectl apply -f https://github.com/antrea-io/antrea/releases/download/<TAG>/antrea-windows-ovs-containerd.yml
```

#### Join Windows worker Nodes

##### 1. (Optional) Install OVS (provided by Antrea or your own)
Expand Down Expand Up @@ -243,6 +251,13 @@ get-service ovsdb-server
get-service ovs-vswitchd
```

If you want to containerize OVS for containerd runtime, OVS userspace processes are
not run on the host and hence you can set the `InstallUserspace` parameter to false.

```powershell
.\Install-OVS.ps1 -InstallUserspace $false
```

##### 2. Disable Windows Firewall

```powershell
Expand Down Expand Up @@ -525,6 +540,13 @@ Extension is disabled by default. In this case, the stale OVS bridge and ports
should be removed. A help script [Clean-AntreaNetwork.ps1](https://raw.githubusercontent.com/antrea-io/antrea/main/hack/windows/Clean-AntreaNetwork.ps1)
can be used to clean the OVS bridge.

```powershell
# If OVS userspace processes were running as a Service on Windows host
./Clean-AntreaNetwork.ps1 -OVSRunMode "Service"
# If OVS userspace processes were running inside a container in antrea-agent Pod
./Clean-AntreaNetwork.ps1 -OVSRunMode "Pod"
```

2. Hyper-V feature cannot be installed on Windows Node due to the processor not
having the required virtualization capabilities.

Expand Down
20 changes: 16 additions & 4 deletions hack/generate-manifest-windows.sh
Original file line number Diff line number Diff line change
Expand Up @@ -22,9 +22,12 @@ function echoerr {

_usage="Usage: $0 [--mode (dev|release)] [--keep] [--help|-h]
Generate a YAML manifest to run Antrea on Windows Nodes, using Kustomize, and print it to stdout.
--mode (dev|release) Choose the configuration variant that you need (default is 'dev')
--keep Debug flag which will preserve the generated kustomization.yml
--help, -h Print this message and exit
--mode (dev|release) Choose the configuration variant that you need (default is 'dev')
--keep Debug flag which will preserve the generated kustomization.yml
--help, -h Print this message and exit
--containerd Support for containerd runtime.
--ovs-run-mode (pod|service) Run Windows OVS processes inside a container in antrea-agent pod or
as a service on Windows host with containerd runtime.
In 'release' mode, environment variables IMG_NAME and IMG_TAG must be set.
Expand All @@ -44,6 +47,7 @@ function print_help {
RUNTIME=""
MODE="dev"
KEEP=false
OVSRunMode="service"

while [[ $# -gt 0 ]]
do
Expand All @@ -62,6 +66,10 @@ case $key in
RUNTIME="containerd"
shift
;;
--ovs-run-mode)
OVSRunMode="$2"
shift 2
;;
-h|--help)
print_usage
exit 0
Expand Down Expand Up @@ -111,7 +119,11 @@ pushd $TMP_DIR > /dev/null

BASE=../../default
if [ "$RUNTIME" == "containerd" ]; then
BASE=../../containerd
if [ "$OVSRunMode" == "pod" ]; then
BASE=../../ovs-containerd
elif [ "$OVSRunMode" == "service" ]; then
BASE=../../containerd
fi
fi

mkdir $MODE && cd $MODE
Expand Down
1 change: 1 addition & 0 deletions hack/release/prepare-assets.sh
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,7 @@ export IMG_NAME=projects.registry.vmware.com/antrea/antrea-ubuntu
export IMG_NAME=projects.registry.vmware.com/antrea/antrea-windows
./hack/generate-manifest-windows.sh --mode release > "$OUTPUT_DIR"/antrea-windows.yml
./hack/generate-manifest-windows.sh --mode release --containerd > "$OUTPUT_DIR"/antrea-windows-containerd.yml
./hack/generate-manifest-windows.sh --mode release --containerd --ovs-run-mode pod > "$OUTPUT_DIR"/antrea-windows-ovs-containerd.yml

export IMG_NAME=projects.registry.vmware.com/antrea/flow-aggregator
./hack/generate-manifest-flow-aggregator.sh --mode release > "$OUTPUT_DIR"/flow-aggregator.yml
Expand Down
Loading

0 comments on commit a78af8e

Please sign in to comment.