Add YAML support to run Windows OVS in container

For antrea-io#4952

Signed-off-by: Kumar Atish <[email protected]>

build/yamls/antrea-windows-ovs-containerd.yml

@@ -0,0 +1,331 @@
+apiVersion: v1
+data:
+  Install-WindowsCNI-Containerd.ps1: |
+    $ErrorActionPreference = "Stop";    
+    mkdir -force c:/var/log/antrea
+
+    $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
+    $mountPath =  ($mountPath.Replace('\', '/')).TrimEnd('/') 
+    mkdir -force C:/var/run/secrets/kubernetes.io/serviceaccount
+    cp $mountPath/var/run/secrets/kubernetes.io/serviceaccount/ca.crt C:/var/run/secrets/kubernetes.io/serviceaccount
+    cp $mountPath/var/run/secrets/kubernetes.io/serviceaccount/token C:/var/run/secrets/kubernetes.io/serviceaccount
+    mkdir -force c:/opt/cni/bin/
+    mkdir -force c:/etc/cni/net.d/
+    cp $mountPath/k/antrea/cni/* c:/opt/cni/bin/
+    cp $mountPath/etc/antrea/antrea-cni.conflist c:/etc/cni/net.d/10-antrea.conflist
+    mkdir -force c:/k/antrea/bin
+    cp $mountPath/k/antrea/bin/antctl.exe c:/k/antrea/bin/antctl.exe
+    
+  Run-AntreaAgent-Containerd.ps1: |
+    $ErrorActionPreference = "Stop"
+    $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
+    $mountPath =  ($mountPath.Replace('\', '/')).TrimEnd('/')
+    & "$mountPath/k/antrea/bin/antrea-agent.exe" --config=$mountPath/etc/antrea/antrea-agent.conf --logtostderr=false --log_dir=c:/var/log/antrea --alsologtostderr --log_file_max_size=100 --log_file_max_num=4 --v=0
+
+  Run-AntreaOVS-Containerd.ps1: |
+    $ErrorActionPreference = "Stop"
+    $OVSInstallDir = "C:\openvswitch"
+    $OVS_DB_SCHEMA_PATH = "$OVSInstallDir\usr\share\openvswitch\vswitch.ovsschema"
+    $OVS_DB_PATH = "$OVSInstallDir\etc\openvswitch\conf.db"
+    if ($(Test-Path $OVS_DB_SCHEMA_PATH) -and !$(Test-Path $OVS_DB_PATH)) {
+        Set-Location -Path "C:\openvswitch\usr\bin
+        ovsdb-tool create "$OVS_DB_PATH" "$OVS_DB_SCHEMA_PATH"
+    }
+    $mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
+    $mountPath = ($mountPath.Replace('\', '/')).TrimEnd('/')
+    [Environment]::SetEnvironmentVariable("PATH", "$([Environment]::GetEnvironmentVariable('PATH', 'Machine'));$mountPath/openvswitch/usr/sbin;$mountPath/openvswitch/usr/bin", "Machine")
+    ovsdb-server $OVS_DB_PATH -vfile:info --remote=punix:db.sock --remote=ptcp:6640 --log-file=/var/log/antrea/openvswitch/ovsdb-server.log --pidfile --detach
+    ovs-vsctl --no-wait init
+
+    # Set OVS version.
+    $OVS_VERSION=$(Get-Item $OVSInstallDir\driver\OVSExt.sys).VersionInfo.ProductVersion
+    ovs-vsctl --no-wait set Open_vSwitch . ovs_version=$OVS_VERSION
+
+    ovs-vswitchd --log-file=/var/log/antrea/openvswitch/ovs-vswitchd.log --pidfile -vfile:info --detach
+
+    $SleepInterval = 30
+    while ($true) {
+      if ( !( Get-Process ovsdb-server ) ) {
+        ovsdb-server $OVS_DB_PATH -vfile:info --remote=punix:db.sock --remote=ptcp:6640 --log-file=/var/log/antrea/openvswitch/ovsdb-server.log --pidfile --detach
+      }
+      if ( !( Get-Process ovs-vswitchd ) ) {
+        ovs-vswitchd --log-file=/var/log/antrea/openvswitch/ovs-vswitchd.log --pidfile -vfile:info --detach
+      }
+      Start-Sleep -Seconds $SleepInterval
+    }
+kind: ConfigMap
+metadata:
+  labels:
+    app: antrea
+  name: antrea-agent-windows-kht6m7hthm
+  namespace: kube-system
+---
+apiVersion: v1
+data:
+  antrea-agent.conf: |
+    # FeatureGates is a map of feature names to bools that enable or disable experimental features.
+    featureGates:
+    # Enable antrea proxy which provides ServiceLB for in-cluster services in antrea agent.
+    # It should be enabled on Windows, otherwise NetworkPolicy will not take effect on
+    # Service traffic.
+    #  AntreaProxy: true
+
+    # Enable EndpointSlice support in AntreaProxy. Don't enable this feature unless that EndpointSlice
+    # API version v1beta1 is supported and set as enabled in Kubernetes. If AntreaProxy is not enabled,
+    # this flag will not take effect.
+    #  EndpointSlice: false
+
+    # Enable NodePortLocal feature to make the Pods reachable externally through NodePort
+    #  NodePortLocal: true
+
+    # Enable flowexporter which exports polled conntrack connections as IPFIX flow records from each agent to a configured collector.
+    #  FlowExporter: false
+
+    # Name of the OpenVSwitch bridge antrea-agent will create and use.
+    # Make sure it doesn't conflict with your existing OpenVSwitch bridges.
+    #ovsBridge: br-int
+
+    # Name of the interface antrea-agent will create and use for host <--> pod communication.
+    # Make sure it doesn't conflict with your existing interfaces.
+    #hostGateway: antrea-gw0
+
+    # Encapsulation mode for communication between Pods across Nodes, supported values:
+    # - geneve (default)
+    # - vxlan
+    # - stt
+    #tunnelType: geneve
+
+    # TunnelPort is the destination port for UDP and TCP based tunnel protocols
+    # (Geneve, VXLAN, and STT). If zero, it will use the assigned IANA port for the
+    # protocol, i.e. 6081 for Geneve, 4789 for VXLAN, and 7471 for STT.
+    #tunnelPort: 0
+
+    # Default MTU to use for the host gateway interface and the network interface of each Pod.
+    # If omitted, antrea-agent will discover the MTU of the Node's primary interface and
+    # also adjust MTU to accommodate for tunnel encapsulation overhead.
+    #defaultMTU: 1450
+
+    # ClusterIP CIDR range for Services. It's required when AntreaProxy is not enabled, and should be
+    # set to the same value as the one specified by --service-cluster-ip-range for kube-apiserver. When
+    # AntreaProxy is enabled, this parameter is not needed and will be ignored if provided.
+    #serviceCIDR: 10.96.0.0/12
+
+    # The port for the antrea-agent APIServer to serve on.
+    #apiPort: 10350
+
+    # Enable metrics exposure via Prometheus. Initializes Prometheus metrics listener.
+    #enablePrometheusMetrics: true
+
+    # Provide the IPFIX collector address as a string with format <HOST>:[<PORT>][:<PROTO>].
+    # HOST can either be the DNS name, IP, or Service name of the Flow Collector. If
+    # using an IP, it can be either IPv4 or IPv6. However, IPv6 address should be
+    # wrapped with []. When the collector is running in-cluster as a Service, set
+    # <HOST> to <Service namespace>/<Service name>. For example,
+    # "flow-aggregator/flow-aggregator" can be provided to connect to the Antrea
+    # Flow Aggregator Service.
+    # If PORT is empty, we default to 4739, the standard IPFIX port.
+    # If no PROTO is given, we consider "tls" as default. We support "tls", "tcp" and
+    # "udp" protocols. "tls" is used for securing communication between flow exporter and
+    # flow aggregator.
+    #flowCollectorAddr: "flow-aggregator/flow-aggregator:4739:tls"
+
+    # Provide flow poll interval as a duration string. This determines how often the
+    # flow exporter dumps connections from the conntrack module. Flow poll interval
+    # should be greater than or equal to 1s (one second).
+    # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+    #flowPollInterval: "5s"
+
+    # Provide the active flow export timeout, which is the timeout after which a flow
+    # record is sent to the collector for active flows. Thus, for flows with a continuous
+    # stream of packets, a flow record will be exported to the collector once the elapsed
+    # time since the last export event is equal to the value of this timeout.
+    # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+    #activeFlowExportTimeout: "30s"
+
+    # Provide the idle flow export timeout, which is the timeout after which a flow
+    # record is sent to the collector for idle flows. A flow is considered idle if no
+    # packet matching this flow has been observed since the last export event.
+    # Valid time units are "ns", "us" (or "µs"), "ms", "s", "m", "h".
+    #idleFlowExportTimeout: "15s"
+
+    # Enable TLS communication from flow exporter to flow aggregator.
+    #enableTLSToFlowAggregator: true
+
+    # Determines how traffic is encapsulated. It has the following options:
+    # encap(default):    Inter-node Pod traffic is always encapsulated and Pod to external network
+    #                    traffic is SNAT'd.
+    # noEncap:           Inter-node Pod traffic is not encapsulated; Pod to external network traffic is
+    #                    SNAT'd if noSNAT is not set to true. Underlying network must be capable of
+    #                    supporting Pod traffic across IP subnets.
+    # hybrid:            noEncap if source and destination Nodes are on the same subnet, otherwise encap.
+    #
+    #trafficEncapMode: encap
+
+    # The name of the interface on Node which is used for tunneling or routing the traffic across Nodes.
+    # If there are multiple IP addresses configured on the interface, the first one is used. The IP
+    # address used for tunneling or routing traffic to remote Nodes is decided in the following order of
+    # preference (from highest to lowest):
+    # 1. transportInterface
+    # 2. transportInterfaceCIDRs
+    # 3. The Node IP
+    #transportInterface:
+
+    # The network CIDRs of the interface on Node which is used for tunneling or routing the traffic across
+    # Nodes. If there are multiple interfaces configured the same network CIDR, the first one is used. The
+    # IP address used for tunneling or routing traffic to remote Nodes is decided in the following order of
+    # preference (from highest to lowest):
+    # 1. transportInterface
+    # 2. transportInterfaceCIDRs
+    # 3. The Node IP
+    #transportInterfaceCIDRs: [<IPv4 CIDR>,<IPv6 CIDR>]
+
+    # Provide the address of Kubernetes apiserver, to override any value provided in kubeconfig or InClusterConfig.
+    # Defaults to "". It must be a host string, a host:port pair, or a URL to the base of the apiserver.
+    #kubeAPIServerOverride: ""
+
+    # Option antreaProxy contains AntreaProxy related configuration options.
+    antreaProxy:
+      # ProxyAll tells antrea-agent to proxy ClusterIP Service traffic, regardless of where they come from.
+      # Therefore, running kube-proxy is no longer required. This requires the AntreaProxy feature to be enabled.
+      # Note that this option is experimental. If kube-proxy is removed, option kubeAPIServerOverride must be used to access
+      # apiserver directly.
+      #proxyAll: false
+
+    nodePortLocal:
+    # Enable NodePortLocal, a feature used to make Pods reachable using port forwarding on the host. To
+    # enable this feature, you need to set "enable" to true, and ensure that the NodePortLocal feature
+    # gate is also enabled (which is the default).
+    #  enable: false
+    # Provide the port range used by NodePortLocal. When the NodePortLocal feature is enabled, a port
+    # from that range will be assigned whenever a Pod's container defines a specific port to be exposed
+    # (each container can define a list of ports as pod.spec.containers[].ports), and all Node traffic
+    # directed to that port will be forwarded to the Pod.
+    #  portRange: 61000-62000
+  antrea-cni.conflist: |
+    {
+        "cniVersion":"0.3.0",
+        "name": "antrea",
+        "plugins": [
+            {
+                "type": "antrea",
+                "ipam": {
+                    "type": "host-local"
+                },
+                "capabilities": {"dns": true}
+            }
+        ]
+    }
+kind: ConfigMap
+metadata:
+  labels:
+    app: antrea
+  name: antrea-windows-config-8kfkb8t957
+  namespace: kube-system
+---
+apiVersion: apps/v1
+kind: DaemonSet
+metadata:
+  labels:
+    app: antrea
+    component: antrea-agent
+  name: antrea-agent-windows
+  namespace: kube-system
+spec:
+  selector:
+    matchLabels:
+      app: antrea
+      component: antrea-agent
+  template:
+    metadata:
+      annotations:
+        "microsoft.com/hostprocess-inherit-user": "true"
+      labels:
+        app: antrea
+        component: antrea-agent
+    spec:
+      securityContext:
+        windowsOptions:
+          runAsUserName: "NT AUTHORITY\\SYSTEM"
+          hostProcess: true
+      containers:
+        - name: antrea-agent
+          image: antrea/antrea-windows:latest
+          imagePullPolicy: IfNotPresent
+          args:
+            - -file
+            - $env:CONTAINER_SANDBOX_MOUNT_POINT/var/lib/antrea-windows/Run-AntreaAgent-Containerd.ps1
+          command:
+          - powershell
+          env:
+          - name: POD_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.name
+          - name: POD_NAMESPACE
+            valueFrom:
+              fieldRef:
+                fieldPath: metadata.namespace
+          - name: NODE_NAME
+            valueFrom:
+              fieldRef:
+                fieldPath: spec.nodeName
+          volumeMounts:
+          - mountPath: /etc/antrea
+            name: antrea-windows-config
+          - mountPath: /var/lib/antrea-windows
+            name: antrea-agent-windows
+          - mountPath: /var/log/antrea/
+            name: var-log-antrea
+        - name: antrea-ovs
+          image: antrea/antrea-windows:latest
+          imagePullPolicy: IfNotPresent
+          args:
+            - -file
+            - $env:CONTAINER_SANDBOX_MOUNT_POINT/var/lib/antrea-windows/Run-AntreaOVS-Containerd.ps1
+          command:
+          - powershell
+          volumeMounts:
+          - mountPath: /var/lib/antrea-windows
+            name: antrea-agent-windows
+          - mountPath: /var/log/openvswitch
+            name: var-log-antrea
+            subPath: openvswitch
+      hostNetwork: true
+      initContainers:
+      - args:
+        - -File
+        - $env:CONTAINER_SANDBOX_MOUNT_POINT/var/lib/antrea-windows/Install-WindowsCNI-Containerd.ps1
+        command:
+        - powershell
+        image: antrea/antrea-windows:latest
+        imagePullPolicy: IfNotPresent
+        name: install-cni
+        volumeMounts:
+        - mountPath: /etc/antrea
+          name: antrea-windows-config
+          readOnly: true
+        - mountPath: /var/lib/antrea-windows
+          name: antrea-agent-windows
+      nodeSelector:
+        kubernetes.io/os: windows
+      priorityClassName: system-node-critical
+      serviceAccountName: antrea-agent
+      tolerations:
+      - key: CriticalAddonsOnly
+        operator: Exists
+      - effect: NoSchedule
+        operator: Exists
+      volumes:
+      - configMap:
+          name: antrea-windows-config-8kfkb8t957
+        name: antrea-windows-config
+      - configMap:
+          defaultMode: 420
+          name: antrea-agent-windows-kht6m7hthm
+        name: antrea-agent-windows
+      - hostPath:
+          path: /var/log/antrea/
+          type: DirectoryOrCreate
+        name: var-log-antrea
+  updateStrategy:
+    type: RollingUpdate

build/yamls/windows/ovs-containerd/conf/Install-WindowsCNI-Containerd.ps1

@@ -0,0 +1,13 @@
+$ErrorActionPreference = "Stop";
+mkdir -force c:/var/log/antrea
+$mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
+$mountPath =  ($mountPath.Replace('\', '/')).TrimEnd('/')
+mkdir -force C:/var/run/secrets/kubernetes.io/serviceaccount
+cp $mountPath/var/run/secrets/kubernetes.io/serviceaccount/ca.crt C:/var/run/secrets/kubernetes.io/serviceaccount
+cp $mountPath/var/run/secrets/kubernetes.io/serviceaccount/token C:/var/run/secrets/kubernetes.io/serviceaccount
+mkdir -force c:/opt/cni/bin/
+mkdir -force c:/etc/cni/net.d/
+cp $mountPath/k/antrea/cni/* c:/opt/cni/bin/
+cp $mountPath/etc/antrea/antrea-cni.conflist c:/etc/cni/net.d/10-antrea.conflist
+mkdir -force c:/k/antrea/bin
+cp $mountPath/k/antrea/bin/antctl.exe c:/k/antrea/bin/antctl.exe

build/yamls/windows/ovs-containerd/conf/Run-AntreaAgent-Containerd.ps1

@@ -0,0 +1,4 @@
+$ErrorActionPreference = "Stop"
+$mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
+$mountPath =  ($mountPath.Replace('\', '/')).TrimEnd('/')
+& "$mountPath/k/antrea/bin/antrea-agent.exe" --config=$mountPath/etc/antrea/antrea-agent.conf --logtostderr=false --log_dir=c:/var/log/antrea --alsologtostderr --log_file_max_size=100 --log_file_max_num=4 --v=0

build/yamls/windows/ovs-containerd/conf/Run-AntreaOVS-Containerd.ps1

@@ -0,0 +1,30 @@
+$ErrorActionPreference = "Stop"
+$OVSInstallDir = "C:\openvswitch"
+$OVS_DB_SCHEMA_PATH = "$OVSInstallDir\usr\share\openvswitch\vswitch.ovsschema"
+$OVS_DB_PATH = "$OVSInstallDir\etc\openvswitch\conf.db"
+if ($(Test-Path $OVS_DB_SCHEMA_PATH) -and !$(Test-Path $OVS_DB_PATH)) {
+    Set-Location -Path "C:\openvswitch\usr\bin
+    ovsdb-tool create "$OVS_DB_PATH" "$OVS_DB_SCHEMA_PATH"
+}
+$mountPath = $env:CONTAINER_SANDBOX_MOUNT_POINT
+$mountPath = ($mountPath.Replace('\', '/')).TrimEnd('/')
+[Environment]::SetEnvironmentVariable("PATH", "$([Environment]::GetEnvironmentVariable('PATH', 'Machine'));$mountPath/openvswitch/usr/sbin;$mountPath/openvswitch/usr/bin", "Machine")
+ovsdb-server $OVS_DB_PATH -vfile:info --remote=punix:db.sock --remote=ptcp:6640 --log-file=/var/log/antrea/openvswitch/ovsdb-server.log --pidfile --detach
+ovs-vsctl --no-wait init
+
+# Set OVS version.
+$OVS_VERSION=$(Get-Item $OVSInstallDir\driver\OVSExt.sys).VersionInfo.ProductVersion
+ovs-vsctl --no-wait set Open_vSwitch . ovs_version=$OVS_VERSION
+
+ovs-vswitchd --log-file=/var/log/antrea/openvswitch/ovs-vswitchd.log --pidfile -vfile:info --detach
+
+$SleepInterval = 30
+while ($true) {
+    if ( !( Get-Process ovsdb-server ) ) {
+        ovsdb-server $OVS_DB_PATH -vfile:info --remote=punix:db.sock --remote=ptcp:6640 --log-file=/var/log/antrea/openvswitch/ovsdb-server.log --pidfile --detach
+    }
+    if ( !( Get-Process ovs-vswitchd ) ) {
+        ovs-vswitchd --log-file=/var/log/antrea/openvswitch/ovs-vswitchd.log --pidfile -vfile:info --detach
+    }
+    Start-Sleep -Seconds $SleepInterval
+}

build/yamls/windows/ovs-containerd/kustomization.yml

@@ -0,0 +1,14 @@
+apiVersion: kustomize.config.k8s.io/v1beta1
+kind: Kustomization
+resources:
+- ovs-containerd.yml
+- ../base
+namespace: kube-system
+commonLabels:
+  app: antrea
+configMapGenerator:
+- files:
+  - conf/Run-AntreaAgent-Containerd.ps1
+  - conf/Install-WindowsCNI-Containerd.ps1
+  - conf/Run-AntreaOVS-Containerd.ps1
+  name: antrea-agent-windows