[WIP] feat: support windows host-process deployment

Makefile

@@ -98,11 +98,16 @@ e2e-test:
 
 .PHONY: e2e-bootstrap
 e2e-bootstrap: install-helm
+ifdef WINDOWS_USE_HOST_PROCESS_CONTAINERS
+	(docker pull $(IMAGE_TAG) && docker pull $(CSI_IMAGEIMAGE_TAG_TAG)-windows-hp) || make container-all push-manifest
+else
 	docker pull $(IMAGE_TAG) || make container-all push-manifest
+endif
 ifdef TEST_WINDOWS
 	helm upgrade csi-driver-smb charts/$(VERSION)/csi-driver-smb --namespace kube-system --wait --timeout=15m -v=5 --debug --install \
 		${E2E_HELM_OPTIONS} \
 		--set windows.enabled=true \
+		--set windows.useHostProcessContainers=${WINDOWS_USE_HOST_PROCESS_CONTAINERS} \
 		--set linux.enabled=false \
 		--set controller.replicas=1 \
 		--set controller.logLevel=6 \
@@ -156,8 +161,24 @@ container-linux-armv7:
 container-windows:
 	docker buildx build --pull --output=type=$(OUTPUT_TYPE) --platform="windows/$(ARCH)" \
 		 -t $(IMAGE_TAG)-windows-$(OSVERSION)-$(ARCH) --build-arg OSVERSION=$(OSVERSION) \
-		--provenance=false --sbom=false \
-		 --build-arg ARCH=$(ARCH) -f ./cmd/smbplugin/Dockerfile.Windows .
+		 --provenance=false --sbom=false \
+		 --build-arg ARCH=$(ARCH) \
+		 --build-arg ADDON_IMAGE=servercore:$(OSVERSION) \
+		 --build-arg BASE_IMAGE=nanoserver:$(OSVERSION) \
+	         -f ./cmd/smbplugin/Dockerfile.Windows .
+
+# workaround: only build hostprocess image once
+ifdef WINDOWS_USE_HOST_PROCESS_CONTAINERS
+ifeq ($(OSVERSION),ltsc2022)
+	$(MAKE) container-windows-hostprocess
+endif
+endif
+
+# Set --provenance=false to not generate the provenance (which is what causes the multi-platform index to be generated, even for a single platform).
+.PHONY: container-windows-hostprocess
+container-windows-hostprocess:
+	docker buildx build --pull --output=type=$(OUTPUT_TYPE) --platform="windows/$(ARCH)" --provenance=false --sbom=false \
+		-t $(IMAGE_TAG)-windows-hp -f ./cmd/smbplugin/Dockerfile.Windows.hostprocess .
 
 .PHONY: container-all
 container-all: smb-windows

charts/README.md

@@ -114,6 +114,7 @@ The following table lists the configurable parameters of the latest SMB CSI Driv
 | `linux.resources.smb.requests.cpu`                      | smb-csi-driver cpu requests limits                                                                         | `10m`                                                   |
 | `linux.resources.smb.requests.memory`                   | smb-csi-driver memory requests limits                                                                      | `20Mi`                                                  |
 | `windows.enabled`                                       | whether enable windows feature                                                                             | `false`                                                 |
+| `windows.useHostProcessContainers`                    | whether deploy driver daemonset with host process containers on windows | `false`                                                             |
 | `windows.dsName`                                        | name of driver daemonset on windows                                                                        | `csi-smb-node-win`                                      |
 | `windows.removeSMBMappingDuringUnmount`                 | remove SMBMapping during unmount on Windows node windows                                                   | `true`                                                  |
 | `windows.resources.livenessProbe.limits.memory`         | liveness-probe memory limits                                                                               | `200Mi`                                                 |

charts/latest/csi-driver-smb/templates/csi-smb-node-windows-hostprocess.yaml

@@ -0,0 +1,109 @@
+{{- if and .Values.windows.enabled .Values.windows.useHostProcessContainers }}
+kind: DaemonSet
+apiVersion: apps/v1
+metadata:
+  name: {{ .Values.windows.dsName }}
+  namespace: {{ .Release.Namespace }}
+{{ include "smb.labels" . | indent 2 }}
+spec:
+  updateStrategy:
+    rollingUpdate:
+      maxUnavailable: {{ .Values.node.maxUnavailable }}
+    type: RollingUpdate
+  selector:
+    matchLabels:
+      app: {{ .Values.windows.dsName }}
+  template:
+    metadata:
+{{ include "smb.labels" . | indent 6 }}
+        app: {{ .Values.windows.dsName }}
+    spec:
+{{- with .Values.windows.tolerations }}
+      tolerations:
+{{ toYaml . | indent 8 }}
+{{- end }}
+      nodeSelector:
+        kubernetes.io/os: windows
+{{- with .Values.node.nodeSelector }}
+{{ toYaml . | indent 8 }}
+{{- end }}
+{{- with .Values.node.affinity }}
+      affinity:
+{{ toYaml . | indent 8 }}
+{{- end }}
+      priorityClassName: {{ .Values.priorityClassName | quote }}
+      securityContext:
+        seccompProfile:
+          type: RuntimeDefault
+        windowsOptions:
+          hostProcess: true
+          runAsUserName: "NT AUTHORITY\\SYSTEM"
+      hostNetwork: true
+      serviceAccountName: {{ .Values.serviceAccount.node }}
+      {{- include "smb.pullSecrets" . | indent 6 }}
+      initContainers:
+        - name: init
+{{- if hasPrefix "/" .Values.image.smb.repository }}
+          image: "{{ .Values.image.baseRepo }}{{ .Values.image.smb.repository }}:{{ .Values.image.smb.tag }}-windows-hp"
+{{- else }}
+          image: "{{ .Values.image.smb.repository }}:{{ .Values.image.smb.tag }}-windows-hp"
+{{- end }}
+          imagePullPolicy: {{ .Values.image.pullPolicy }}
+          command:
+            - "powershell.exe"
+            - "-c"
+            - "New-Item -ItemType Directory -Path C:\\var\\lib\\kubelet\\plugins\\{{ .Values.driver.name }}\\ -Force"
+      containers:
+        - name: node-driver-registrar
+{{- if hasPrefix "/" .Values.image.nodeDriverRegistrar.repository }}
+          image: "{{ .Values.image.baseRepo }}{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}"
+{{- else }}
+          image: "{{ .Values.image.nodeDriverRegistrar.repository }}:{{ .Values.image.nodeDriverRegistrar.tag }}"
+{{- end }}
+          command:
+            - "csi-node-driver-registrar.exe"
+          args:
+            - --v=2
+            - --csi-address=$(CSI_ENDPOINT)
+            - --kubelet-registration-path=$(DRIVER_REG_SOCK_PATH)
+            - --plugin-registration-path=$(PLUGIN_REG_DIR)
+          env:
+            - name: CSI_ENDPOINT
+              value: unix://{{ .Values.windows.kubelet | replace "\\" "\\\\" }}\\plugins\\{{ .Values.driver.name }}\\csi.sock
+            - name: DRIVER_REG_SOCK_PATH
+              value: {{ .Values.windows.kubelet | replace "\\" "\\\\" }}\\plugins\\{{ .Values.driver.name }}\\csi.sock
+            - name: PLUGIN_REG_DIR
+              value: {{ .Values.windows.kubelet | replace "\\" "\\\\" }}\\plugins_registry\\
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  fieldPath: spec.nodeName
+          imagePullPolicy: {{ .Values.image.nodeDriverRegistrar.pullPolicy }}
+          resources: {{- toYaml .Values.windows.resources.nodeDriverRegistrar | nindent 12 }}
+        - name: smb
+{{- if hasPrefix "/" .Values.image.smb.repository }}
+          image: "{{ .Values.image.baseRepo }}{{ .Values.image.smb.repository }}:{{ .Values.image.smb.tag }}-windows-hp"
+{{- else }}
+          image: "{{ .Values.image.smb.repository }}:{{ .Values.image.smb.tag }}-windows-hp"
+{{- end }}
+          imagePullPolicy: {{ .Values.image.smb.pullPolicy }}
+          command:
+            - "smbplugin.exe"
+          args:
+            - "--v={{ .Values.node.logLevel }}"
+            - "--drivername={{ .Values.driver.name }}"
+            - --endpoint=$(CSI_ENDPOINT)
+            - --nodeid=$(KUBE_NODE_NAME)
+            - "--enable-get-volume-stats={{ .Values.feature.enableGetVolumeStats }}"
+            - "--remove-smb-mapping-during-unmount={{ .Values.windows.removeSMBMappingDuringUnmount }}"
+            - "--enable-windows-host-process=true"
+          env:
+            - name: CSI_ENDPOINT
+              value: unix://{{ .Values.windows.kubelet | replace "\\" "\\\\" }}\\plugins\\{{ .Values.driver.name }}\\csi.sock
+            - name: KUBE_NODE_NAME
+              valueFrom:
+                fieldRef:
+                  apiVersion: v1
+                  fieldPath: spec.nodeName
+          resources: {{- toYaml .Values.windows.resources.smb | nindent 12 }}
+{{- end -}}

charts/latest/csi-driver-smb/templates/csi-smb-node-windows.yaml

@@ -1,4 +1,4 @@
-{{- if .Values.windows.enabled}}
+{{- if and .Values.windows.enabled (not .Values.windows.useHostProcessContainers) }}
 kind: DaemonSet
 apiVersion: apps/v1
 metadata:

charts/latest/csi-driver-smb/values.yaml

@@ -117,7 +117,8 @@ linux:
         memory: 20Mi
 
 windows:
-  enabled: false # Unless you already had csi proxy installed, windows.csiproxy.enabled=true is required
+  enabled: false
+  useHostProcessContainers: false
   dsName: csi-smb-node-win # daemonset name
   kubelet: 'C:\var\lib\kubelet'
   removeSMBMappingDuringUnmount: true
@@ -145,7 +146,7 @@ windows:
         cpu: 10m
         memory: 40Mi
   csiproxy:
-    enabled: false # required if windows.enabled is true, but may be installed manually also
+    enabled: false # set as false if csi-proxy is already installed on the node or useHostProcessContainers is true
     dsName: csi-proxy-win # daemonset name
     tolerations: {}
     affinity: {}

cmd/smbplugin/Dockerfile.Windows.hostprocess

@@ -0,0 +1,24 @@
+# Copyright 2023 The Kubernetes Authors.
+#
+# Licensed under the Apache License, Version 2.0 (the "License");
+# you may not use this file except in compliance with the License.
+# You may obtain a copy of the License at
+#
+#     http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+
+FROM mcr.microsoft.com/oss/kubernetes/windows-host-process-containers-base-image:v1.0.0
+LABEL description="CSI SMB plugin"
+
+ARG ARCH=amd64
+ARG binary=./_output/${ARCH}/smbplugin.exe
+COPY ${binary} /smbplugin.exe
+ENV PATH="C:\Windows\system32;C:\Windows;C:\WINDOWS\System32\WindowsPowerShell\v1.0\;"
+USER ContainerAdministrator
+ENTRYPOINT ["/smbplugin.exe"]
+

cmd/smbplugin/main.go

@@ -48,6 +48,7 @@ var (
 	krb5CacheDirectory            = flag.String("krb5-cache-directory", smb.DefaultKrb5CacheDirectory, "The directory for kerberos cache")
 	krb5Prefix                    = flag.String("krb5-prefix", smb.DefaultKrb5CCName, "The prefix for kerberos cache")
 	defaultOnDeletePolicy         = flag.String("default-ondelete-policy", "", "default policy for deleting subdirectory when deleting a volume")
+	enableWindowsHostProcess      = flag.Bool("enable-windows-host-process", false, "enable windows host process")
 )
 
 func main() {
@@ -80,6 +81,7 @@ func handle() {
 		Krb5CacheDirectory:            *krb5CacheDirectory,
 		Krb5Prefix:                    *krb5Prefix,
 		DefaultOnDeletePolicy:         *defaultOnDeletePolicy,
+		EnableWindowsHostProcess:      *enableWindowsHostProcess,
 	}
 	driver := smb.NewDriver(&driverOptions)
 	driver.Run(*endpoint, *kubeconfig, false)

deploy/example/smb-provisioner/smb-server-lb.yaml

@@ -30,6 +30,13 @@ spec:
     spec:
       nodeSelector:
         "kubernetes.io/os": linux
+      tolerations:
+        - key: "node-role.kubernetes.io/controlplane"
+          operator: "Exists"
+          effect: "NoSchedule"
+        - key: "node-role.kubernetes.io/control-plane"
+          operator: "Exists"
+          effect: "NoSchedule"
       containers:
         - name: smb-server
           image: andyzhangx/samba:win-fix

deploy/example/smb-provisioner/smb-server.yaml

@@ -30,6 +30,13 @@ spec:
     spec:
       nodeSelector:
         "kubernetes.io/os": linux
+      tolerations:
+        - key: "node-role.kubernetes.io/controlplane"
+          operator: "Exists"
+          effect: "NoSchedule"
+        - key: "node-role.kubernetes.io/control-plane"
+          operator: "Exists"
+          effect: "NoSchedule"
       containers:
         - name: smb-server
           image: andyzhangx/samba:win-fix

pkg/csi-common/utils.go

@@ -18,7 +18,10 @@ package csicommon
 
 import (
 	"fmt"
+	"os"
+	"os/exec"
 	"strings"
+	"sync"
 
 	"context"
 
@@ -28,6 +31,10 @@ import (
 	"k8s.io/klog/v2"
 )
 
+const MaxPathLengthWindows = 260
+
+var mutex = &sync.Mutex{}
+
 func ParseEndpoint(ep string) (string, string, error) {
 	if strings.HasPrefix(strings.ToLower(ep), "unix://") || strings.HasPrefix(strings.ToLower(ep), "tcp://") {
 		s := strings.SplitN(ep, "://", 2)
@@ -84,3 +91,14 @@ func logGRPC(ctx context.Context, req interface{}, info *grpc.UnaryServerInfo, h
 	}
 	return resp, err
 }
+
+func RunPowershellCmd(command string, envs ...string) ([]byte, error) {
+	// only one powershell command can be executed at a time to avoid OOM
+	mutex.Lock()
+	defer mutex.Unlock()
+
+	cmd := exec.Command("powershell", "-Mta", "-NoProfile", "-Command", command)
+	cmd.Env = append(os.Environ(), envs...)
+	klog.V(8).Infof("Executing command: %q", cmd.String())
+	return cmd.CombinedOutput()
+}