feat: improve hostNetwork mode of NodePool by adding NodeAffinity to …

…pods with specified annotation (openyurtio#1935)

charts/yurt-manager/templates/yurt-manager-auto-generated.yaml

@@ -632,6 +632,27 @@ webhooks:
     resources:
     - platformadmins
   sideEffects: None
+- admissionReviewVersions:
+  - v1
+  - v1beta1
+  clientConfig:
+    service:
+      name: yurt-manager-webhook-service
+      namespace: {{ .Release.Namespace }}
+      path: /mutate-core-openyurt-io-v1-pod
+  failurePolicy: Ignore
+  name: mutate.core.v1.pod.openyurt.io
+  rules:
+  - apiGroups:
+    - ""
+    apiVersions:
+    - v1
+    operations:
+    - CREATE
+    - UPDATE
+    resources:
+    - pods
+  sideEffects: None
 - admissionReviewVersions:
   - v1
   - v1beta1

pkg/apis/apps/well_known_labels_annotations.go

@@ -44,3 +44,9 @@ const (
 	NodePoolHostNetworkLabel = "nodepool.openyurt.io/hostnetwork"
 	NodePoolChangedEvent     = "NodePoolChanged"
 )
+
+// Pod related labels and annotations
+const (
+	// AnnotationExcludeHostNetworkPool indicates the pod don't want to be scheduled to nodes in hostNetwork mode NodePool
+	AnnotationExcludeHostNetworkPool = "apps.openyurt.io/exclude-host-network-pool"
+)

pkg/yurtmanager/webhook/pod/v1alpha1/pod_default.go

@@ -0,0 +1,59 @@
+/*
+Copyright 2024 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the License);
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an AS IS BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"context"
+	"fmt"
+
+	"github.com/openyurtio/openyurt/pkg/apis/apps"
+	corev1 "k8s.io/api/core/v1"
+	apierrors "k8s.io/apimachinery/pkg/api/errors"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+// Default implements builder.CustomDefaulter.
+func (webhook *PodHandler) Default(ctx context.Context, obj runtime.Object, req admission.Request) error {
+	pod, ok := obj.(*corev1.Pod)
+	if !ok {
+		return apierrors.NewBadRequest(fmt.Sprintf("expected a Pod but got a %T", obj))
+	}
+
+	// Add NodeAffinity to pods in order to avoid pods to be scheduled on the nodes in the hostNetwork mode NodePool
+	annotations := pod.ObjectMeta.GetAnnotations()
+	if annotations != nil && annotations[apps.AnnotationExcludeHostNetworkPool] == "true" {
+		pod.Spec.Affinity = &corev1.Affinity{
+			NodeAffinity: &corev1.NodeAffinity{
+				RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
+					NodeSelectorTerms: []corev1.NodeSelectorTerm{
+						{
+							MatchExpressions: []corev1.NodeSelectorRequirement{
+								{
+									Key:      "nodepool.openyurt.io/hostnetwork",
+									Operator: corev1.NodeSelectorOpNotIn,
+									Values:   []string{"true"},
+								},
+							},
+						},
+					},
+				},
+			},
+		}
+	}
+	return nil
+}

pkg/yurtmanager/webhook/pod/v1alpha1/pod_default_test.go

@@ -0,0 +1,114 @@
+/*
+Copyright 2024 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the License);
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an AS IS BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"context"
+	"reflect"
+	"testing"
+
+	"github.com/openyurtio/openyurt/pkg/apis/apps"
+	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
+	"k8s.io/apimachinery/pkg/runtime"
+	"sigs.k8s.io/controller-runtime/pkg/webhook/admission"
+)
+
+func TestDefault(t *testing.T) {
+	testcases := map[string]struct {
+		obj         runtime.Object
+		errHappened bool
+		wantedPod   *corev1.Pod
+	}{
+		"it is not a pod": {
+			obj:         &corev1.Node{},
+			errHappened: true,
+		},
+		"pod with annotation['apps.openyurt.io/exclude-host-network-pool'] = true": {
+			obj: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod0",
+					Namespace: metav1.NamespaceDefault,
+					Annotations: map[string]string{
+						apps.AnnotationExcludeHostNetworkPool: "true",
+					},
+				},
+			},
+			wantedPod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod0",
+					Namespace: metav1.NamespaceDefault,
+					Annotations: map[string]string{
+						apps.AnnotationExcludeHostNetworkPool: "true",
+					},
+				},
+				Spec: corev1.PodSpec{
+					Affinity: &corev1.Affinity{
+						NodeAffinity: &corev1.NodeAffinity{
+							RequiredDuringSchedulingIgnoredDuringExecution: &corev1.NodeSelector{
+								NodeSelectorTerms: []corev1.NodeSelectorTerm{
+									{
+										MatchExpressions: []corev1.NodeSelectorRequirement{
+											{
+												Key:      "nodepool.openyurt.io/hostnetwork",
+												Operator: corev1.NodeSelectorOpNotIn,
+												Values:   []string{"true"},
+											},
+										},
+									},
+								},
+							},
+						},
+					},
+				},
+			},
+		},
+		"pod without annotation['apps.openyurt.io/exclude-host-network-pool'] = true": {
+			obj: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod1",
+					Namespace: metav1.NamespaceDefault,
+				},
+			},
+			wantedPod: &corev1.Pod{
+				ObjectMeta: metav1.ObjectMeta{
+					Name:      "pod1",
+					Namespace: metav1.NamespaceDefault,
+				},
+			},
+		},
+	}
+
+	for k, tc := range testcases {
+		t.Run(k, func(t *testing.T) {
+			h := PodHandler{}
+			err := h.Default(context.TODO(), tc.obj, admission.Request{})
+			if tc.errHappened {
+				if err == nil {
+					t.Errorf("expect error, got nil")
+				}
+			} else if err != nil {
+				t.Errorf("expect no error, but got %v", err)
+			} else {
+				currentPod := tc.obj.(*corev1.Pod)
+				if !reflect.DeepEqual(currentPod, tc.wantedPod) {
+					t.Errorf("expect %#+v, got %#+v", tc.wantedPod, currentPod)
+				}
+			}
+		})
+	}
+}

pkg/yurtmanager/webhook/pod/v1alpha1/pod_handler.go

@@ -0,0 +1,56 @@
+/*
+Copyright 2024 The OpenYurt Authors.
+
+Licensed under the Apache License, Version 2.0 (the License);
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an AS IS BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package v1alpha1
+
+import (
+	"github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/builder"
+	"github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util"
+	corev1 "k8s.io/api/core/v1"
+	ctrl "sigs.k8s.io/controller-runtime"
+	"sigs.k8s.io/controller-runtime/pkg/client"
+	"sigs.k8s.io/controller-runtime/pkg/client/apiutil"
+)
+
+const (
+	WebhookName = "pod"
+)
+
+// SetupWebhookWithManager sets up Cluster webhooks. mutate path, validate path, error
+func (webhook *PodHandler) SetupWebhookWithManager(mgr ctrl.Manager) (string, string, error) {
+	// init
+	webhook.Client = mgr.GetClient()
+
+	gvk, err := apiutil.GVKForObject(&corev1.Pod{}, mgr.GetScheme())
+	if err != nil {
+		return "", "", err
+	}
+	return util.GenerateMutatePath(gvk),
+		util.GenerateValidatePath(gvk),
+		builder.WebhookManagedBy(mgr).
+			For(&corev1.Pod{}).
+			WithDefaulter(webhook).
+			Complete()
+}
+
+// +kubebuilder:webhook:path=/mutate-core-openyurt-io-v1-pod,mutating=true,failurePolicy=ignore,sideEffects=None,admissionReviewVersions=v1;v1beta1,groups="",resources=pods,verbs=create;update,versions=v1,name=mutate.core.v1.pod.openyurt.io
+
+// PodHandler implements a validating and defaulting webhook for Cluster.
+type PodHandler struct {
+	Client client.Client
+}
+
+var _ builder.CustomDefaulter = &PodHandler{}

pkg/yurtmanager/webhook/server.go

@@ -36,6 +36,7 @@ import (
 	v1beta1nodepool "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/nodepool/v1beta1"
 	v1alpha1platformadmin "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/platformadmin/v1alpha1"
 	v1alpha2platformadmin "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/platformadmin/v1alpha2"
+	v1alpha1pod "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/pod/v1alpha1"
 	"github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util"
 	webhookcontroller "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/util/controller"
 	v1alpha1yurtappdaemon "github.com/openyurtio/openyurt/pkg/yurtmanager/webhook/yurtappdaemon/v1alpha1"
@@ -81,6 +82,7 @@ func init() {
 	addControllerWebhook(names.YurtAppOverriderController, &v1alpha1deploymentrender.DeploymentRenderHandler{})
 
 	independentWebhooks[v1node.WebhookName] = &v1node.NodeHandler{}
+	independentWebhooks[v1alpha1pod.WebhookName] = &v1alpha1pod.PodHandler{}
 }
 
 // Note !!! @kadisi