From 7c2684b02b5f3447537b1be37f43c8ac75b33584 Mon Sep 17 00:00:00 2001 From: shaofan-hs Date: Tue, 15 Aug 2023 18:23:46 +0800 Subject: [PATCH] fix, register podopslifecycle webhook --- apis/apps/v1alpha1/types.go | 4 ++ pkg/webhook/server/add_generic.go | 1 + .../generic/pod/opslifecycle/webhook.go | 64 ++++++++++++++++--- .../generic/pod/opslifecycle/webhook_test.go | 37 ++++++++++- .../generic/pod/pod_mutating_handler.go | 14 ++-- .../generic/pod/pod_validating_handler.go | 4 +- pkg/webhook/server/generic/pod/pod_webhook.go | 23 ++++--- 7 files changed, 115 insertions(+), 32 deletions(-) diff --git a/apis/apps/v1alpha1/types.go b/apis/apps/v1alpha1/types.go index acf4928b..9c65a069 100644 --- a/apis/apps/v1alpha1/types.go +++ b/apis/apps/v1alpha1/types.go @@ -16,6 +16,10 @@ limitations under the License. package v1alpha1 +const ( + PodOperationProtectionFinalizerPrefix = "prot.lifecycle.kafed.kusionstack.io" +) + // +kubebuilder:object:generate=false type PodAvailableConditions struct { ExpectedFinalizers []string `json:"expectedFinalizers,omitempty"` // indicate the expected finalizers of a pod diff --git a/pkg/webhook/server/add_generic.go b/pkg/webhook/server/add_generic.go index 85b8d577..e898dae1 100644 --- a/pkg/webhook/server/add_generic.go +++ b/pkg/webhook/server/add_generic.go @@ -22,6 +22,7 @@ import ( "k8s.io/klog/v2" "kusionstack.io/kafed/pkg/webhook/server/generic" + _ "kusionstack.io/kafed/pkg/webhook/server/generic/pod" ) func init() { diff --git a/pkg/webhook/server/generic/pod/opslifecycle/webhook.go b/pkg/webhook/server/generic/pod/opslifecycle/webhook.go index 3908b7b8..0b234403 100644 --- a/pkg/webhook/server/generic/pod/opslifecycle/webhook.go +++ b/pkg/webhook/server/generic/pod/opslifecycle/webhook.go @@ -34,6 +34,10 @@ import ( "kusionstack.io/kafed/pkg/log" ) +const ( + waitingForLifecycleSeconds int64 = 5 +) + var ( // some labels must exist together and have the same id, and they are a pair pairLabelPrefixesMap = map[string]string{ @@ -52,17 +56,19 @@ var ( ) type ReadyToUpgrade func(pod *corev1.Pod) (bool, []string, *time.Duration) +type SatisfyExpectedFinalizers func(pod *corev1.Pod) (bool, []string, error) type TimeLabelValue func() string type OpsLifecycle struct { - readyToUpgrade ReadyToUpgrade - - timeLabelValue TimeLabelValue + readyToUpgrade ReadyToUpgrade // for testing + satisfyExpectedFinalizers SatisfyExpectedFinalizers + timeLabelValue TimeLabelValue } -func New(readyToUpgrade ReadyToUpgrade) *OpsLifecycle { +func New() *OpsLifecycle { return &OpsLifecycle{ - readyToUpgrade: readyToUpgrade, + readyToUpgrade: hasNoBlockingFinalizer, + satisfyExpectedFinalizers: satisfyExpectedFinalizers, timeLabelValue: func() string { return strconv.FormatInt(time.Now().Unix(), 10) }, @@ -121,7 +127,7 @@ func (lc *OpsLifecycle) Validating(ctx context.Context, pod *corev1.Pod, logger } func (lc *OpsLifecycle) Mutating(ctx context.Context, oldPod, newPod *corev1.Pod, _ client.Client, logger *log.Logger) error { - lc.addReadinessGates(newPod, v1alpha1.ReadinessGatePodServiceReady) + addReadinessGates(newPod, v1alpha1.ReadinessGatePodServiceReady) newIdToLabelsMap, typeToNumsMap, err := podopslifecycle.PodIDAndTypesMap(newPod) if err != nil { @@ -245,7 +251,7 @@ func (lc *OpsLifecycle) addLabelWithTime(pod *corev1.Pod, key string) { pod.Labels[key] = lc.timeLabelValue() } -func (lc *OpsLifecycle) addReadinessGates(pod *corev1.Pod, conditionType corev1.PodConditionType) { +func addReadinessGates(pod *corev1.Pod, conditionType corev1.PodConditionType) { for _, v := range pod.Spec.ReadinessGates { if v.ConditionType == conditionType { return @@ -256,11 +262,11 @@ func (lc *OpsLifecycle) addReadinessGates(pod *corev1.Pod, conditionType corev1. }) } -func (lc *OpsLifecycle) satisfyExpectedFinalizers(pod *corev1.Pod) (bool, []string, error) { +func satisfyExpectedFinalizers(pod *corev1.Pod) (bool, []string, error) { satisfy := true var expectedFinalizer []string // expected finalizers that are not satisfied - availableConditions, err := lc.podAvailableConditions(pod) + availableConditions, err := podAvailableConditions(pod) if err != nil { return satisfy, expectedFinalizer, err } @@ -282,7 +288,7 @@ func (lc *OpsLifecycle) satisfyExpectedFinalizers(pod *corev1.Pod) (bool, []stri return satisfy, expectedFinalizer, nil } -func (lc *OpsLifecycle) podAvailableConditions(pod *corev1.Pod) (*v1alpha1.PodAvailableConditions, error) { +func podAvailableConditions(pod *corev1.Pod) (*v1alpha1.PodAvailableConditions, error) { if pod.Annotations == nil { return nil, nil } @@ -298,3 +304,41 @@ func (lc *OpsLifecycle) podAvailableConditions(pod *corev1.Pod) (*v1alpha1.PodAv } return availableConditions, nil } + +func hasNoBlockingFinalizer(pod *corev1.Pod) (bool, []string, *time.Duration) { + if pod == nil { + return true, nil, nil + } + + hasReadinessGate := false + if pod.Spec.ReadinessGates != nil { + for _, readinessGate := range pod.Spec.ReadinessGates { + if readinessGate.ConditionType == v1alpha1.ReadinessGatePodServiceReady { + hasReadinessGate = true + break + } + } + } + if !hasReadinessGate { + // if has no service-ready ReadinessGate, treat it as normal pod. + return true, nil, nil + } + + if pod.ObjectMeta.Finalizers == nil || len(pod.ObjectMeta.Finalizers) == 0 { + return true, nil, nil + } + + var finalizers []string + for _, f := range pod.ObjectMeta.Finalizers { + if strings.HasPrefix(f, v1alpha1.PodOperationProtectionFinalizerPrefix) { + finalizers = append(finalizers, f) + } + } + + if len(finalizers) > 0 { + requeneAfter := time.Duration(waitingForLifecycleSeconds) * time.Second + return false, finalizers, &requeneAfter + } + + return true, nil, nil +} diff --git a/pkg/webhook/server/generic/pod/opslifecycle/webhook_test.go b/pkg/webhook/server/generic/pod/opslifecycle/webhook_test.go index 88ab1d87..5045dea6 100644 --- a/pkg/webhook/server/generic/pod/opslifecycle/webhook_test.go +++ b/pkg/webhook/server/generic/pod/opslifecycle/webhook_test.go @@ -107,7 +107,8 @@ func TestMutating(t *testing.T) { newPodLabels map[string]string expectedLabels map[string]string - readyToUpgrade ReadyToUpgrade + satisfyExpectedFinalizers SatisfyExpectedFinalizers + readyToUpgrade ReadyToUpgrade keyWords string // used to check the error message }{ @@ -376,6 +377,27 @@ func TestMutating(t *testing.T) { }, }, + { + notes: "wait for removing finalizers", + newPodLabels: map[string]string{ + fmt.Sprintf("%s/%s", v1alpha1.PodOperatedLabelPrefix, "123"): "1402144848", + fmt.Sprintf("%s/%s", v1alpha1.PodDoneOperationTypeLabelPrefix, "123"): "upgrade", + fmt.Sprintf("%s/%s", v1alpha1.PodPostCheckLabelPrefix, "123"): "1402144848", + fmt.Sprintf("%s/%s", v1alpha1.PodPostCheckedLabelPrefix, "123"): "1402144848", + + fmt.Sprintf("%s/%s", v1alpha1.PodCompleteLabelPrefix, "123"): "1402144848", + }, + expectedLabels: map[string]string{ + fmt.Sprintf("%s/%s", v1alpha1.PodOperatedLabelPrefix, "123"): "1402144848", + fmt.Sprintf("%s/%s", v1alpha1.PodDoneOperationTypeLabelPrefix, "123"): "upgrade", + fmt.Sprintf("%s/%s", v1alpha1.PodPostCheckLabelPrefix, "123"): "1402144848", + fmt.Sprintf("%s/%s", v1alpha1.PodPostCheckedLabelPrefix, "123"): "1402144848", + + fmt.Sprintf("%s/%s", v1alpha1.PodCompleteLabelPrefix, "123"): "1402144848", + }, + satisfyExpectedFinalizers: satifyExpectedFinalizersReturnFalse, + }, + { notes: "all finished", newPodLabels: map[string]string{ @@ -423,6 +445,11 @@ func TestMutating(t *testing.T) { opslifecycle.readyToUpgrade = readyToUpgradeReturnTrue } + opslifecycle.satisfyExpectedFinalizers = v.satisfyExpectedFinalizers + if opslifecycle.satisfyExpectedFinalizers == nil { + opslifecycle.satisfyExpectedFinalizers = satifyExpectedFinalizersReturnTrue + } + t.Logf("notes: %s", v.notes) err := opslifecycle.Mutating(context.Background(), oldPod, newPod, nil, nil) if v.keyWords == "" { @@ -443,3 +470,11 @@ func readyToUpgradeReturnTrue(pod *corev1.Pod) (bool, []string, *time.Duration) func readyToUpgradeReturnFalse(pod *corev1.Pod) (bool, []string, *time.Duration) { return false, nil, nil } + +func satifyExpectedFinalizersReturnTrue(pod *corev1.Pod) (bool, []string, error) { + return true, nil, nil +} + +func satifyExpectedFinalizersReturnFalse(pod *corev1.Pod) (bool, []string, error) { + return false, nil, nil +} diff --git a/pkg/webhook/server/generic/pod/pod_mutating_handler.go b/pkg/webhook/server/generic/pod/pod_mutating_handler.go index 7d330cd3..755b7baa 100644 --- a/pkg/webhook/server/generic/pod/pod_mutating_handler.go +++ b/pkg/webhook/server/generic/pod/pod_mutating_handler.go @@ -41,14 +41,14 @@ var ( ) type MutatingHandler struct { - needLifecycle NeedOpsLifecycle - opsLifecycle *opslifecycle.OpsLifecycle + needOpsLifecycle NeedOpsLifecycle + opsLifecycle *opslifecycle.OpsLifecycle } -func NewMutatingHandler(needLifecycle NeedOpsLifecycle, readyToUpgrade opslifecycle.ReadyToUpgrade) *MutatingHandler { +func NewMutatingHandler(needOpsLifecycle NeedOpsLifecycle) *MutatingHandler { return &MutatingHandler{ - needLifecycle: needLifecycle, - opsLifecycle: opslifecycle.New(readyToUpgrade), + needOpsLifecycle: needOpsLifecycle, + opsLifecycle: opslifecycle.New(), } } @@ -69,7 +69,7 @@ func (h *MutatingHandler) Handle(ctx context.Context, req admission.Request, c c } if req.Operation == admissionv1.Create { - if !h.needLifecycle(nil, pod) { + if h.needOpsLifecycle != nil && !h.needOpsLifecycle(nil, pod) { return admission.Patched("Not need opslifecycle mutating") } } @@ -80,7 +80,7 @@ func (h *MutatingHandler) Handle(ctx context.Context, req admission.Request, c c return admission.Errored(http.StatusBadRequest, fmt.Errorf("fail to unmarshal old object: %s", err)) } - if !h.needLifecycle(old, pod) { + if h.needOpsLifecycle != nil && !h.needOpsLifecycle(old, pod) { return admission.Patched("Not need opslifecycle mutating") } diff --git a/pkg/webhook/server/generic/pod/pod_validating_handler.go b/pkg/webhook/server/generic/pod/pod_validating_handler.go index cad875ab..6212d4aa 100644 --- a/pkg/webhook/server/generic/pod/pod_validating_handler.go +++ b/pkg/webhook/server/generic/pod/pod_validating_handler.go @@ -51,8 +51,8 @@ func (h *ValidatingHandler) Handle(ctx context.Context, req admission.Request, c s, _ := json.Marshal(req) return admission.Errored(http.StatusBadRequest, fmt.Errorf("fail to decode old object from request %s: %s", s, err)) } - if !h.needOpsLifecycle(nil, obj) { - return admission.Allowed("pod is allowed by opslifecycle") + if h.needOpsLifecycle != nil && !h.needOpsLifecycle(nil, obj) { + return admission.Allowed("pod is ignored by opslifecycle") } if err := h.opslifecycle.Validating(ctx, obj, logger); err != nil { diff --git a/pkg/webhook/server/generic/pod/pod_webhook.go b/pkg/webhook/server/generic/pod/pod_webhook.go index 84e16b69..a2f2d18d 100644 --- a/pkg/webhook/server/generic/pod/pod_webhook.go +++ b/pkg/webhook/server/generic/pod/pod_webhook.go @@ -19,25 +19,24 @@ package pod import ( v1 "k8s.io/api/core/v1" + "kusionstack.io/kafed/apis/apps/v1alpha1" "kusionstack.io/kafed/pkg/webhook/server/generic" - "kusionstack.io/kafed/pkg/webhook/server/generic/pod/opslifecycle" ) type NeedOpsLifecycle func(oldPod, newPod *v1.Pod) bool -type PodWebhook struct { - mutatingHandler *MutatingHandler - validatingHandler *ValidatingHandler +func init() { + generic.MutatingTypeHandlerMap["Pod"] = NewMutatingHandler(ControlledByKafed) + generic.ValidatingTypeHandlerMap["Pod"] = NewValidatingHandler(ControlledByKafed) } -func NewPodWebhook(needLifecycle NeedOpsLifecycle, readyToUpgrade opslifecycle.ReadyToUpgrade) *PodWebhook { - return &PodWebhook{ - mutatingHandler: NewMutatingHandler(needLifecycle, readyToUpgrade), - validatingHandler: NewValidatingHandler(needLifecycle), +func ControlledByKafed(oldPod, newPod *v1.Pod) bool { + if newPod == nil || newPod.Labels == nil { + return false + } + if v, ok := newPod.Labels[v1alpha1.KafedSystemLabel]; !ok || v != "true" { + return false } -} -func (h *PodWebhook) RegisterToDispatcher() { - generic.MutatingTypeHandlerMap["Pod"] = h.mutatingHandler - generic.ValidatingTypeHandlerMap["Pod"] = h.validatingHandler + return true }