From ca9c52bf69f89971017d2d8ff87236db6072cf66 Mon Sep 17 00:00:00 2001 From: "bingchang.tbc" Date: Tue, 16 Apr 2024 17:19:57 +0800 Subject: [PATCH] aggregated annotations and labels. --- .../network/well_known_labels_annotations.go | 7 +- .../loadbalancerset/aggregatetoannotations.go | 47 +++++ .../loadbalancerset/aggregatetolabels.go | 46 +++++ .../loadbalancerset/annotations.go | 167 ------------------ .../loadbalancerset_controller.go | 8 +- .../loadbalancerset_controller_test.go | 87 ++++----- .../loadbalancerset/map_utils.go | 154 ++++++++++++++++ .../loadbalancerset/predicate.go | 2 +- .../loadbalancerset/predicate_test.go | 14 +- 9 files changed, 308 insertions(+), 224 deletions(-) create mode 100644 pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/aggregatetoannotations.go create mode 100644 pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/aggregatetolabels.go delete mode 100644 pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/annotations.go create mode 100644 pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/map_utils.go diff --git a/pkg/apis/network/well_known_labels_annotations.go b/pkg/apis/network/well_known_labels_annotations.go index f91eb1ff0a9..fdd2a2e6b67 100644 --- a/pkg/apis/network/well_known_labels_annotations.go +++ b/pkg/apis/network/well_known_labels_annotations.go @@ -17,8 +17,7 @@ limitations under the License. package network const ( - LabelServiceName = "openyurt.io/service-name" - LabelNodePoolName = "openyurt.io/pool-name" - AnnotationNodePoolSelector = "service.openyurt.io/nodepool-labelselector" - AggregateAnnotationsKeyPrefix = "service.openyurt.io" + LabelServiceName = "openyurt.io/service-name" + LabelNodePoolName = "openyurt.io/pool-name" + AnnotationNodePoolSelector = "service.openyurt.io/nodepool-labelselector" ) diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/aggregatetoannotations.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/aggregatetoannotations.go new file mode 100644 index 00000000000..121f70c55fb --- /dev/null +++ b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/aggregatetoannotations.go @@ -0,0 +1,47 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package loadbalancerset + +import ( + "reflect" + + corev1 "k8s.io/api/core/v1" + + netv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/network/v1alpha1" +) + +func aggregatePoolServicesAnnotations(poolServices []netv1alpha1.PoolService) map[string]string { + aggregatedAnnotations := make(map[string]string) + for _, ps := range poolServices { + aggregatedAnnotations = mergeMap(aggregatedAnnotations, ps.Status.AggregateToAnnotations) + } + + return addAggregatePrefix(aggregatedAnnotations) +} + +func compareAndUpdateServiceAnnotations(svc *corev1.Service, aggregatedAnnotations map[string]string) bool { + currentAggregatedServiceAnnotations := filterIgnoredKeys(svc.Annotations) + + if reflect.DeepEqual(currentAggregatedServiceAnnotations, aggregatedAnnotations) { + return false + } + + update, deletion := diffMap(currentAggregatedServiceAnnotations, aggregatedAnnotations) + svc.Annotations = updateMap(svc.Annotations, update, deletion) + + return true +} diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/aggregatetolabels.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/aggregatetolabels.go new file mode 100644 index 00000000000..84de7c11ea4 --- /dev/null +++ b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/aggregatetolabels.go @@ -0,0 +1,46 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package loadbalancerset + +import ( + "reflect" + + v1 "k8s.io/api/core/v1" + + "github.com/openyurtio/openyurt/pkg/apis/network/v1alpha1" +) + +func aggregatePoolServicesLabels(poolServices []v1alpha1.PoolService) map[string]string { + aggregatedLabels := make(map[string]string) + for _, ps := range poolServices { + aggregatedLabels = mergeMap(aggregatedLabels, ps.Status.AggregateToLabels) + } + return addAggregatePrefix(aggregatedLabels) +} + +func compareAndUpdateServiceLabels(svc *v1.Service, labels map[string]string) bool { + currentAggregatedLabels := filterIgnoredKeys(svc.Labels) + + if reflect.DeepEqual(currentAggregatedLabels, labels) { + return false + } + + update, deletion := diffMap(currentAggregatedLabels, labels) + svc.Labels = updateMap(svc.Labels, update, deletion) + + return true +} diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/annotations.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/annotations.go deleted file mode 100644 index 8dcd8f5569a..00000000000 --- a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/annotations.go +++ /dev/null @@ -1,167 +0,0 @@ -/* -Copyright 2024 The OpenYurt Authors. - -Licensed under the Apache License, Version 2.0 (the License); -you may not use this file except in compliance with the License. -You may obtain a copy of the License at - - http://www.apache.org/licenses/LICENSE-2.0 - -Unless required by applicable law or agreed to in writing, software -distributed under the License is distributed on an AS IS BASIS, -WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. -See the License for the specific language governing permissions and -limitations under the License. -*/ - -package loadbalancerset - -import ( - "reflect" - "sort" - "strings" - - corev1 "k8s.io/api/core/v1" - - "github.com/openyurtio/openyurt/pkg/apis/network" - netv1alpha1 "github.com/openyurtio/openyurt/pkg/apis/network/v1alpha1" -) - -func aggregatePoolServicesAnnotations(poolServices []netv1alpha1.PoolService) map[string]string { - aggregatedAnnotations := make(map[string]string) - for _, ps := range poolServices { - aggregatedAnnotations = mergeAnnotations(aggregatedAnnotations, filterIgnoredKeys(ps.Annotations)) - } - - return aggregatedAnnotations -} - -func filterIgnoredKeys(annotations map[string]string) map[string]string { - newAnnotations := make(map[string]string) - for key, value := range annotations { - if key == network.AnnotationNodePoolSelector { - continue - } - if !strings.HasPrefix(key, network.AggregateAnnotationsKeyPrefix) { - continue - } - newAnnotations[key] = value - } - return newAnnotations -} - -func mergeAnnotations(m map[string]string, elem map[string]string) map[string]string { - if len(elem) == 0 { - return m - } - - if m == nil { - m = make(map[string]string) - } - - for k, v := range elem { - m[k] = mergeAnnotationValue(m[k], v) - } - - return m -} - -func mergeAnnotationValue(originalValue, addValue string) string { - if len(originalValue) == 0 { - return addValue - } - - if len(addValue) == 0 { - return originalValue - } - - splitOriginalValues := strings.Split(originalValue, ",") - if valueIsExist(splitOriginalValues, addValue) { - return originalValue - } - - return joinNewValue(splitOriginalValues, addValue) -} - -func valueIsExist(originalValueList []string, addValue string) bool { - for _, oldValue := range originalValueList { - if addValue == oldValue { - return true - } - } - return false -} - -func joinNewValue(originalValueList []string, addValue string) string { - originalValueList = append(originalValueList, addValue) - sort.Strings(originalValueList) - - return strings.Join(originalValueList, ",") -} - -func compareAndUpdateServiceAnnotations(svc *corev1.Service, aggregatedAnnotations map[string]string) bool { - currentAggregatedServiceAnnotations := filterIgnoredKeys(svc.Annotations) - - if reflect.DeepEqual(currentAggregatedServiceAnnotations, aggregatedAnnotations) { - return false - } - - update, deletion := diffAnnotations(currentAggregatedServiceAnnotations, aggregatedAnnotations) - updateAnnotations(svc.Annotations, update, deletion) - - return true -} - -func diffAnnotations(currentAnnotations, desiredAnnotations map[string]string) (update map[string]string, deletion map[string]string) { - if currentAnnotations == nil { - return desiredAnnotations, nil - } - if desiredAnnotations == nil { - return nil, currentAnnotations - } - - update = make(map[string]string) - for key, value := range desiredAnnotations { - if currentAnnotations[key] != value { - update[key] = value - } - } - - deletion = make(map[string]string) - for key, value := range currentAnnotations { - if _, exist := desiredAnnotations[key]; !exist { - deletion[key] = value - } - } - return -} - -func updateAnnotations(annotations, update, deletion map[string]string) { - if len(update) == 0 && len(deletion) == 0 { - return - } - if annotations == nil { - annotations = make(map[string]string) - } - for key, value := range update { - annotations[key] = value - } - - for key := range deletion { - delete(annotations, key) - } -} - -func annotationValueIsEqual(oldAnnotations, newAnnotations map[string]string, key string) bool { - var oldValue string - if oldAnnotations != nil { - oldValue = oldAnnotations[key] - } - - var newValue string - if newAnnotations != nil { - newValue = newAnnotations[key] - } - - return oldValue == newValue -} diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller.go index 654bac4d6e9..c5bd0d5444a 100644 --- a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller.go +++ b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller.go @@ -489,17 +489,19 @@ func (r *ReconcileLoadBalancerSet) syncService(svc *corev1.Service) error { return errors.Wrapf(err, "failed to get current pool services for service %s/%s", svc.Namespace, svc.Name) } + aggregatedLabels := aggregatePoolServicesLabels(poolServices) aggregatedAnnotations := aggregatePoolServicesAnnotations(poolServices) aggregatedLbStatus := aggregateLbStatus(poolServices) - return r.compareAndUpdateService(svc, aggregatedAnnotations, aggregatedLbStatus) + return r.compareAndUpdateService(svc, aggregatedLabels, aggregatedAnnotations, aggregatedLbStatus) } -func (r *ReconcileLoadBalancerSet) compareAndUpdateService(svc *corev1.Service, annotations map[string]string, lbStatus corev1.LoadBalancerStatus) error { +func (r *ReconcileLoadBalancerSet) compareAndUpdateService(svc *corev1.Service, labels, annotations map[string]string, lbStatus corev1.LoadBalancerStatus) error { isUpdatedAnnotations := compareAndUpdateServiceAnnotations(svc, annotations) isUpdatedLbStatus := compareAndUpdateServiceLbStatus(svc, lbStatus) + isUpdatedLabels := compareAndUpdateServiceLabels(svc, labels) - if !isUpdatedLbStatus && !isUpdatedAnnotations { + if !isUpdatedLbStatus && !isUpdatedAnnotations && !isUpdatedLabels { return nil } diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller_test.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller_test.go index 1fd243d0cb2..b2e984dad47 100644 --- a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller_test.go +++ b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/loadbalancerset_controller_test.go @@ -129,8 +129,8 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { np1 := newNodepool("np123", "name=np123,app=deploy") np2 := newNodepool("np234", "name=np234") - ps1 := newPoolService(v1.NamespaceDefault, "np123", nil, nil) - ps2 := newPoolService(v1.NamespaceDefault, "np234", nil, nil) + ps1 := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) + ps2 := newPoolService(v1.NamespaceDefault, "np234", nil, nil, nil) c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(np1).WithObjects(np2).WithObjects(ps1).WithObjects(ps2).Build() @@ -153,9 +153,9 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { np1 := newNodepool("np123", "name=np123,app=deploy") np2 := newNodepool("np234", "name=np234,app=deploy") - ps1 := newPoolService(v1.NamespaceDefault, "np123", nil, []corev1.LoadBalancerIngress{{IP: "2.2.3.4"}}) - ps2 := newPoolService(v1.NamespaceDefault, "np234", nil, []corev1.LoadBalancerIngress{{IP: "1.2.3.4"}}) - ps3 := newPoolService(v1.NamespaceSystem, "np234", nil, []corev1.LoadBalancerIngress{{IP: "3.4.5.6"}}) + ps1 := newPoolService(v1.NamespaceDefault, "np123", nil, nil, []corev1.LoadBalancerIngress{{IP: "2.2.3.4"}}) + ps2 := newPoolService(v1.NamespaceDefault, "np234", nil, nil, []corev1.LoadBalancerIngress{{IP: "1.2.3.4"}}) + ps3 := newPoolService(v1.NamespaceSystem, "np234", nil, nil, []corev1.LoadBalancerIngress{{IP: "3.4.5.6"}}) c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(np1).WithObjects(np2).WithObjects(ps1).WithObjects(ps2).WithObjects(ps3).Build() @@ -176,16 +176,16 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { assertLoadBalancerStatusEqual(t, expectedStatus, newSvc.Status.LoadBalancer.Ingress) }) - t.Run("test aggregate annotations multi pool services", func(t *testing.T) { + t.Run("test aggregate annotations/labels multi pool services", func(t *testing.T) { svc := newService(v1.NamespaceDefault, mockServiceName) np1 := newNodepool("np123", "name=np123,app=deploy") np2 := newNodepool("np234", "name=np234,app=deploy") - ps1 := newPoolService(v1.NamespaceDefault, "np123", map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb34567"}, nil) - ps2 := newPoolService(v1.NamespaceDefault, "np234", map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb23456"}, nil) - ps3 := newPoolService(v1.NamespaceDefault, "np345", map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb12345"}, nil) - ps4 := newPoolService(v1.NamespaceDefault, "np456", map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb12345"}, nil) + ps1 := newPoolService(v1.NamespaceDefault, "np123", map[string]string{"lb-id": "lb34567"}, map[string]string{"lb-id": "lb34567"}, nil) + ps2 := newPoolService(v1.NamespaceDefault, "np234", map[string]string{"lb-id": "lb23456"}, map[string]string{"lb-id": "lb23456"}, nil) + ps3 := newPoolService(v1.NamespaceDefault, "np345", map[string]string{"lb-id": "lb12345"}, map[string]string{"lb-id": "lb12345"}, nil) + ps4 := newPoolService(v1.NamespaceDefault, "np456", map[string]string{"lb-id": "lb12345"}, map[string]string{"lb-id": "lb12345"}, nil) c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(np1).WithObjects(np2). WithObjects(ps1).WithObjects(ps2).WithObjects(ps3).WithObjects(ps4).Build() @@ -203,9 +203,11 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { Name: mockServiceName, }, newSvc) - expectedAnnotations := map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb23456,lb34567"} + expectedAnnotations := map[string]string{aggregateKeyPrefix + "lb-id": "lb23456,lb34567"} + expectedLabels := map[string]string{aggregateKeyPrefix + "lb-id": "lb23456,lb34567"} - assertOpenYurtAnnotationsEqual(t, expectedAnnotations, newSvc.Annotations) + assertMapEqual(t, expectedAnnotations, newSvc.Annotations) + assertMapEqual(t, expectedLabels, newSvc.Labels) assertResourceVersion(t, "1001", newSvc.ResourceVersion) }) @@ -215,9 +217,9 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { np1 := newNodepool("np123", "name=np123,app=deploy") np2 := newNodepool("np234", "name=np234,app=deploy") - ps1 := newPoolService(v1.NamespaceDefault, "np123", map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb34567"}, nil) - ps2 := newPoolService(v1.NamespaceDefault, "np234", map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb23456"}, nil) - svc.Annotations[network.AggregateAnnotationsKeyPrefix+"/lb-id"] = "lb23456,lb34567" + ps1 := newPoolService(v1.NamespaceDefault, "np123", nil, map[string]string{"lb-id": "lb34567"}, nil) + ps2 := newPoolService(v1.NamespaceDefault, "np234", nil, map[string]string{"lb-id": "lb23456"}, nil) + svc.Annotations[aggregateKeyPrefix+"lb-id"] = "lb23456,lb34567" c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(np1).WithObjects(np2). WithObjects(ps1).WithObjects(ps2).Build() @@ -235,9 +237,9 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { Name: mockServiceName, }, newSvc) - expectedAnnotations := map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb23456,lb34567"} + expectedAnnotations := map[string]string{aggregateKeyPrefix + "lb-id": "lb23456,lb34567"} - assertOpenYurtAnnotationsEqual(t, expectedAnnotations, newSvc.Annotations) + assertMapEqual(t, expectedAnnotations, newSvc.Annotations) assertResourceVersion(t, "1000", newSvc.ResourceVersion) }) @@ -247,8 +249,8 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { np1 := newNodepool("np123", "name=np123,app=deploy") np2 := newNodepool("np234", "name=np234,app=deploy") - ps1 := newPoolService(v1.NamespaceDefault, "np123", map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb34567"}, nil) - svc.Annotations[network.AggregateAnnotationsKeyPrefix+"/lb-id"] = "lb23456,lb34567" + ps1 := newPoolService(v1.NamespaceDefault, "np123", nil, map[string]string{"lb-id": "lb34567"}, nil) + svc.Annotations["lb-id"] = "lb23456,lb34567" c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(np1).WithObjects(np2). WithObjects(ps1).Build() @@ -266,9 +268,9 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { Name: mockServiceName, }, newSvc) - expectedAnnotations := map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb34567"} + expectedAnnotations := map[string]string{aggregateKeyPrefix + "lb-id": "lb34567"} - assertOpenYurtAnnotationsEqual(t, expectedAnnotations, newSvc.Annotations) + assertMapEqual(t, expectedAnnotations, newSvc.Annotations) assertResourceVersion(t, "1001", newSvc.ResourceVersion) }) @@ -278,7 +280,7 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { np1 := newNodepool("np123", "name=np123,app=deploy") np2 := newNodepool("np234", "name=np234,app=deploy") - svc.Annotations[network.AggregateAnnotationsKeyPrefix+"/lb-id"] = "lb23456,lb34567" + svc.Annotations[aggregateKeyPrefix+"lb-id"] = "lb23456,lb34567" c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(np1).WithObjects(np2).Build() @@ -297,7 +299,7 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { expectedAnnotations := map[string]string{} - assertOpenYurtAnnotationsEqual(t, expectedAnnotations, newSvc.Annotations) + assertMapEqual(t, expectedAnnotations, newSvc.Annotations) assertResourceVersion(t, "1001", newSvc.ResourceVersion) }) @@ -375,7 +377,7 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { controllerutil.AddFinalizer(svc, poolServiceFinalizer) svc.DeletionTimestamp = &v1.Time{Time: time.Now()} - ps1 := newPoolService(v1.NamespaceDefault, "np123", map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb34567"}, nil) + ps1 := newPoolService(v1.NamespaceDefault, "np123", nil, map[string]string{"lb-id": "lb34567"}, nil) c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(ps1).Build() @@ -403,8 +405,8 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { controllerutil.AddFinalizer(svc, poolServiceFinalizer) svc.DeletionTimestamp = &v1.Time{Time: time.Now()} - ps1 := newPoolService(v1.NamespaceDefault, "np123", map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb34567"}, nil) - ps2 := newPoolService(v1.NamespaceDefault, "np234", map[string]string{network.AggregateAnnotationsKeyPrefix + "/lb-id": "lb45678"}, nil) + ps1 := newPoolService(v1.NamespaceDefault, "np123", nil, map[string]string{"lb-id": "lb34567"}, nil) + ps2 := newPoolService(v1.NamespaceDefault, "np234", nil, map[string]string{"lb-id": "lb45678"}, nil) controllerutil.AddFinalizer(ps1, "elb.openyurt.io/resources") c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(ps1).WithObjects(ps2).Build() @@ -476,7 +478,7 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { np := newNodepool("np123", "name=np123,app=deploy") np.DeletionTimestamp = &v1.Time{Time: time.Now()} - ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(np).WithObjects(ps).Build() @@ -496,7 +498,7 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { svc := newService(v1.NamespaceDefault, mockServiceName) svc.Spec.Type = corev1.ServiceTypeClusterIP np := newNodepool("np123", "name=np123,app=deploy") - ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(np).WithObjects(ps).Build() @@ -523,7 +525,7 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { delete(svc.Annotations, network.AnnotationNodePoolSelector) np := newNodepool("np123", "name=np123,app=deploy") - ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(np).WithObjects(ps).Build() @@ -540,7 +542,7 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { t.Run("not managed by controller", func(t *testing.T) { svc := newService(v1.NamespaceDefault, mockServiceName) - ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) delete(ps.Labels, labelManageBy) c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(ps).Build() @@ -559,7 +561,7 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { t.Run("Modifying the service name to reference a non-existent service", func(t *testing.T) { svc := newService(v1.NamespaceDefault, mockServiceName) - ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) ps.Labels[network.LabelServiceName] = "mock" ps.OwnerReferences = nil ps.Labels[network.LabelNodePoolName] = "np111" @@ -617,7 +619,7 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { svc1 := newService(v1.NamespaceDefault, mockServiceName) svc2 := newService(v1.NamespaceDefault, "mock") - ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) ps.Labels[network.LabelServiceName] = "mock" ps.OwnerReferences = nil ps.Labels[network.LabelNodePoolName] = "np111" @@ -648,7 +650,7 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { svc := newService(v1.NamespaceDefault, mockServiceName) np := newNodepool("np123", "name=np123,app=deploy") - ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) ps.Labels[network.LabelNodePoolName] = "np111" c := fakeclient.NewClientBuilder().WithScheme(scheme).WithObjects(svc).WithObjects(ps).WithObjects(np).Build() @@ -676,7 +678,7 @@ func TestReconcilePoolService_Reconcile(t *testing.T) { t.Run("modifying the pool service is not managed by controller", func(t *testing.T) { svc := newService(v1.NamespaceDefault, mockServiceName) - ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) ps.Labels[network.LabelServiceName] = "mock" delete(ps.Labels, labelManageBy) @@ -831,7 +833,7 @@ func assertPoolServiceLabels(t testing.TB, psl *v1alpha1.PoolServiceList, servic } } -func newPoolService(namespace string, poolName string, annotations map[string]string, lbIngress []corev1.LoadBalancerIngress) *v1alpha1.PoolService { +func newPoolService(namespace string, poolName string, aggregatedLabels, aggregatedAnnotations map[string]string, lbIngress []corev1.LoadBalancerIngress) *v1alpha1.PoolService { blockOwnerDeletion := true controller := true return &v1alpha1.PoolService{ @@ -840,10 +842,9 @@ func newPoolService(namespace string, poolName string, annotations map[string]st APIVersion: v1alpha1.GroupVersion.String(), }, ObjectMeta: v1.ObjectMeta{ - Namespace: namespace, - Name: mockServiceName + "-" + poolName, - Labels: map[string]string{network.LabelServiceName: mockServiceName, network.LabelNodePoolName: poolName, labelManageBy: names.LoadBalancerSetController}, - Annotations: annotations, + Namespace: namespace, + Name: mockServiceName + "-" + poolName, + Labels: map[string]string{network.LabelServiceName: mockServiceName, network.LabelNodePoolName: poolName, labelManageBy: names.LoadBalancerSetController}, OwnerReferences: []v1.OwnerReference{ { APIVersion: "v1", @@ -869,6 +870,8 @@ func newPoolService(namespace string, poolName string, annotations map[string]st LoadBalancer: corev1.LoadBalancerStatus{ Ingress: lbIngress, }, + AggregateToAnnotations: aggregatedAnnotations, + AggregateToLabels: aggregatedLabels, }, } } @@ -892,11 +895,11 @@ func assertLoadBalancerStatusEqual(t testing.TB, expected, got []corev1.LoadBala } } -func assertOpenYurtAnnotationsEqual(t testing.TB, expected, got map[string]string) { +func assertMapEqual(t testing.TB, expected, got map[string]string) { t.Helper() for k, v := range expected { - if !strings.HasPrefix(k, network.AggregateAnnotationsKeyPrefix) || (k == network.AnnotationNodePoolSelector) { + if !strings.HasPrefix(k, aggregateKeyPrefix) { continue } if got[k] != v { @@ -905,7 +908,7 @@ func assertOpenYurtAnnotationsEqual(t testing.TB, expected, got map[string]strin } for k, v := range got { - if !strings.HasPrefix(k, network.AggregateAnnotationsKeyPrefix) || (k == network.AnnotationNodePoolSelector) { + if !strings.HasPrefix(k, aggregateKeyPrefix) { continue } diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/map_utils.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/map_utils.go new file mode 100644 index 00000000000..90de60f5a92 --- /dev/null +++ b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/map_utils.go @@ -0,0 +1,154 @@ +/* +Copyright 2024 The OpenYurt Authors. + +Licensed under the Apache License, Version 2.0 (the License); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an AS IS BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package loadbalancerset + +import ( + "sort" + "strings" +) + +const ( + aggregateKeyPrefix = "aggregated." +) + +func mergeMap(m map[string]string, elem map[string]string) map[string]string { + if len(elem) == 0 { + return m + } + + if m == nil { + m = make(map[string]string) + } + + for k, v := range elem { + m[k] = mergeMapValue(m[k], v) + } + + return m +} + +func mergeMapValue(originalValue, addValue string) string { + if len(originalValue) == 0 { + return addValue + } + + if len(addValue) == 0 { + return originalValue + } + + splitOriginalValues := strings.Split(originalValue, ",") + if valueIsExist(splitOriginalValues, addValue) { + return originalValue + } + + return joinNewValue(splitOriginalValues, addValue) +} + +func valueIsExist(originalValueList []string, addValue string) bool { + for _, oldValue := range originalValueList { + if addValue == oldValue { + return true + } + } + return false +} + +func joinNewValue(originalValueList []string, addValue string) string { + originalValueList = append(originalValueList, addValue) + sort.Strings(originalValueList) + + return strings.Join(originalValueList, ",") +} + +func addAggregatePrefix(m map[string]string) map[string]string { + if m == nil { + return nil + } + + newMap := make(map[string]string) + for key, value := range m { + newMap[aggregateKeyPrefix+key] = value + } + + return newMap +} + +func filterIgnoredKeys(m map[string]string) map[string]string { + newMap := make(map[string]string) + for key, value := range m { + if !strings.HasPrefix(key, aggregateKeyPrefix) { + continue + } + newMap[key] = value + } + return newMap +} + +func diffMap(currentMap, desiredMap map[string]string) (update map[string]string, deletion map[string]string) { + if currentMap == nil { + return desiredMap, nil + } + if desiredMap == nil { + return nil, currentMap + } + + update = make(map[string]string) + for key, value := range desiredMap { + if currentMap[key] != value { + update[key] = value + } + } + + deletion = make(map[string]string) + for key, value := range currentMap { + if _, exist := desiredMap[key]; !exist { + deletion[key] = value + } + } + return +} + +func updateMap(m, update, deletion map[string]string) map[string]string { + if len(update) == 0 && len(deletion) == 0 { + return m + } + if m == nil { + m = make(map[string]string) + } + for key, value := range update { + m[key] = value + } + + for key := range deletion { + delete(m, key) + } + return m +} + +func mapValueIsEqual(oldMap, newMap map[string]string, key string) bool { + var oldValue string + if oldMap != nil { + oldValue = oldMap[key] + } + + var newValue string + if newMap != nil { + newValue = newMap[key] + } + + return oldValue == newValue +} diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/predicate.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/predicate.go index 0c4f0d55152..fa7ffde0478 100644 --- a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/predicate.go +++ b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/predicate.go @@ -128,7 +128,7 @@ func isDeleteTimeChange(oldSvc, newSvc *v1.Service) bool { } func isNodePoolSelectorChange(oldAnnotations, newAnnotations map[string]string) bool { - return !annotationValueIsEqual(oldAnnotations, newAnnotations, network.AnnotationNodePoolSelector) + return !mapValueIsEqual(oldAnnotations, newAnnotations, network.AnnotationNodePoolSelector) } func isServiceTypeChange(oldSvc, newSvc *v1.Service) bool { diff --git a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/predicate_test.go b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/predicate_test.go index ffd6add52c6..6a9b2a8c890 100644 --- a/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/predicate_test.go +++ b/pkg/yurtmanager/controller/loadbalancerset/loadbalancerset/predicate_test.go @@ -28,7 +28,7 @@ import ( ) const ( - mockAnnotationLbId = network.AggregateAnnotationsKeyPrefix + "/lb-id" + mockAnnotationLbId = aggregateKeyPrefix + "lb-id" ) func TestServicePredicate(t *testing.T) { @@ -186,7 +186,7 @@ func TestPoolServicePredicated(t *testing.T) { f := NewPoolServicePredicated() t.Run("create/delete/update/generic pool service not managed and service name", func(t *testing.T) { - ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) delete(ps.Labels, labelManageBy) delete(ps.Labels, network.LabelServiceName) @@ -197,7 +197,7 @@ func TestPoolServicePredicated(t *testing.T) { }) t.Run("create/delete/update/generic pool service not managed", func(t *testing.T) { - ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) delete(ps.Labels, labelManageBy) assertBool(t, false, f.Create(event.CreateEvent{Object: ps})) @@ -207,7 +207,7 @@ func TestPoolServicePredicated(t *testing.T) { }) t.Run("create/delete/update/generic pool service not service name", func(t *testing.T) { - ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) delete(ps.Labels, network.LabelServiceName) assertBool(t, false, f.Create(event.CreateEvent{Object: ps})) @@ -218,7 +218,7 @@ func TestPoolServicePredicated(t *testing.T) { }) t.Run("create/delete/update/generic pool service", func(t *testing.T) { - ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) assertBool(t, true, f.Create(event.CreateEvent{Object: ps})) assertBool(t, true, f.Update(event.UpdateEvent{ObjectOld: ps, ObjectNew: ps})) @@ -228,8 +228,8 @@ func TestPoolServicePredicated(t *testing.T) { }) t.Run("create/delete/update/generic pool service not service name", func(t *testing.T) { - ps1 := newPoolService(v1.NamespaceDefault, "np123", nil, nil) - ps2 := newPoolService(v1.NamespaceDefault, "np123", nil, nil) + ps1 := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) + ps2 := newPoolService(v1.NamespaceDefault, "np123", nil, nil, nil) delete(ps2.Labels, network.LabelServiceName) delete(ps2.Labels, labelManageBy)