From 6bb0903b46e60b668c143c53f6a76edc7b190566 Mon Sep 17 00:00:00 2001 From: yandongxiao Date: Tue, 27 Jun 2023 12:26:56 +0800 Subject: [PATCH] [feature] support to add annotations on k8s service Signed-off-by: yandongxiao --- .../starrocks.com_starrocksclusters.yaml | 15 ++++++++ deploy/starrocks.com_starrocksclusters.yaml | 12 ++++++ .../templates/starrocks/starrockscluster.yaml | 12 ++++++ helm-charts/charts/kube-starrocks/values.yaml | 14 +++++-- .../starrocks/v1/starrockscluster_types.go | 3 ++ .../starrocks/v1/zz_generated.deepcopy.go | 7 ++++ pkg/common/resource_utils/service.go | 20 ++++++++-- pkg/common/resource_utils/service_test.go | 38 +++++++++++++++++++ pkg/common/resource_utils/volume.go | 1 - 9 files changed, 114 insertions(+), 8 deletions(-) delete mode 100644 pkg/common/resource_utils/volume.go diff --git a/config/crd/bases/starrocks.com_starrocksclusters.yaml b/config/crd/bases/starrocks.com_starrocksclusters.yaml index 4b706aae..61b2ff0f 100644 --- a/config/crd/bases/starrocks.com_starrocksclusters.yaml +++ b/config/crd/bases/starrocks.com_starrocksclusters.yaml @@ -1177,6 +1177,11 @@ spec: description: Service defines the template for the associated Kubernetes Service object. the service for user access be. properties: + annotations: + additionalProperties: + type: string + description: Annotations store Kubernetes Service annotations. + type: object loadBalancerIP: description: 'Only applies to Service Type: LoadBalancer. This feature depends on whether the underlying cloud-provider @@ -3097,6 +3102,11 @@ spec: description: Service defines the template for the associated Kubernetes Service object. the service for user access cn. properties: + annotations: + additionalProperties: + type: string + description: Annotations store Kubernetes Service annotations. + type: object loadBalancerIP: description: 'Only applies to Service Type: LoadBalancer. This feature depends on whether the underlying cloud-provider @@ -4314,6 +4324,11 @@ spec: description: Service defines the template for the associated Kubernetes Service object. properties: + annotations: + additionalProperties: + type: string + description: Annotations store Kubernetes Service annotations. + type: object loadBalancerIP: description: 'Only applies to Service Type: LoadBalancer. This feature depends on whether the underlying cloud-provider diff --git a/deploy/starrocks.com_starrocksclusters.yaml b/deploy/starrocks.com_starrocksclusters.yaml index 5bcbad6e..108d433d 100644 --- a/deploy/starrocks.com_starrocksclusters.yaml +++ b/deploy/starrocks.com_starrocksclusters.yaml @@ -556,6 +556,10 @@ spec: type: array service: properties: + annotations: + additionalProperties: + type: string + type: object loadBalancerIP: type: string name: @@ -1461,6 +1465,10 @@ spec: type: array service: properties: + annotations: + additionalProperties: + type: string + type: object loadBalancerIP: type: string name: @@ -2020,6 +2028,10 @@ spec: type: array service: properties: + annotations: + additionalProperties: + type: string + type: object loadBalancerIP: type: string name: diff --git a/helm-charts/charts/kube-starrocks/templates/starrocks/starrockscluster.yaml b/helm-charts/charts/kube-starrocks/templates/starrocks/starrockscluster.yaml index 53859fec..4f554c54 100644 --- a/helm-charts/charts/kube-starrocks/templates/starrocks/starrockscluster.yaml +++ b/helm-charts/charts/kube-starrocks/templates/starrocks/starrockscluster.yaml @@ -23,6 +23,10 @@ spec: {{- if and (eq "LoadBalancer" .Values.starrocksFESpec.service.type) .Values.starrocksFESpec.service.loadbalancerIP }} loadBalancerIP: {{ .Values.starrocksFESpec.service.loadbalancerIP }} {{- end }} +{{- if .Values.starrocksFESpec.service.annotations }} + annotations: +{{ toYaml .Values.starrocksFESpec.service.annotations | indent 8 }} +{{- end }} {{- end }} {{- if .Values.starrocksFESpec.annotations }} annotations: @@ -108,6 +112,10 @@ spec: {{- if and (eq "LoadBalancer" .Values.starrocksBeSpec.service.type) .Values.starrocksBeSpec.service.loadbalancerIP }} loadBalancerIP: {{ .Values.starrocksBeSpec.service.loadbalancerIP }} {{- end }} +{{- if .Values.starrocksBeSpec.service.annotations }} + annotations: +{{ toYaml .Values.starrocksBeSpec.service.annotations | indent 8 }} +{{- end }} {{- end }} {{- if .Values.starrocksBeSpec.annotations }} annotations: @@ -228,6 +236,10 @@ spec: {{- if and (eq "LoadBalancer" .Values.starrocksCnSpec.service.type) .Values.starrocksCnSpec.service.loadbalancerIP }} loadBalancerIP: {{ .Values.starrocksCnSpec.service.loadbalancerIP }} {{- end }} +{{- if .Values.starrocksCnSpec.service.annotations }} + annotations: +{{ toYaml .Values.starrocksCnSpec.service.annotations | indent 8 }} +{{- end }} {{- end }} {{- if .Values.starrocksCnSpec.annotations }} annotations: diff --git a/helm-charts/charts/kube-starrocks/values.yaml b/helm-charts/charts/kube-starrocks/values.yaml index cd8b0fbd..0aa7d9f6 100644 --- a/helm-charts/charts/kube-starrocks/values.yaml +++ b/helm-charts/charts/kube-starrocks/values.yaml @@ -65,6 +65,8 @@ starrocksFESpec: type: "ClusterIP" # the loadBalancerIP for static ip config when the type=LoadBalancer and loadbalancerIp is not empty. loadbalancerIP: "" + # add annotations for fe service. + annotations: {} # contains the secret name imagePullSecrets: [] @@ -183,16 +185,18 @@ starrocksCnSpec: # specify the service name and port config and serviceType # the service type refer https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types service: - # the fe service type, only supported ClusterIP, NodePort, LoadBalancer, ExternalName + # the cn service type, only supported ClusterIP, NodePort, LoadBalancer, ExternalName type: "ClusterIP" # the loadBalancerIP for static ip config when the type=LoadBalancer and loadBalancerIp is not empty. loadbalancerIP: "" + # add annotations for cn service. + annotations: {} # contains the secret name imagePullSecrets: [] # - name: "image-pull-secret" - # serviceAccount for fe access cloud service. + # serviceAccount for cn access cloud service. serviceAccount: "" # If specified, the pod's nodeSelector，displayName="Map of nodeSelectors to match when scheduling pods on nodes" @@ -318,16 +322,18 @@ starrocksBeSpec: # specify the service name and port config and serviceType # the service type refer https://kubernetes.io/docs/concepts/services-networking/service/#publishing-services-service-types service: - # the fe service type, only supported ClusterIP, NodePort, LoadBalancer, ExternalName + # the be service type, only supported ClusterIP, NodePort, LoadBalancer, ExternalName type: "ClusterIP" # the loadbalancerIP for static ip config when the type=LoadBalancer and loadbalancerIp is not empty. loadbalancerIP: "" + # add annotations for be service. + annotations: {} # contains the secret name imagePullSecrets: [] # - name: "image-pull-secret" - # serviceAccount for fe access cloud service. + # serviceAccount for be access cloud service. serviceAccount: "" # If specified, the pod's nodeSelector，displayName="Map of nodeSelectors to match when scheduling pods on nodes" diff --git a/pkg/apis/starrocks/v1/starrockscluster_types.go b/pkg/apis/starrocks/v1/starrockscluster_types.go index c38d4236..0257a330 100644 --- a/pkg/apis/starrocks/v1/starrockscluster_types.go +++ b/pkg/apis/starrocks/v1/starrockscluster_types.go @@ -565,6 +565,9 @@ type StarRocksService struct { //Ports the components exposed ports and listen ports in pod. // +optional Ports []StarRocksServicePort `json:"ports"` + + // Annotations store Kubernetes Service annotations. + Annotations map[string]string `json:"annotations,omitempty"` } type StarRocksServicePort struct { diff --git a/pkg/apis/starrocks/v1/zz_generated.deepcopy.go b/pkg/apis/starrocks/v1/zz_generated.deepcopy.go index 5a58049b..e2fcc37a 100644 --- a/pkg/apis/starrocks/v1/zz_generated.deepcopy.go +++ b/pkg/apis/starrocks/v1/zz_generated.deepcopy.go @@ -947,6 +947,13 @@ func (in *StarRocksService) DeepCopyInto(out *StarRocksService) { *out = make([]StarRocksServicePort, len(*in)) copy(*out, *in) } + if in.Annotations != nil { + in, out := &in.Annotations, &out.Annotations + *out = make(map[string]string, len(*in)) + for key, val := range *in { + (*out)[key] = val + } + } } // DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new StarRocksService. diff --git a/pkg/common/resource_utils/service.go b/pkg/common/resource_utils/service.go index 1bc65aba..50d9f4aa 100644 --- a/pkg/common/resource_utils/service.go +++ b/pkg/common/resource_utils/service.go @@ -2,7 +2,6 @@ package resource_utils import ( srapi "github.com/StarRocks/starrocks-kubernetes-operator/pkg/apis/starrocks/v1" - "github.com/StarRocks/starrocks-kubernetes-operator/pkg/common/hash" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -44,11 +43,13 @@ func BuildExternalService(src *srapi.StarRocksCluster, name string, serviceType } svc.Finalizers = append(svc.Finalizers, srapi.SERVICE_FINALIZER) + anno := map[string]string{} if serviceType == FeService { if svc.Name == "" { svc.Name = src.Name + "-" + srapi.DEFAULT_FE } setServiceType(src.Spec.StarRocksFeSpec.Service, &svc) + anno = getServiceAnnotations(src.Spec.StarRocksFeSpec.Service) srPorts = getFeServicePorts(config) } else if serviceType == BeService { if svc.Name == "" { @@ -56,6 +57,7 @@ func BuildExternalService(src *srapi.StarRocksCluster, name string, serviceType } setServiceType(src.Spec.StarRocksBeSpec.Service, &svc) + anno = getServiceAnnotations(src.Spec.StarRocksBeSpec.Service) srPorts = getBeServicePorts(config) } else if serviceType == CnService { if svc.Name == "" { @@ -63,6 +65,7 @@ func BuildExternalService(src *srapi.StarRocksCluster, name string, serviceType } setServiceType(src.Spec.StarRocksCnSpec.Service, &svc) + anno = getServiceAnnotations(src.Spec.StarRocksCnSpec.Service) srPorts = getCnServicePorts(config) } @@ -84,12 +87,12 @@ func BuildExternalService(src *srapi.StarRocksCluster, name string, serviceType TargetPort: intstr.FromInt(int(sp.ContainerPort)), }) } + // set Ports field before calculate resource hash + svc.Spec.Ports = ports hso := serviceHashObject(&svc) - anno := map[string]string{} anno[srapi.ComponentResourceHash] = hash.HashObject(hso) svc.Annotations = anno - svc.Spec.Ports = ports return svc } @@ -158,6 +161,17 @@ func setServiceType(svc *srapi.StarRocksService, service *corev1.Service) { } } +func getServiceAnnotations(svc *srapi.StarRocksService) map[string]string { + if svc != nil && svc.Annotations != nil { + annotations := map[string]string{} + for key, val := range svc.Annotations { + annotations[key] = val + } + return annotations + } + return map[string]string{} +} + func ServiceDeepEqual(nsvc, oldsvc *corev1.Service) bool { var nhsvcValue, ohsvcValue string if _, ok := nsvc.Annotations[srapi.ComponentResourceHash]; ok { diff --git a/pkg/common/resource_utils/service_test.go b/pkg/common/resource_utils/service_test.go index e4d4bf78..b443ec81 100644 --- a/pkg/common/resource_utils/service_test.go +++ b/pkg/common/resource_utils/service_test.go @@ -5,6 +5,7 @@ import ( "github.com/stretchr/testify/require" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "reflect" "testing" ) @@ -27,3 +28,40 @@ func Test_BuildExternalService(t *testing.T) { svc := BuildExternalService(&src, "test", FeService, make(map[string]interface{}), make(map[string]string), make(map[string]string)) require.Equal(t, corev1.ServiceTypeLoadBalancer, svc.Spec.Type) } + +func Test_getServiceAnnotations(t *testing.T) { + type args struct { + svc *srapi.StarRocksService + } + tests := []struct { + name string + args args + want map[string]string + }{ + { + name: "empty service", + args: args{ + svc: &srapi.StarRocksService{}, + }, + want: map[string]string{}, + }, + { + name: "service with annotations", + args: args{ + svc: &srapi.StarRocksService{ + Annotations: map[string]string{ + "test": "test", + }, + }, + }, + want: map[string]string{"test": "test"}, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + if got := getServiceAnnotations(tt.args.svc); !reflect.DeepEqual(got, tt.want) { + t.Errorf("getServiceAnnotations() = %v, want %v", got, tt.want) + } + }) + } +} diff --git a/pkg/common/resource_utils/volume.go b/pkg/common/resource_utils/volume.go deleted file mode 100644 index a44179ca..00000000 --- a/pkg/common/resource_utils/volume.go +++ /dev/null @@ -1 +0,0 @@ -package resource_utils