From 11c8260063aa114c6cb06713d288ef8b343d0ab9 Mon Sep 17 00:00:00 2001 From: Sina Siadat Date: Fri, 30 Aug 2024 10:43:15 -0700 Subject: [PATCH 1/7] Make VitessCell scalable by HPA Signed-off-by: Sina Siadat --- deploy/crds/planetscale.com_vitesscells.yaml | 10 ++++++++ docs/api/index.html | 24 +++++++++++++++++++ pkg/apis/planetscale/v2/vitesscell_types.go | 8 +++++++ pkg/controller/vitesscell/reconcile_vtgate.go | 4 ++++ 4 files changed, 46 insertions(+) diff --git a/deploy/crds/planetscale.com_vitesscells.yaml b/deploy/crds/planetscale.com_vitesscells.yaml index d81f76c3..dfed37fb 100644 --- a/deploy/crds/planetscale.com_vitesscells.yaml +++ b/deploy/crds/planetscale.com_vitesscells.yaml @@ -738,6 +738,12 @@ spec: properties: available: type: string + labelSelector: + type: string + replicas: + format: int32 + minimum: 0 + type: integer serviceName: type: string type: object @@ -768,4 +774,8 @@ spec: served: true storage: true subresources: + scale: + labelSelectorPath: .status.gateway.labelSelector + specReplicasPath: .spec.gateway.replicas + statusReplicasPath: .status.gateway.replicas status: {} diff --git a/docs/api/index.html b/docs/api/index.html index 41caa361..43797cd9 100644 --- a/docs/api/index.html +++ b/docs/api/index.html @@ -3614,6 +3614,30 @@

VitessCellGatewayStatus

ServiceName is the name of the Service for this cell’s vtgate.

+ + +labelSelector
+ +string + + + +

LabelSelector is required by the Scale subresource, which is used by +HorizontalPodAutoscaler when reading pod metrics.

+ + + + +replicas
+ +int32 + + + +

Replicas is required by the Scale subresource, which is used by +HorizontalPodAutoscaler to determine the current number of replicas.

+ +

VitessCellImages diff --git a/pkg/apis/planetscale/v2/vitesscell_types.go b/pkg/apis/planetscale/v2/vitesscell_types.go index d65645a6..eb2458ce 100644 --- a/pkg/apis/planetscale/v2/vitesscell_types.go +++ b/pkg/apis/planetscale/v2/vitesscell_types.go @@ -39,6 +39,7 @@ import ( // just like a Deployment can manage Pods that run on multiple Nodes. // +kubebuilder:resource:path=vitesscells,shortName=vtc // +kubebuilder:subresource:status +// +kubebuilder:subresource:scale:specpath=.spec.gateway.replicas,statuspath=.status.gateway.replicas,selectorpath=.status.gateway.labelSelector type VitessCell struct { metav1.TypeMeta `json:",inline"` metav1.ObjectMeta `json:"metadata,omitempty"` @@ -252,6 +253,13 @@ type VitessCellGatewayStatus struct { Available corev1.ConditionStatus `json:"available,omitempty"` // ServiceName is the name of the Service for this cell's vtgate. ServiceName string `json:"serviceName,omitempty"` + // LabelSelector is required by the Scale subresource, which is used by + // HorizontalPodAutoscaler when reading pod metrics. + LabelSelector string `json:"labelSelector,omitempty"` + // Replicas is required by the Scale subresource, which is used by + // HorizontalPodAutoscaler to determine the current number of replicas. + // +kubebuilder:validation:Minimum=0 + Replicas int32 `json:"replicas,omitempty"` } // VitessCellStatus defines the observed state of VitessCell diff --git a/pkg/controller/vitesscell/reconcile_vtgate.go b/pkg/controller/vitesscell/reconcile_vtgate.go index 136e5055..40e54ee4 100644 --- a/pkg/controller/vitesscell/reconcile_vtgate.go +++ b/pkg/controller/vitesscell/reconcile_vtgate.go @@ -161,6 +161,10 @@ func (r *ReconcileVitessCell) reconcileVtgate(ctx context.Context, vtc *planetsc curObj := obj.(*appsv1.Deployment) status := &vtc.Status.Gateway + if replicas := curObj.Spec.Replicas; replicas != nil { + status.Replicas = *replicas + } + status.LabelSelector = curObj.Spec.Selector.String() if available := conditions.Deployment(curObj.Status.Conditions, appsv1.DeploymentAvailable); available != nil { status.Available = available.Status } From aa5c2047a2dfb723d83b79e39a86857bfe460858 Mon Sep 17 00:00:00 2001 From: Sina Siadat Date: Tue, 10 Sep 2024 23:15:50 +0000 Subject: [PATCH 2/7] Autoscale vtgates with HPA Signed-off-by: Sina Siadat --- deploy/crds/planetscale.com_vitesscells.yaml | 328 ++++++++++++++++++ .../crds/planetscale.com_vitessclusters.yaml | 328 ++++++++++++++++++ docs/api/index.html | 87 +++++ pkg/apis/planetscale/v2/vitesscell_types.go | 28 ++ .../planetscale/v2/zz_generated.deepcopy.go | 43 +++ pkg/controller/vitesscell/reconcile_vtgate.go | 31 ++ .../vtgate/horizontal_pod_autoscaler.go | 68 ++++ .../vitesscluster/vitesscluster_test.go | 13 + 8 files changed, 926 insertions(+) create mode 100644 pkg/operator/vtgate/horizontal_pod_autoscaler.go diff --git a/deploy/crds/planetscale.com_vitesscells.yaml b/deploy/crds/planetscale.com_vitesscells.yaml index dfed37fb..5e66e58c 100644 --- a/deploy/crds/planetscale.com_vitesscells.yaml +++ b/deploy/crds/planetscale.com_vitesscells.yaml @@ -62,6 +62,334 @@ spec: type: object type: object type: object + autoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + minimum: 0 + type: integer + metrics: + items: + properties: + containerResource: + properties: + container: + type: string + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + minimum: 0 + type: integer + type: object extraEnv: items: properties: diff --git a/deploy/crds/planetscale.com_vitessclusters.yaml b/deploy/crds/planetscale.com_vitessclusters.yaml index 818304f1..690b7ad2 100644 --- a/deploy/crds/planetscale.com_vitessclusters.yaml +++ b/deploy/crds/planetscale.com_vitessclusters.yaml @@ -296,6 +296,334 @@ spec: type: object type: object type: object + autoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + minimum: 0 + type: integer + metrics: + items: + properties: + containerResource: + properties: + container: + type: string + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + minimum: 0 + type: integer + type: object extraEnv: items: properties: diff --git a/docs/api/index.html b/docs/api/index.html index 43797cd9..f9c3bad1 100644 --- a/docs/api/index.html +++ b/docs/api/index.html @@ -506,6 +506,78 @@

VitessCluster +

AutoscalerSpec +

+

+(Appears on: +VitessCellGatewaySpec) +

+

+

AutoscalerSpec defines the vtgate’s pod autoscaling specification.

+

+ + + + + + + + + + + + + + + + + + + + + + + + + +
FieldDescription
+minReplicas
+ +int32 + +		 +

MinReplicas is the minimum number of instances of vtgate to run in +this cell when autoscaling is enabled.

+
+maxReplicas
+ +int32 + +		 +

MaxReplicas is the maximum number of instances of vtgate to run in +this cell when autoscaling is enabled.

+
+behavior
+ + +Kubernetes autoscaling/v2.HorizontalPodAutoscalerBehavior + + +		 +(Optional) +
+metrics
+ + +[]Kubernetes autoscaling/v2.MetricSpec + + +		 +(Optional) +

Metrics is meant to provide a customizable way to configure HPA metrics. +currently the only supported custom metrics is type=Pod. +Use TargetCPUUtilization or TargetMemoryUtilization instead if scaling on these common resource metrics.

+

AzblobBackupLocation

@@ -3337,6 +3409,21 @@

VitessCellGatewaySpec +autoscaler
+ + +AutoscalerSpec + + + + +(Optional) +

Autoscaler specifies the pod autoscaling configuration to use +for the vtgate workload.

+ + + + resources
diff --git a/pkg/apis/planetscale/v2/vitesscell_types.go b/pkg/apis/planetscale/v2/vitesscell_types.go index eb2458ce..0e911e77 100644 --- a/pkg/apis/planetscale/v2/vitesscell_types.go +++ b/pkg/apis/planetscale/v2/vitesscell_types.go @@ -17,6 +17,7 @@ limitations under the License. package v2 import ( + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) @@ -118,12 +119,39 @@ type VitessCellImages struct { Vtgate string `json:"vtgate,omitempty"` } +// AutoscalerSpec defines the vtgate's pod autoscaling specification. +type AutoscalerSpec struct { + // MinReplicas is the minimum number of instances of vtgate to run in + // this cell when autoscaling is enabled. + // +kubebuilder:validation:Minimum=0 + MinReplicas *int32 `json:"minReplicas,omitempty"` + + // MaxReplicas is the maximum number of instances of vtgate to run in + // this cell when autoscaling is enabled. + // +kubebuilder:validation:Minimum=0 + MaxReplicas *int32 `json:"maxReplicas,omitempty"` + + // +optional + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` + + // Metrics is meant to provide a customizable way to configure HPA metrics. + // currently the only supported custom metrics is type=Pod. + // Use TargetCPUUtilization or TargetMemoryUtilization instead if scaling on these common resource metrics. + // +optional + Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` +} + // VitessCellGatewaySpec specifies the per-cell deployment parameters for vtgate. type VitessCellGatewaySpec struct { // Replicas is the number of vtgate instances to deploy in this cell. // +kubebuilder:validation:Minimum=0 Replicas *int32 `json:"replicas,omitempty"` + // Autoscaler specifies the pod autoscaling configuration to use + // for the vtgate workload. + // +optional + Autoscaler *AutoscalerSpec `json:"autoscaler,omitempty"` + // Resources determines the compute resources reserved for each vtgate replica. Resources corev1.ResourceRequirements `json:"resources,omitempty"` diff --git a/pkg/apis/planetscale/v2/zz_generated.deepcopy.go b/pkg/apis/planetscale/v2/zz_generated.deepcopy.go index 4db30177..a6636bd4 100644 --- a/pkg/apis/planetscale/v2/zz_generated.deepcopy.go +++ b/pkg/apis/planetscale/v2/zz_generated.deepcopy.go @@ -6,10 +6,48 @@ package v2 import ( + autoscalingv2 "k8s.io/api/autoscaling/v2" "k8s.io/api/core/v1" runtime "k8s.io/apimachinery/pkg/runtime" ) +// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. +func (in *AutoscalerSpec) DeepCopyInto(out *AutoscalerSpec) { + *out = *in + if in.MinReplicas != nil { + in, out := &in.MinReplicas, &out.MinReplicas + *out = new(int32) + **out = **in + } + if in.MaxReplicas != nil { + in, out := &in.MaxReplicas, &out.MaxReplicas + *out = new(int32) + **out = **in + } + if in.Behavior != nil { + in, out := &in.Behavior, &out.Behavior + *out = new(autoscalingv2.HorizontalPodAutoscalerBehavior) + (*in).DeepCopyInto(*out) + } + if in.Metrics != nil { + in, out := &in.Metrics, &out.Metrics + *out = make([]autoscalingv2.MetricSpec, len(*in)) + for i := range *in { + (*in)[i].DeepCopyInto(&(*out)[i]) + } + } +} + +// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new AutoscalerSpec. +func (in *AutoscalerSpec) DeepCopy() *AutoscalerSpec { + if in == nil { + return nil + } + out := new(AutoscalerSpec) + in.DeepCopyInto(out) + return out +} + // DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil. func (in *AzblobBackupLocation) DeepCopyInto(out *AzblobBackupLocation) { *out = *in @@ -1097,6 +1135,11 @@ func (in *VitessCellGatewaySpec) DeepCopyInto(out *VitessCellGatewaySpec) { *out = new(int32) **out = **in } + if in.Autoscaler != nil { + in, out := &in.Autoscaler, &out.Autoscaler + *out = new(AutoscalerSpec) + (*in).DeepCopyInto(*out) + } in.Resources.DeepCopyInto(&out.Resources) in.Authentication.DeepCopyInto(&out.Authentication) if in.SecureTransport != nil { diff --git a/pkg/controller/vitesscell/reconcile_vtgate.go b/pkg/controller/vitesscell/reconcile_vtgate.go index 40e54ee4..be18da44 100644 --- a/pkg/controller/vitesscell/reconcile_vtgate.go +++ b/pkg/controller/vitesscell/reconcile_vtgate.go @@ -20,6 +20,7 @@ import ( "context" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" "k8s.io/apimachinery/pkg/runtime" apitypes "k8s.io/apimachinery/pkg/types" @@ -174,5 +175,35 @@ func (r *ReconcileVitessCell) reconcileVtgate(ctx context.Context, vtc *planetsc resultBuilder.Error(err) } + var wantHpa bool + var hpaSpec *vtgate.HpaSpec + + if vtc.Spec.Gateway.Autoscaler != nil { + wantHpa = vtc.Spec.Gateway.Autoscaler.MaxReplicas != nil + hpaSpec = &vtgate.HpaSpec{ + Labels: labels, + MinReplicas: vtc.Spec.Gateway.Autoscaler.MinReplicas, + MaxReplicas: vtc.Spec.Gateway.Autoscaler.MaxReplicas, + Behavior: vtc.Spec.Gateway.Autoscaler.Behavior, + Metrics: vtc.Spec.Gateway.Autoscaler.Metrics, + } + } + + // Reconcile vtgate HorizontalPodAutoscaler. + err = r.reconciler.ReconcileObject(ctx, vtc, key, labels, wantHpa, reconciler.Strategy{ + Kind: &autoscalingv2.HorizontalPodAutoscaler{}, + + New: func(key client.ObjectKey) runtime.Object { + return vtgate.NewHorizontalPodAutoscaler(key, hpaSpec) + }, + UpdateInPlace: func(key client.ObjectKey, obj runtime.Object) { + newObj := obj.(*autoscalingv2.HorizontalPodAutoscaler) + vtgate.UpdateHorizontalPodAutoscaler(newObj, hpaSpec) + }, + }) + if err != nil { + resultBuilder.Error(err) + } + return resultBuilder.Result() } diff --git a/pkg/operator/vtgate/horizontal_pod_autoscaler.go b/pkg/operator/vtgate/horizontal_pod_autoscaler.go new file mode 100644 index 00000000..4b7abb3e --- /dev/null +++ b/pkg/operator/vtgate/horizontal_pod_autoscaler.go @@ -0,0 +1,68 @@ +/* +Copyright 2019 PlanetScale Inc. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + +package vtgate + +import ( + autoscalingv2 "k8s.io/api/autoscaling/v2" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" + "sigs.k8s.io/controller-runtime/pkg/client" + + "planetscale.dev/vitess-operator/pkg/operator/update" +) + +// Spec specifies all the internal parameters needed to deploy vtgate, +// as opposed to the API type planetscalev2.VitessCellGatewaySpec, which is the public API. +type HpaSpec struct { + Labels map[string]string + MinReplicas *int32 + MaxReplicas *int32 + Behavior *autoscalingv2.HorizontalPodAutoscalerBehavior `json:"behavior,omitempty"` + Metrics []autoscalingv2.MetricSpec `json:"metrics,omitempty"` +} + +// NewHorizontalPodAutoscaler creates a new HorizontalPodAutoscaler object for vtgate. +func NewHorizontalPodAutoscaler(key client.ObjectKey, spec *HpaSpec) *autoscalingv2.HorizontalPodAutoscaler { + // Fill in the immutable parts. + obj := &autoscalingv2.HorizontalPodAutoscaler{ + ObjectMeta: metav1.ObjectMeta{ + Namespace: key.Namespace, + Name: key.Name, + }, + Spec: autoscalingv2.HorizontalPodAutoscalerSpec{ + ScaleTargetRef: autoscalingv2.CrossVersionObjectReference{ + APIVersion: "planetscale.com/v2", + Kind: "VitessCell", + Name: key.Name, + }, + }, + } + // Set everything else. + UpdateHorizontalPodAutoscaler(obj, spec) + return obj +} + +// UpdateHorizontalPodAutoscaler updates the mutable parts of the vtgate HorizontalPodAutoscaler. +func UpdateHorizontalPodAutoscaler(obj *autoscalingv2.HorizontalPodAutoscaler, spec *HpaSpec) { + // Set labels on the HorizontalPodAutoscaler object. + update.Labels(&obj.Labels, spec.Labels) + + // Set the specs for the HorizontalPodAutoscaler object. + obj.Spec.MinReplicas = spec.MinReplicas + obj.Spec.MaxReplicas = *spec.MaxReplicas + obj.Spec.Metrics = spec.Metrics + obj.Spec.Behavior = spec.Behavior +} diff --git a/test/integration/vitesscluster/vitesscluster_test.go b/test/integration/vitesscluster/vitesscluster_test.go index b7f2440d..41390acd 100644 --- a/test/integration/vitesscluster/vitesscluster_test.go +++ b/test/integration/vitesscluster/vitesscluster_test.go @@ -21,6 +21,7 @@ import ( "testing" appsv1 "k8s.io/api/apps/v1" + autoscalingv2 "k8s.io/api/autoscaling/v2" corev1 "k8s.io/api/core/v1" apilabels "k8s.io/apimachinery/pkg/labels" "sigs.k8s.io/controller-runtime/pkg/client" @@ -38,6 +39,17 @@ spec: - name: cell1 - name: cell2 - name: cell3 + gateway: + autoscaler: + minReplicas: 1 + maxReplicas: 2 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 keyspaces: - name: keyspace1 partitionings: @@ -134,6 +146,7 @@ func verifyBasicVitessCluster(f *framework.Fixture, ns, cluster string) { verifyBasicVitessCell(f, ns, cluster, "cell1") verifyBasicVitessCell(f, ns, cluster, "cell2") verifyBasicVitessCell(f, ns, cluster, "cell3") + f.MustGet(ns, names.JoinWithConstraints(names.DefaultConstraints, cluster, "cell3", "vtgate"), &autoscalingv2.HorizontalPodAutoscaler{}) // VitessCluster creates VitessKeyspaces. verifyBasicVitessKeyspace(f, ns, cluster, "keyspace1") From 6816227fba647a4c2e350b262092e7f2054b6274 Mon Sep 17 00:00:00 2001 From: Sina Siadat Date: Thu, 12 Sep 2024 13:50:33 +0000 Subject: [PATCH 3/7] Only reconcile replicas if autoscaling is disabled Signed-off-by: Sina Siadat --- pkg/controller/vitesscluster/reconcile_cells.go | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/pkg/controller/vitesscluster/reconcile_cells.go b/pkg/controller/vitesscluster/reconcile_cells.go index 0c8266f5..d7bfaa15 100644 --- a/pkg/controller/vitesscluster/reconcile_cells.go +++ b/pkg/controller/vitesscluster/reconcile_cells.go @@ -152,9 +152,12 @@ func updateVitessCellInPlace(key client.ObjectKey, vtc *planetscalev2.VitessCell // Update labels, but ignore existing ones we don't set. update.Labels(&vtc.Labels, newCell.Labels) - // We allow immediate update of replica counts for stateless workloads, - // like Deployment does. - vtc.Spec.Gateway.Replicas = newCell.Spec.Gateway.Replicas + // Only update replicas if autoscaling is disabled. + if vtc.Spec.Gateway.Autoscaler != nil && vtc.Spec.Gateway.Autoscaler.MaxReplicas != nil { + // We allow immediate update of replica counts for stateless workloads, + // like Deployment does. + vtc.Spec.Gateway.Replicas = newCell.Spec.Gateway.Replicas + } } func updateVitessCell(key client.ObjectKey, vtc *planetscalev2.VitessCell, vt *planetscalev2.VitessCluster, parentLabels map[string]string, cell *planetscalev2.VitessCellTemplate) { From 9fe97262ede0d91d699b6dc18b9fc938ea8b654d Mon Sep 17 00:00:00 2001 From: Sina Siadat Date: Wed, 18 Sep 2024 23:12:05 +0000 Subject: [PATCH 4/7] Update end-to-end test YAML files Signed-off-by: Sina Siadat --- deploy/role.yaml | 8 +- test/endtoend/operator/cluster_autoscale.yaml | 103 +++ test/endtoend/operator/operator-latest.yaml | 672 ++++++++++++++++++ 3 files changed, 782 insertions(+), 1 deletion(-) create mode 100644 test/endtoend/operator/cluster_autoscale.yaml diff --git a/deploy/role.yaml b/deploy/role.yaml index d482ebe6..239fc0f1 100644 --- a/deploy/role.yaml +++ b/deploy/role.yaml @@ -78,4 +78,10 @@ rules: resources: - jobs verbs: - - '*' \ No newline at end of file + - '*' +- apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - '*' diff --git a/test/endtoend/operator/cluster_autoscale.yaml b/test/endtoend/operator/cluster_autoscale.yaml new file mode 100644 index 00000000..53bac5ba --- /dev/null +++ b/test/endtoend/operator/cluster_autoscale.yaml @@ -0,0 +1,103 @@ +# The following example is minimalist. The security policies +# and resource specifications are not meant to be used in production. +# Please refer to the operator documentation for recommendations on +# production settings. +apiVersion: planetscale.com/v2 +kind: VitessCluster +metadata: + name: example +spec: + images: + vtctld: vitess/lite:v20.0.1 + vtgate: vitess/lite:v20.0.1 + vttablet: vitess/lite:v20.0.1 + vtorc: vitess/lite:v20.0.1 + vtbackup: vitess/lite:v20.0.1 + mysqld: + mysql80Compatible: mysql:8.0.30 + mysqldExporter: prom/mysqld-exporter:v0.11.0 + cells: + - name: zone1 + gateway: + autoscaler: + minReplicas: 2 + maxReplicas: 3 + metrics: + - type: Resource + resource: + name: cpu + target: + type: Utilization + averageUtilization: 80 + authentication: + static: + secret: + name: example-cluster-config + key: users.json + replicas: 1 + resources: + requests: + cpu: 100m + memory: 256Mi + limits: + memory: 256Mi + vitessDashboard: + cells: + - zone1 + extraFlags: + security_policy: read-only + replicas: 1 + resources: + limits: + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + + keyspaces: + - name: commerce + durabilityPolicy: semi_sync + turndownPolicy: Immediate + vitessOrchestrator: + resources: + limits: + memory: 128Mi + requests: + cpu: 100m + memory: 128Mi + extraFlags: + recovery-period-block-duration: 5s + partitionings: + - equal: + parts: 1 + shardTemplate: + databaseInitScriptSecret: + name: example-cluster-config + key: init_db.sql + tabletPools: + - cell: zone1 + type: replica + replicas: 3 + vttablet: + extraFlags: + db_charset: utf8mb4 + resources: + limits: + memory: 256Mi + requests: + cpu: 100m + memory: 256Mi + mysqld: + resources: + limits: + memory: 1024Mi + requests: + cpu: 100m + memory: 512Mi + dataVolumeClaimTemplate: + accessModes: ["ReadWriteOnce"] + resources: + requests: + storage: 10Gi + updateStrategy: + type: Immediate diff --git a/test/endtoend/operator/operator-latest.yaml b/test/endtoend/operator/operator-latest.yaml index b2e6122f..14fe5a84 100644 --- a/test/endtoend/operator/operator-latest.yaml +++ b/test/endtoend/operator/operator-latest.yaml @@ -779,6 +779,334 @@ spec: type: object type: object type: object + autoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + minimum: 0 + type: integer + metrics: + items: + properties: + containerResource: + properties: + container: + type: string + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + minimum: 0 + type: integer + type: object extraEnv: items: properties: @@ -1455,6 +1783,12 @@ spec: properties: available: type: string + labelSelector: + type: string + replicas: + format: int32 + minimum: 0 + type: integer serviceName: type: string type: object @@ -1485,6 +1819,10 @@ spec: served: true storage: true subresources: + scale: + labelSelectorPath: .status.gateway.labelSelector + specReplicasPath: .spec.gateway.replicas + statusReplicasPath: .status.gateway.replicas status: {} --- apiVersion: apiextensions.k8s.io/v1 @@ -1784,6 +2122,334 @@ spec: type: object type: object type: object + autoscaler: + properties: + behavior: + properties: + scaleDown: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + scaleUp: + properties: + policies: + items: + properties: + periodSeconds: + format: int32 + type: integer + type: + type: string + value: + format: int32 + type: integer + required: + - periodSeconds + - type + - value + type: object + type: array + x-kubernetes-list-type: atomic + selectPolicy: + type: string + stabilizationWindowSeconds: + format: int32 + type: integer + type: object + type: object + maxReplicas: + format: int32 + minimum: 0 + type: integer + metrics: + items: + properties: + containerResource: + properties: + container: + type: string + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - container + - name + - target + type: object + external: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + object: + properties: + describedObject: + properties: + apiVersion: + type: string + kind: + type: string + name: + type: string + required: + - kind + - name + type: object + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - describedObject + - metric + - target + type: object + pods: + properties: + metric: + properties: + name: + type: string + selector: + properties: + matchExpressions: + items: + properties: + key: + type: string + operator: + type: string + values: + items: + type: string + type: array + required: + - key + - operator + type: object + type: array + matchLabels: + additionalProperties: + type: string + type: object + type: object + x-kubernetes-map-type: atomic + required: + - name + type: object + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - metric + - target + type: object + resource: + properties: + name: + type: string + target: + properties: + averageUtilization: + format: int32 + type: integer + averageValue: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + type: + type: string + value: + anyOf: + - type: integer + - type: string + pattern: ^(\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))(([KMGTPE]i)|[numkMGTPE]|([eE](\+|-)?(([0-9]+(\.[0-9]*)?)|(\.[0-9]+))))?$ + x-kubernetes-int-or-string: true + required: + - type + type: object + required: + - name + - target + type: object + type: + type: string + required: + - type + type: object + type: array + minReplicas: + format: int32 + minimum: 0 + type: integer + type: object extraEnv: items: properties: @@ -6817,6 +7483,12 @@ rules: - jobs verbs: - '*' + - apiGroups: + - autoscaling + resources: + - horizontalpodautoscalers + verbs: + - '*' --- apiVersion: rbac.authorization.k8s.io/v1 kind: RoleBinding From ad19f55dee5ecac5f0e23f16d682b27eddd35120 Mon Sep 17 00:00:00 2001 From: Sina Siadat Date: Wed, 18 Sep 2024 23:13:30 +0000 Subject: [PATCH 5/7] Add HPA to watchResources in the vitesscell pkg Signed-off-by: Sina Siadat --- pkg/controller/vitesscell/vitesscell_controller.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/pkg/controller/vitesscell/vitesscell_controller.go b/pkg/controller/vitesscell/vitesscell_controller.go index 025f7ef9..cd64aeb7 100644 --- a/pkg/controller/vitesscell/vitesscell_controller.go +++ b/pkg/controller/vitesscell/vitesscell_controller.go @@ -37,6 +37,7 @@ import ( "sigs.k8s.io/controller-runtime/pkg/reconcile" "sigs.k8s.io/controller-runtime/pkg/source" + autoscalingv2 "k8s.io/api/autoscaling/v2" planetscalev2 "planetscale.dev/vitess-operator/pkg/apis/planetscale/v2" "planetscale.dev/vitess-operator/pkg/operator/environment" "planetscale.dev/vitess-operator/pkg/operator/metrics" @@ -60,6 +61,7 @@ var log = logrus.WithField("controller", "VitessCell") var watchResources = []client.Object{ &corev1.Service{}, &appsv1.Deployment{}, + &autoscalingv2.HorizontalPodAutoscaler{}, &planetscalev2.EtcdLockserver{}, } From 8249554a0d2cb9d95641876fdda07b5a7157f1dd Mon Sep 17 00:00:00 2001 From: Sina Siadat Date: Wed, 18 Sep 2024 23:14:11 +0000 Subject: [PATCH 6/7] Update comments Signed-off-by: Sina Siadat --- pkg/operator/vtgate/horizontal_pod_autoscaler.go | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pkg/operator/vtgate/horizontal_pod_autoscaler.go b/pkg/operator/vtgate/horizontal_pod_autoscaler.go index 4b7abb3e..6560d566 100644 --- a/pkg/operator/vtgate/horizontal_pod_autoscaler.go +++ b/pkg/operator/vtgate/horizontal_pod_autoscaler.go @@ -1,5 +1,5 @@ /* -Copyright 2019 PlanetScale Inc. +Copyright 2024 PlanetScale Inc. Licensed under the Apache License, Version 2.0 (the "License"); you may not use this file except in compliance with the License. @@ -24,8 +24,8 @@ import ( "planetscale.dev/vitess-operator/pkg/operator/update" ) -// Spec specifies all the internal parameters needed to deploy vtgate, -// as opposed to the API type planetscalev2.VitessCellGatewaySpec, which is the public API. +// HpaSpec specifies all the internal parameters needed to create a HorizontalPodAutoscaler +// for vtgate. type HpaSpec struct { Labels map[string]string MinReplicas *int32 From 52d64bd875170eec433a2587d5e34b18648f6d42 Mon Sep 17 00:00:00 2001 From: Sina Siadat Date: Wed, 18 Sep 2024 23:14:42 +0000 Subject: [PATCH 7/7] Add endtoend test for HorizontalPodAutoscaler Signed-off-by: Sina Siadat --- Makefile | 4 ++++ test/endtoend/hpa_test.sh | 44 +++++++++++++++++++++++++++++++++++++++ 2 files changed, 48 insertions(+) create mode 100755 test/endtoend/hpa_test.sh diff --git a/Makefile b/Makefile index 36c6be63..12d3694f 100644 --- a/Makefile +++ b/Makefile @@ -77,3 +77,7 @@ vtorc-vtadmin-test: build e2e-test-setup unmanaged-tablet-test: build e2e-test-setup echo "Running Unmanaged Tablet test" test/endtoend/unmanaged_tablet_test.sh + +hpa-test: build e2e-test-setup + echo "Running HPA test" + test/endtoend/hpa_test.sh diff --git a/test/endtoend/hpa_test.sh b/test/endtoend/hpa_test.sh new file mode 100755 index 00000000..e66f7f62 --- /dev/null +++ b/test/endtoend/hpa_test.sh @@ -0,0 +1,44 @@ +#!/bin/bash + +source ./tools/test.env +source ./test/endtoend/utils.sh + +# Test setup +echo "Building the docker image" +docker build -f build/Dockerfile.release -t vitess-operator-pr:latest . +echo "Creating Kind cluster" +kind create cluster --wait 30s --name kind-${BUILDKITE_BUILD_ID} --image ${KIND_VERSION} +echo "Loading docker image into Kind cluster" +kind load docker-image vitess-operator-pr:latest --name kind-${BUILDKITE_BUILD_ID} + +cd "$PWD/test/endtoend/operator" +killall kubectl +setupKubectlAccessForCI + +echo "Apply latest operator-latest.yaml" +kubectl apply -f "operator-latest.yaml" +checkPodStatusWithTimeout "vitess-operator(.*)1/1(.*)Running(.*)" + +echo "Apply cluster_autoscale.yaml" +kubectl apply -f cluster_autoscale.yaml + +function verifyHpa() { + regex=$1 + for i in {1..600} ; do + out=$(kubectl get hpa --no-headers -o custom-columns=":metadata.name,:spec.maxReplicas,:spec.minReplicas,:spec.scaleTargetRef.name") + echo "$out" | grep -E "$regex" > /dev/null 2>&1 + if [[ $? -eq 0 ]]; then + echo "HorizontalPodAutoscaler $regex found" + return 0 + fi + sleep 1 + done + echo -e "ERROR: HorizontalPodAutoscaler $regex not found" + exit 1 +} + +verifyHpa "example-zone1-vtgate(.*)3\s+2\s+example-zone1-vtgate(.*)" + +# Teardown +echo "Deleting Kind cluster. This also deletes the volume associated with it" +kind delete cluster --name kind-${BUILDKITE_BUILD_ID}