Skip to content

Commit

Permalink
Merge pull request #437 from yikuaibro/vpa
Browse files Browse the repository at this point in the history
feat: add vpa for AutoScaler
  • Loading branch information
mlycore authored Jul 13, 2023
2 parents ef909ce + f9c8ae0 commit 64c54da
Show file tree
Hide file tree
Showing 6 changed files with 285 additions and 0 deletions.
45 changes: 45 additions & 0 deletions shardingsphere-operator/pkg/controllers/auto_scaler_controller.go
Original file line number Diff line number Diff line change
Expand Up @@ -32,6 +32,7 @@ import (
"k8s.io/apimachinery/pkg/runtime"
"k8s.io/apimachinery/pkg/runtime/schema"
"k8s.io/apimachinery/pkg/types"
autoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
ctrl "sigs.k8s.io/controller-runtime"
"sigs.k8s.io/controller-runtime/pkg/client"
)
Expand Down Expand Up @@ -91,6 +92,11 @@ func (r *AutoScalerReconciler) reconcileAutoScaler(ctx context.Context, as *v1al
return err
}
}
if pg.Provider == "KubernetesVPA" && pg.Vertical != nil {
if err := r.reconcileVPA(ctx, &as.ObjectMeta, gvk, &pg); err != nil {
return err
}
}
}

return nil
Expand All @@ -111,6 +117,7 @@ func (r *AutoScalerReconciler) getHPAByNamespacedName(ctx context.Context, names
return r.Resources.HPA().GetByNamespacedName(ctx, namespacedName)
}

// nolint:dupl
func (r *AutoScalerReconciler) updateHPA(ctx context.Context, meta *metav1.ObjectMeta, gvk schema.GroupVersionKind, policy *v1alpha1.ScalingPolicy, hpa *autoscalingv2beta2.HorizontalPodAutoscaler) error {
exp := r.Builder.BuildHorizontalPodAutoScaler(ctx, meta, gvk, policy)
exp.ObjectMeta = hpa.ObjectMeta
Expand All @@ -131,3 +138,41 @@ func (r *AutoScalerReconciler) createHPA(ctx context.Context, meta *metav1.Objec
}
return err
}

func (r *AutoScalerReconciler) reconcileVPA(ctx context.Context, meta *metav1.ObjectMeta, gvk schema.GroupVersionKind, policy *v1alpha1.ScalingPolicy) error {
vpa, err := r.getVPAByNamespacedName(ctx, types.NamespacedName{Namespace: meta.Namespace, Name: meta.Name})
if err != nil {
return err
}
if vpa != nil {
return r.updateVPA(ctx, meta, gvk, policy, vpa)
}
return r.createVPA(ctx, meta, gvk, policy)
}

func (r *AutoScalerReconciler) getVPAByNamespacedName(ctx context.Context, namespacedName types.NamespacedName) (*autoscalingv1.VerticalPodAutoscaler, error) {
return r.Resources.VPA().GetByNamespacedName(ctx, namespacedName)
}

// nolint:dupl
func (r *AutoScalerReconciler) updateVPA(ctx context.Context, meta *metav1.ObjectMeta, gvk schema.GroupVersionKind, policy *v1alpha1.ScalingPolicy, vpa *autoscalingv1.VerticalPodAutoscaler) error {
exp := r.Builder.BuildVerticalPodAutoscaler(ctx, meta, gvk, policy)
exp.ObjectMeta = vpa.ObjectMeta
exp.Labels = vpa.Labels
exp.Annotations = vpa.Annotations

if !reflect.DeepEqual(vpa.Spec, exp.Spec) {
return r.Resources.VPA().Update(ctx, vpa)
}
return nil
}

// nolint:dupl
func (r *AutoScalerReconciler) createVPA(ctx context.Context, meta *metav1.ObjectMeta, gvk schema.GroupVersionKind, policy *v1alpha1.ScalingPolicy) error {
vpa := r.Builder.BuildVerticalPodAutoscaler(ctx, meta, gvk, policy)
err := r.Resources.VPA().Create(ctx, vpa)
if err != nil && apierrors.IsAlreadyExists(err) || err == nil {
return nil
}
return err
}
9 changes: 9 additions & 0 deletions shardingsphere-operator/pkg/kubernetes/resources.go
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"github.com/apache/shardingsphere-on-cloud/shardingsphere-operator/pkg/kubernetes/deployment"
"github.com/apache/shardingsphere-on-cloud/shardingsphere-operator/pkg/kubernetes/hpa"
"github.com/apache/shardingsphere-on-cloud/shardingsphere-operator/pkg/kubernetes/service"
"github.com/apache/shardingsphere-on-cloud/shardingsphere-operator/pkg/kubernetes/vpa"

"sigs.k8s.io/controller-runtime/pkg/client"
)
Expand All @@ -40,6 +41,7 @@ type KubernetesResources interface {
Service() service.Service
ConfigMap() configmap.ConfigMap
HPA() hpa.HorizontalPodAutoscaler
VPA() vpa.VerticalPodAutoscaler
// Job()
}

Expand All @@ -57,6 +59,7 @@ func NewResources(c client.Client) Resources {
service: service.NewServiceClient(c),
configmap: configmap.NewConfigMapClient(c),
hpa: hpa.NewHorizontalPodAutoscalerClient(c),
vpa: vpa.NewVerticalPodAutoscalerClient(c),
},
ExtendedResources: &extended{
chaosmesh: chaosmesh.NewChaos(c),
Expand All @@ -77,6 +80,7 @@ type kubernetes struct {
service service.Service
configmap configmap.ConfigMap
hpa hpa.HorizontalPodAutoscaler
vpa vpa.VerticalPodAutoscaler
}

// Deployment returns a Kubernetes deployment
Expand All @@ -99,6 +103,11 @@ func (r *kubernetes) HPA() hpa.HorizontalPodAutoscaler {
return r.hpa
}

// VPA returns a Kubernetes VPA
func (r *kubernetes) VPA() vpa.VerticalPodAutoscaler {
return r.vpa
}

var _ ExtendedResources = &extended{}

type extended struct {
Expand Down
76 changes: 76 additions & 0 deletions shardingsphere-operator/pkg/kubernetes/vpa/builder.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,76 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package vpa

import (
"github.com/apache/shardingsphere-on-cloud/shardingsphere-operator/pkg/kubernetes/metadata"
v1 "k8s.io/api/autoscaling/v1"
autoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
)

// VerticalPodAutoscalerBuilder is a builder for VPA
type VerticalPodAutoscalerBuilder interface {
metadata.MetadataBuilder

SetTargetRef(ref v1.CrossVersionObjectReference) VerticalPodAutoscalerBuilder
SetUpdatePolicy(pu *autoscalingv1.PodUpdatePolicy) VerticalPodAutoscalerBuilder
SetResourcePolicy(pr *autoscalingv1.PodResourcePolicy) VerticalPodAutoscalerBuilder
SetRecommenders(rs []*autoscalingv1.VerticalPodAutoscalerRecommenderSelector) VerticalPodAutoscalerBuilder

BuildVPA() *autoscalingv1.VerticalPodAutoscaler
}

// NewVerticalPodAutoscalerBuilder returns a VerticalPodAutoscalerBuilder for VPA
func NewVerticalPodAutoscalerBuilder() VerticalPodAutoscalerBuilder {
return &vpaBuilder{}
}

type vpaBuilder struct {
vpa *autoscalingv1.VerticalPodAutoscaler
metadata.MetadataBuilder
}

// SetTargetRef set the scale target
func (v *vpaBuilder) SetTargetRef(ref v1.CrossVersionObjectReference) VerticalPodAutoscalerBuilder {
v.vpa.Spec.TargetRef = &ref
return v
}

// SetUpdatePolicy set the rules on how changes are applied to the pods
func (v *vpaBuilder) SetUpdatePolicy(pu *autoscalingv1.PodUpdatePolicy) VerticalPodAutoscalerBuilder {
v.vpa.Spec.UpdatePolicy = pu
return v
}

// SetResourcePolicy set how the autoscaler computes recommended resources
func (v *vpaBuilder) SetResourcePolicy(pr *autoscalingv1.PodResourcePolicy) VerticalPodAutoscalerBuilder {
v.vpa.Spec.ResourcePolicy = pr
return v
}

// SetRecommenders set the recommenders
func (v *vpaBuilder) SetRecommenders(rs []*autoscalingv1.VerticalPodAutoscalerRecommenderSelector) VerticalPodAutoscalerBuilder {
v.vpa.Spec.Recommenders = rs
return v
}

// BuildVPA returns a VPA
func (v *vpaBuilder) BuildVPA() *autoscalingv1.VerticalPodAutoscaler {
v.vpa.ObjectMeta = *v.BuildMetadata()
return v.vpa
}
92 changes: 92 additions & 0 deletions shardingsphere-operator/pkg/kubernetes/vpa/vpa.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,92 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package vpa

import (
"context"

apierrors "k8s.io/apimachinery/pkg/api/errors"
"k8s.io/apimachinery/pkg/types"
autoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
"sigs.k8s.io/controller-runtime/pkg/client"
)

// NewVerticalPodAutoscalerClient creates a new VerticalPodAutoscaler
func NewVerticalPodAutoscalerClient(c client.Client) VerticalPodAutoscaler {
return vpaClient{
getter: getter{
Client: c,
},
setter: setter{
Client: c,
},
}
}

// VerticalPodAutoscaler interface contains setter and getter
type VerticalPodAutoscaler interface {
Getter
Setter
}

type vpaClient struct {
getter
setter
}

// Getter get VerticalPodAutoscaler from different parameters
type Getter interface {
GetByNamespacedName(context.Context, types.NamespacedName) (*autoscalingv1.VerticalPodAutoscaler, error)
}

type getter struct {
client.Client
}

// GetByNamespacedName returns Deployment from given namespaced name
func (dg getter) GetByNamespacedName(ctx context.Context, namespacedName types.NamespacedName) (*autoscalingv1.VerticalPodAutoscaler, error) {
hpa := &autoscalingv1.VerticalPodAutoscaler{}
if err := dg.Client.Get(ctx, namespacedName, hpa); err != nil {
if apierrors.IsNotFound(err) {
return nil, nil
}
return nil, err
} else {
return hpa, nil
}
}

// Setter get VerticalPodAutoscaler from different parameters
type Setter interface {
Create(context.Context, *autoscalingv1.VerticalPodAutoscaler) error
Update(context.Context, *autoscalingv1.VerticalPodAutoscaler) error
}

type setter struct {
client.Client
}

// Create creates VerticalPodAutoscaler
func (ds setter) Create(ctx context.Context, vp *autoscalingv1.VerticalPodAutoscaler) error {
return ds.Client.Create(ctx, vp)
}

// Update updates VerticalPodAutoscaler
func (ds setter) Update(ctx context.Context, vp *autoscalingv1.VerticalPodAutoscaler) error {
return ds.Client.Update(ctx, vp)
}
Original file line number Diff line number Diff line change
Expand Up @@ -20,6 +20,8 @@ package autoscaler
import (
"context"

autoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"

"github.com/apache/shardingsphere-on-cloud/shardingsphere-operator/api/v1alpha1"

autoscalingv2 "k8s.io/api/autoscaling/v2"
Expand All @@ -30,6 +32,7 @@ import (
// Builder build HPA from given AutoScaler
type Builder interface {
BuildHorizontalPodAutoScaler(context.Context, *metav1.ObjectMeta, schema.GroupVersionKind, *v1alpha1.ScalingPolicy) *autoscalingv2.HorizontalPodAutoscaler
BuildVerticalPodAutoscaler(context.Context, *metav1.ObjectMeta, schema.GroupVersionKind, *v1alpha1.ScalingPolicy) *autoscalingv1.VerticalPodAutoscaler
}

// NewBulder builds resources needed by AutoScaler
Expand Down
60 changes: 60 additions & 0 deletions shardingsphere-operator/pkg/reconcile/autoscaler/vpa.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,60 @@
/*
* Licensed to the Apache Software Foundation (ASF) under one or more
* contributor license agreements. See the NOTICE file distributed with
* this work for additional information regarding copyright ownership.
* The ASF licenses this file to You under the Apache License, Version 2.0
* (the "License"); you may not use this file except in compliance with
* the License. You may obtain a copy of the License at
*
* http://www.apache.org/licenses/LICENSE-2.0
*
* Unless required by applicable law or agreed to in writing, software
* distributed under the License is distributed on an "AS IS" BASIS,
* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
* See the License for the specific language governing permissions and
* limitations under the License.
*/

package autoscaler

import (
"context"

"github.com/apache/shardingsphere-on-cloud/shardingsphere-operator/api/v1alpha1"
"github.com/apache/shardingsphere-on-cloud/shardingsphere-operator/pkg/kubernetes/vpa"
v1 "k8s.io/api/autoscaling/v1"

metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
"k8s.io/apimachinery/pkg/runtime/schema"
autoscalingv1 "k8s.io/autoscaler/vertical-pod-autoscaler/pkg/apis/autoscaling.k8s.io/v1"
)

func (b builder) BuildVerticalPodAutoscaler(ctx context.Context, meta *metav1.ObjectMeta, gvk schema.GroupVersionKind, policy *v1alpha1.ScalingPolicy) *autoscalingv1.VerticalPodAutoscaler {
blder := vpa.NewVerticalPodAutoscalerBuilder()
blder.SetName(meta.Name).SetNamespace(meta.Namespace).SetLabels(meta.Labels).SetAnnotations(meta.Annotations).SetOwnerReferences([]metav1.OwnerReference{
*metav1.NewControllerRef(meta.GetObjectMeta(), gvk),
})

blder.SetTargetRef(v1.CrossVersionObjectReference{
Kind: "ComputeNode",
Name: policy.TargetSelector.ObjectRef.Name,
APIVersion: "shardingsphere.apache.org/v1alpha1",
})

if policy.Vertical.UpdatePolicy != nil {
blder.SetUpdatePolicy(policy.Vertical.UpdatePolicy)
}
if policy.Vertical.ResourcePolicy != nil {
blder.SetResourcePolicy(policy.Vertical.ResourcePolicy)
}
if policy.Vertical.Recommenders != nil {
var r []*autoscalingv1.VerticalPodAutoscalerRecommenderSelector
for _, recommender := range policy.Vertical.Recommenders {
recommenderCopy := recommender
r = append(r, &recommenderCopy)
}
blder.SetRecommenders(r)
}

return blder.BuildVPA()
}

0 comments on commit 64c54da

Please sign in to comment.