Support for overriding primary scaler replicas

Adding support for overriding the primary scaler replica count via
.spec.autoscalerRef.primaryScalerReplicas, a feature which would enable
users to define a different scaling configurations for the primary.

This can be useful in the situation where the user does not want to
scale the canary workload to the exact same size as the primary,
especially when opting for a canary deployment pattern where only a
small portion of traffic is routed to the canary workload pods.

Signed-off-by: Aurel Canciu <[email protected]>

artifacts/flagger/crd.yaml

@@ -121,6 +121,13 @@ spec:
                       type: object
                       additionalProperties:
                         type: string
+                    primaryScalerReplicas:
+                      type: object
+                      properties:
+                        minReplicas:
+                          type: number
+                        maxReplicas:
+                          type: number
                 ingressRef:
                   description: Ingress selector
                   type: object

charts/flagger/crds/crd.yaml

@@ -121,6 +121,13 @@ spec:
                       type: object
                       additionalProperties:
                         type: string
+                    primaryScalerReplicas:
+                      type: object
+                      properties:
+                        minReplicas:
+                          type: number
+                        maxReplicas:
+                          type: number
                 ingressRef:
                   description: Ingress selector
                   type: object

go.sum

@@ -20,12 +20,8 @@ cloud.google.com/go/bigquery v1.4.0/go.mod h1:S8dzgnTigyfTmLBfrtrhyYhwRxG72rYxvf
 cloud.google.com/go/bigquery v1.5.0/go.mod h1:snEHRnqQbz117VIFhE8bmtwIDY80NLUZUMb4Nv6dBIg=
 cloud.google.com/go/bigquery v1.7.0/go.mod h1://okPTzCYNXSlb24MZs83e2Do+h+VXtc4gLoIoXIAPc=
 cloud.google.com/go/bigquery v1.8.0/go.mod h1:J5hqkt3O0uAFnINi6JXValWIb1v0goeZM77hZzJN/fQ=
-cloud.google.com/go/compute v1.12.1 h1:gKVJMEyqV5c/UnpzjjQbo3Rjvvqpr9B1DFSbJC4OXr0=
-cloud.google.com/go/compute v1.12.1/go.mod h1:e8yNOBcBONZU1vJKCvCoDw/4JQsA0dpM4x/6PIIOocU=
 cloud.google.com/go/compute v1.13.0 h1:AYrLkB8NPdDRslNp4Jxmzrhdr03fUAIDbiGFjLWowoU=
 cloud.google.com/go/compute v1.13.0/go.mod h1:5aPTS0cUNMIc1CE546K+Th6weJUNQErARyZtRXDJ8GE=
-cloud.google.com/go/compute/metadata v0.2.1 h1:efOwf5ymceDhK6PKMnnrTHP4pppY5L22mle96M1yP48=
-cloud.google.com/go/compute/metadata v0.2.1/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
 cloud.google.com/go/compute/metadata v0.2.2 h1:aWKAjYaBaOSrpKl57+jnS/3fJRQnxL7TvR/u1VVbt6k=
 cloud.google.com/go/compute/metadata v0.2.2/go.mod h1:jgHgmJd2RKBGzXqF5LR2EZMGxBkeanZ9wwa75XHJgOM=
 cloud.google.com/go/datastore v1.0.0/go.mod h1:LXYbyblFSglQ5pkeyhO+Qmw7ukd3C+pD7TKLgZqpHYE=
@@ -45,8 +41,6 @@ cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9
 dmitri.shuralyov.com/gpu/mtl v0.0.0-20190408044501-666a987793e9/go.mod h1:H6x//7gZCb22OMCxBHrMx7a5I7Hp++hsVxbQ4BYO7hU=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
-github.com/Masterminds/semver/v3 v3.1.1 h1:hLg3sBzpNErnxhQtUy/mmLR2I9foDujNK030IGemrRc=
-github.com/Masterminds/semver/v3 v3.1.1/go.mod h1:VPu/7SZ7ePZ3QOrcuXROw5FAcLl4a0cBrbBpGY/8hQs=
 github.com/Masterminds/semver/v3 v3.2.0 h1:3MEsd0SM6jqZojhjLWWeBY+Kcjy9i6MQAeY7YgDP83g=
 github.com/Masterminds/semver/v3 v3.2.0/go.mod h1:qvl/7zhW3nngYb5+80sSMF+FG2BjYrf8m9wsX0PNOMQ=
 github.com/PuerkitoBio/purell v1.1.1 h1:WEQqlqaGbrPkxLJWfBwQmfEAE1Z7ONdDLqrN38tNFfI=
@@ -58,8 +52,6 @@ github.com/alecthomas/template v0.0.0-20190718012654-fb15b899a751/go.mod h1:LOuy
 github.com/alecthomas/units v0.0.0-20151022065526-2efee857e7cf/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190717042225-c3de453c63f4/go.mod h1:ybxpYRFXyAe+OPACYpWeL0wqObRcbAqCMya13uyzqw0=
 github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk541a8SKzHPHnH3zbiI+7dagKZ0cgpgrD7Fyho=
-github.com/aws/aws-sdk-go v1.44.144 h1:mMWdnYL8HZsobrQe1mwvQ18Xt8UbOVhWgipjuma5Mkg=
-github.com/aws/aws-sdk-go v1.44.144/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/aws/aws-sdk-go v1.44.155 h1:PMHMuUS0atPD4LhiXuYrLasrlIm4u3lpNQBl9h+Lr2s=
 github.com/aws/aws-sdk-go v1.44.155/go.mod h1:aVsgQcEevwlmQ7qHE9I3h+dtQgpqhFB+i8Phjh7fkwI=
 github.com/benbjohnson/clock v1.1.0 h1:Q92kusRqC1XV2MjkWETPvjJVqKetz1OzxZB7mHJLju8=
@@ -205,8 +197,6 @@ github.com/hashicorp/golang-lru v0.5.1/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ
 github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc=
 github.com/imdario/mergo v0.3.6 h1:xTNEAn+kxVO7dTZGu0CegyqKZmoWFI0rF8UxjlB2d28=
 github.com/imdario/mergo v0.3.6/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA=
-github.com/influxdata/influxdb-client-go/v2 v2.12.0 h1:LGct9uIp36IT+8RAJdmJGQbNonGi26YfYYSpDIyq8fI=
-github.com/influxdata/influxdb-client-go/v2 v2.12.0/go.mod h1:YteV91FiQxRdccyJ2cHvj2f/5sq4y4Njqu1fQzsQCOU=
 github.com/influxdata/influxdb-client-go/v2 v2.12.1 h1:RrjoDNyBGFYvjKfjmtIyYAn6GY/SrtocSo4RPlt+Lng=
 github.com/influxdata/influxdb-client-go/v2 v2.12.1/go.mod h1:YteV91FiQxRdccyJ2cHvj2f/5sq4y4Njqu1fQzsQCOU=
 github.com/influxdata/line-protocol v0.0.0-20200327222509-2487e7298839 h1:W9WBk7wlPfJLvMCdtV4zPulc4uCPrlywQOmbFOhgQNU=
@@ -346,8 +336,6 @@ go.uber.org/goleak v1.1.11 h1:wy28qYRKZgnJTxGxvye5/wgWr1EKjmUDGYox5mGlRlI=
 go.uber.org/multierr v1.6.0 h1:y6IPFStTAIT5Ytl7/XYmHvzXQ7S3g/IeZW9hyZ5thw4=
 go.uber.org/multierr v1.6.0/go.mod h1:cdWPpRnG4AhwMwsgIHip0KRBQjJy5kYEpYjJxpXp9iU=
 go.uber.org/zap v1.19.0/go.mod h1:xg/QME4nWcxGxrpdeYfq7UvYrLh66cuVKdrbD1XF/NI=
-go.uber.org/zap v1.23.0 h1:OjGQ5KQDEUawVHxNwQgPpiypGHOxo2mNZsOqTak4fFY=
-go.uber.org/zap v1.23.0/go.mod h1:D+nX8jyLsMHMYrln8A0rJjFt/T/9/bGgIhAqxv5URuY=
 go.uber.org/zap v1.24.0 h1:FiJd5l1UOLj0wCgbSE0rwwXHzEdAZS6hiiSnxJN/D60=
 go.uber.org/zap v1.24.0/go.mod h1:2kMP+WWQ8aoFoedH3T2sq6iJ2yDWpHbP0f6MQbS9Gkg=
 golang.org/x/crypto v0.0.0-20180904163835-0709b304e793/go.mod h1:6SG95UA2DQfeDnfUPMdvaQW0Q7yPrPDi9nlGo2tz2b4=
@@ -577,8 +565,6 @@ google.golang.org/api v0.24.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0M
 google.golang.org/api v0.28.0/go.mod h1:lIXQywCXRcnZPGlsd8NbLnOjtAoL6em04bJ9+z0MncE=
 google.golang.org/api v0.29.0/go.mod h1:Lcubydp8VUV7KeIHD9z2Bys/sm/vGKnG1UHuDBSrHWM=
 google.golang.org/api v0.30.0/go.mod h1:QGmEvQ87FHZNiUVJkT14jQNYJ4ZJjdRF23ZXz5138Fc=
-google.golang.org/api v0.103.0 h1:9yuVqlu2JCvcLg9p8S3fcFLZij8EPSyvODIY1rkMizQ=
-google.golang.org/api v0.103.0/go.mod h1:hGtW6nK1AC+d9si/UBhw8Xli+QMOf6xyNAyJw4qU9w0=
 google.golang.org/api v0.104.0 h1:KBfmLRqdZEbwQleFlSLnzpQJwhjpmNOk4cKQIBDZ9mg=
 google.golang.org/api v0.104.0/go.mod h1:JCspTXJbBxa5ySXw4UgUqVer7DfVxbvc/CTUFqAED5U=
 google.golang.org/appengine v1.1.0/go.mod h1:EbEs0AVv82hx2wNQdGPgUI5lhzA/G0D9YwlJXL52JkM=
@@ -619,10 +605,6 @@ google.golang.org/genproto v0.0.0-20200729003335-053ba62fc06f/go.mod h1:FWY/as6D
 google.golang.org/genproto v0.0.0-20200804131852-c06518451d9c/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20200825200019-8632dd797987/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
 google.golang.org/genproto v0.0.0-20201019141844-1ed22bb0c154/go.mod h1:FWY/as6DDZQgahTzZj3fqbO1CbirC29ZNUFHwi0/+no=
-google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6 h1:a2S6M0+660BgMNl++4JPlcAO/CjkqYItDEZwkoDQK7c=
-google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6/go.mod h1:rZS5c/ZVYMaOGBfO68GWtjOw/eLaZM1X6iVtgjZ+EWg=
-google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6 h1:AGXp12e/9rItf6/4QymU7WsAUwCf+ICW75cuR91nJIc=
-google.golang.org/genproto v0.0.0-20221206210731-b1a01be3a5f6/go.mod h1:1dOng4TWOomJrDGhpXjfCD35wQC6jnC7HpRmOFRqEV0=
 google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37 h1:jmIfw8+gSvXcZSgaFAGyInDXeWzUhvYH57G/5GKMn70=
 google.golang.org/genproto v0.0.0-20221207170731-23e4bf6bdc37/go.mod h1:RGgjbofJ8xD9Sq1VVhDM1Vok1vRONV+rg+CjzG4SZKM=
 google.golang.org/grpc v1.19.0/go.mod h1:mqu4LbDTu4XGKhr4mRzUsmM4RtVoemTSY81AxZiDr8c=

kustomize/base/flagger/crd.yaml

@@ -121,6 +121,13 @@ spec:
                       type: object
                       additionalProperties:
                         type: string
+                    primaryScalerReplicas:
+                      type: object
+                      properties:
+                        minReplicas:
+                          type: number
+                        maxReplicas:
+                          type: number
                 ingressRef:
                   description: Ingress selector
                   type: object

pkg/apis/flagger/v1beta1/canary.go

@@ -447,6 +447,19 @@ type AutoscalerRefernce struct {
 	// scaler, if a scaler supports scaling using queries.
 	// +optional
 	PrimaryScalerQueries map[string]string `json:"primaryScalerQueries"`
+
+	// PrimaryScalerReplicas defines overrides for the primary
+	// autoscaler replicas.
+	// +optional
+	PrimaryScalerReplicas *ScalerReplicas `json:"primaryScalerReplicas,omitempty"`
+}
+
+// ScalerReplicas holds overrides for autoscaler replicas
+type ScalerReplicas struct {
+	// +optional
+	MinReplicas *int32 `json:"minReplicas,omitempty"`
+	// +optional
+	MaxReplicas *int32 `json:"maxReplicas,omitempty"`
 }
 
 // CustomMetadata holds labels and annotations to set on generated objects.

pkg/canary/hpa_reconciler.go

@@ -77,6 +77,15 @@ func (hr *HPAReconciler) reconcilePrimaryHpaV2(cd *flaggerv1.Canary, hpa *hpav2.
 		Behavior:    hpa.Spec.Behavior,
 	}
 
+	if replicas := cd.Spec.AutoscalerRef.PrimaryScalerReplicas; replicas != nil {
+		if minReplicas := replicas.MinReplicas; minReplicas != nil {
+			hpaSpec.MinReplicas = minReplicas
+		}
+		if maxReplicas := replicas.MaxReplicas; maxReplicas != nil {
+			hpaSpec.MaxReplicas = *maxReplicas
+		}
+	}
+
 	primaryHpaName := fmt.Sprintf("%s-primary", cd.Spec.AutoscalerRef.Name)
 	primaryHpa, err := hr.kubeClient.AutoscalingV2().HorizontalPodAutoscalers(cd.Namespace).Get(context.TODO(), primaryHpaName, metav1.GetOptions{})
 
@@ -161,6 +170,15 @@ func (hr *HPAReconciler) reconcilePrimaryHpaV2Beta2(cd *flaggerv1.Canary, hpa *h
 		Behavior:    hpa.Spec.Behavior,
 	}
 
+	if replicas := cd.Spec.AutoscalerRef.PrimaryScalerReplicas; replicas != nil {
+		if minReplicas := replicas.MinReplicas; minReplicas != nil {
+			hpaSpec.MinReplicas = minReplicas
+		}
+		if maxReplicas := replicas.MaxReplicas; maxReplicas != nil {
+			hpaSpec.MaxReplicas = *maxReplicas
+		}
+	}
+
 	primaryHpaName := fmt.Sprintf("%s-primary", cd.Spec.AutoscalerRef.Name)
 	primaryHpa, err := hr.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers(cd.Namespace).Get(context.TODO(), primaryHpaName, metav1.GetOptions{})
 

pkg/canary/hpa_reconciler_test.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"testing"
 
+	flaggerv1 "github.com/fluxcd/flagger/pkg/apis/flagger/v1beta1"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/require"
 	hpav2 "k8s.io/api/autoscaling/v2"
@@ -73,6 +74,19 @@ func Test_reconcilePrimaryHpaV2(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, int(*primaryHPA.Spec.Metrics[0].Resource.Target.AverageUtilization), 50)
 	assert.Equal(t, int(primaryHPA.Spec.MaxReplicas), 10)
+
+	// Test reconcile with PrimaryHorizontalPodAutoscalerOverride
+	mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas = &flaggerv1.ScalerReplicas{
+		MinReplicas: int32p(2),
+		MaxReplicas: int32p(15),
+	}
+	err = hpaReconciler.reconcilePrimaryHpaV2(mocks.canary, hpa, false)
+	require.NoError(t, err)
+
+	primaryHPA, err = mocks.kubeClient.AutoscalingV2().HorizontalPodAutoscalers("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
+	require.NoError(t, err)
+	assert.Equal(t, primaryHPA.Spec.MinReplicas, mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas.MinReplicas)
+	assert.Equal(t, primaryHPA.Spec.MaxReplicas, *mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas.MaxReplicas)
 }
 
 func Test_reconcilePrimaryHpaV2Beta2(t *testing.T) {
@@ -105,4 +119,17 @@ func Test_reconcilePrimaryHpaV2Beta2(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, int(*primaryHPA.Spec.Metrics[0].Resource.Target.AverageUtilization), 50)
 	assert.Equal(t, int(primaryHPA.Spec.MaxReplicas), 10)
+
+	// Test reconcile with PrimaryHorizontalPodAutoscalerOverride
+	mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas = &flaggerv1.ScalerReplicas{
+		MinReplicas: int32p(2),
+		MaxReplicas: int32p(15),
+	}
+	err = hpaReconciler.reconcilePrimaryHpaV2Beta2(mocks.canary, hpa, false)
+	require.NoError(t, err)
+
+	primaryHPA, err = mocks.kubeClient.AutoscalingV2beta2().HorizontalPodAutoscalers("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
+	require.NoError(t, err)
+	assert.Equal(t, primaryHPA.Spec.MinReplicas, mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas.MinReplicas)
+	assert.Equal(t, primaryHPA.Spec.MaxReplicas, *mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas.MaxReplicas)
 }

pkg/canary/scaled_object_reconciler.go

@@ -63,6 +63,16 @@ func (sor *ScaledObjectReconciler) reconcilePrimaryScaler(cd *flaggerv1.Canary,
 		Fallback:         targetSoClone.Spec.Fallback,
 		IdleReplicaCount: targetSoClone.Spec.IdleReplicaCount,
 	}
+
+	if replicas := cd.Spec.AutoscalerRef.PrimaryScalerReplicas; replicas != nil {
+		if minReplicas := replicas.MinReplicas; minReplicas != nil {
+			soSpec.MinReplicaCount = minReplicas
+		}
+		if maxReplicas := replicas.MaxReplicas; maxReplicas != nil {
+			soSpec.MaxReplicaCount = maxReplicas
+		}
+	}
+
 	primarySoName := fmt.Sprintf("%s-primary", cd.Spec.AutoscalerRef.Name)
 	primarySo, err := sor.flaggerClient.KedaV1alpha1().ScaledObjects(cd.Namespace).Get(context.TODO(), primarySoName, metav1.GetOptions{})
 	if errors.IsNotFound(err) {

pkg/canary/scaled_object_reconciler_test.go

@@ -46,6 +46,20 @@ func Test_reconcilePrimaryScaledObject(t *testing.T) {
 	require.NoError(t, err)
 	assert.Equal(t, int(*primarySO.Spec.PollingInterval), 20)
 	assert.Equal(t, primarySO.Spec.Triggers[0].Metadata["query"], `sum(rate(http_requests_total{app="podinfo-primary"}[10m]))`)
+
+	// Test reconcile with PrimaryScaledObjectOverride
+	mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas = &flaggerv1.ScalerReplicas{
+		MinReplicas: int32p(2),
+		MaxReplicas: int32p(15),
+	}
+	err = soReconciler.reconcilePrimaryScaler(mocks.canary, false)
+	require.NoError(t, err)
+
+	primarySO, err = mocks.flaggerClient.KedaV1alpha1().ScaledObjects("default").Get(context.TODO(), "podinfo-primary", metav1.GetOptions{})
+	require.NoError(t, err)
+	assert.Equal(t, int(*primarySO.Spec.PollingInterval), 20)
+	assert.Equal(t, primarySO.Spec.MinReplicaCount, mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas.MinReplicas)
+	assert.Equal(t, primarySO.Spec.MaxReplicaCount, mocks.canary.Spec.AutoscalerRef.PrimaryScalerReplicas.MaxReplicas)
 }
 
 func Test_pauseScaledObject(t *testing.T) {