From c2653665b51fc043efef9827c4a39a0bfca71082 Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Wed, 10 Aug 2022 15:10:09 -0400 Subject: [PATCH 01/10] fix: return an error when we cannot swap the replicaset hashes fixes #2050 Signed-off-by: Jack Andersen --- rollout/canary.go | 6 ++++++ rollout/canary_test.go | 8 +++++++- rollout/service.go | 10 +++++++++- rollout/service_test.go | 3 ++- 4 files changed, 24 insertions(+), 3 deletions(-) diff --git a/rollout/canary.go b/rollout/canary.go index 7c31506a82..c2300fc02d 100644 --- a/rollout/canary.go +++ b/rollout/canary.go @@ -1,6 +1,7 @@ package rollout import ( + "errors" "sort" appsv1 "k8s.io/api/apps/v1" @@ -50,6 +51,11 @@ func (c *rolloutContext) rolloutCanary() error { } if err := c.reconcileStableAndCanaryService(); err != nil { + // if we cannot reconcile the stable and canary services then + // we should not continue to adjust traffic routing + if errors.Is(err, DelayServiceSelectorSwapError) { + return nil + } return err } diff --git a/rollout/canary_test.go b/rollout/canary_test.go index 56445f3978..d59ab1bff4 100644 --- a/rollout/canary_test.go +++ b/rollout/canary_test.go @@ -1271,7 +1271,13 @@ func TestCanarySVCSelectors(t *testing.T) { informers.Start(stopchan) informers.WaitForCacheSync(stopchan) err := rc.reconcileStableAndCanaryService() - assert.NoError(t, err, "unable to reconcileStableAndCanaryService") + // There is an error returned here because we could not reconcile + // unhealthy services. + if tc.shouldTargetNewRS { + assert.NoError(t, err, "unable to reconcileStableAndCanaryService") + } else { + assert.Error(t, err, "able to reconcileStableAndCanaryService for unhealthy replicas") + } updatedCanarySVC, err := servicesLister.Services(rc.rollout.Namespace).Get(canaryService.Name) assert.NoError(t, err, "unable to get updated canary service") if tc.shouldTargetNewRS { diff --git a/rollout/service.go b/rollout/service.go index a2861be0c2..8a657efb8d 100644 --- a/rollout/service.go +++ b/rollout/service.go @@ -45,6 +45,14 @@ const ( }` ) +type delayServiceSelectorSwapError struct{} + +func (e delayServiceSelectorSwapError) Error() string { + return "Selectors cannot be swapped yet because not all pods are ready" +} + +var DelayServiceSelectorSwapError = delayServiceSelectorSwapError{} + func generatePatch(service *corev1.Service, newRolloutUniqueLabelValue string, r *v1alpha1.Rollout) string { if _, ok := service.Annotations[v1alpha1.ManagedByRolloutsKey]; !ok { return fmt.Sprintf(switchSelectorAndAddManagedByPatch, r.Name, newRolloutUniqueLabelValue) @@ -282,7 +290,7 @@ func (c *rolloutContext) ensureSVCTargets(svcName string, rs *appsv1.ReplicaSet, if checkRsAvailability && !replicasetutil.IsReplicaSetAvailable(rs) { logCtx := c.log.WithField(logutil.ServiceKey, svc.Name) logCtx.Infof("delaying service switch from %s to %s: ReplicaSet not fully available", currSelector, desiredSelector) - return nil + return DelayServiceSelectorSwapError } err = c.switchServiceSelector(svc, desiredSelector, c.rollout) if err != nil { diff --git a/rollout/service_test.go b/rollout/service_test.go index 8a402d001c..9ea2a411a2 100644 --- a/rollout/service_test.go +++ b/rollout/service_test.go @@ -781,7 +781,8 @@ func TestDelayCanaryStableServiceLabelInjection(t *testing.T) { roCtx.stableRS = newReplicaSetWithStatus(ro2, 3, 0) err = roCtx.reconcileStableAndCanaryService() - assert.NoError(t, err) + // an error is returned because we are delaying + assert.Equal(t, err, DelayServiceSelectorSwapError) _, canaryInjected := canarySvc.Spec.Selector[v1alpha1.DefaultRolloutUniqueLabelKey] assert.False(t, canaryInjected) _, stableInjected := stableSvc.Spec.Selector[v1alpha1.DefaultRolloutUniqueLabelKey] From b5ad50440aab441250305542a0cab375dac53086 Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Fri, 25 Nov 2022 19:59:03 -0800 Subject: [PATCH 02/10] update USERS.md as per PR guidelines Signed-off-by: Jack Andersen --- USERS.md | 1 + 1 file changed, 1 insertion(+) diff --git a/USERS.md b/USERS.md index 944b4ad722..f3c04b35fa 100644 --- a/USERS.md +++ b/USERS.md @@ -38,3 +38,4 @@ Organizations below are **officially** using Argo Rollouts. Please send a PR wit 1. [Twilio SendGrid](https://sendgrid.com) 1. [Ubie](https://ubie.life/) 1. [VISITS Technologies](https://visits.world/en) +1. [Plaid](https://plaid.com) From b223b336ce325778097edfee80d03e000a687de0 Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Wed, 10 Aug 2022 16:02:48 -0400 Subject: [PATCH 03/10] Add a test that the error returns a non empty message Signed-off-by: Jack Andersen --- rollout/service_test.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/rollout/service_test.go b/rollout/service_test.go index 9ea2a411a2..ba34b3c31a 100644 --- a/rollout/service_test.go +++ b/rollout/service_test.go @@ -806,3 +806,8 @@ func TestDelayCanaryStableServiceLabelInjection(t *testing.T) { } } + +func TestDelayServiceSelectorSwapError(t *testing.T) { + assert.Error(t, DelayServiceSelectorSwapError) + assert.NotEqual(t, DelayServiceSelectorSwapError.Error(), "") +} From 9bc2cc019e94d0170b0daf28d73505aaeaf5df22 Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Wed, 10 Aug 2022 17:11:42 -0400 Subject: [PATCH 04/10] Add a unit test for testing the failing reconciliation Signed-off-by: Jack Andersen --- rollout/canary_test.go | 119 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 119 insertions(+) diff --git a/rollout/canary_test.go b/rollout/canary_test.go index d59ab1bff4..1e0192bd62 100644 --- a/rollout/canary_test.go +++ b/rollout/canary_test.go @@ -19,6 +19,8 @@ import ( "github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1" "github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake" + "github.com/argoproj/argo-rollouts/rollout/mocks" + "github.com/argoproj/argo-rollouts/rollout/trafficrouting" "github.com/argoproj/argo-rollouts/utils/annotations" "github.com/argoproj/argo-rollouts/utils/conditions" "github.com/argoproj/argo-rollouts/utils/hash" @@ -60,6 +62,123 @@ func bumpVersion(rollout *v1alpha1.Rollout) *v1alpha1.Rollout { return newRollout } +func TestCanaryRollout(t *testing.T) { + for _, tc := range []struct { + canaryReplicas int32 + canaryAvailReplicas int32 + + shouldRouteTraffic bool + }{ + {0, 0, false}, + {2, 0, false}, + {2, 1, false}, + {2, 2, true}, + } { + namespace := "namespace" + selectorNewRSVal := "new-rs-xxx" + stableService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "stable", + Namespace: namespace, + }, + } + canaryService := &corev1.Service{ + ObjectMeta: metav1.ObjectMeta{ + Name: "canary", + Namespace: namespace, + }, + } + canaryReplicaset := &v1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "canary", + Namespace: namespace, + Labels: map[string]string{ + v1alpha1.DefaultRolloutUniqueLabelKey: selectorNewRSVal, + }, + }, + Spec: v1.ReplicaSetSpec{ + Replicas: pointer.Int32Ptr(tc.canaryReplicas), + }, + Status: v1.ReplicaSetStatus{ + AvailableReplicas: tc.canaryAvailReplicas, + }, + } + + stableReplicaset := &v1.ReplicaSet{ + ObjectMeta: metav1.ObjectMeta{ + Name: "stable", + Namespace: namespace, + }, + Spec: v1.ReplicaSetSpec{ + Replicas: pointer.Int32Ptr(0), + }, + } + + fake := fake.Clientset{} + kubeclient := k8sfake.NewSimpleClientset( + stableService, canaryService, canaryReplicaset, stableReplicaset) + informers := k8sinformers.NewSharedInformerFactory(kubeclient, 0) + servicesLister := informers.Core().V1().Services().Lister() + + rollout := &v1alpha1.Rollout{ + ObjectMeta: metav1.ObjectMeta{ + Name: "selector-labels-test", + Namespace: namespace, + }, + Spec: v1alpha1.RolloutSpec{ + Strategy: v1alpha1.RolloutStrategy{ + Canary: &v1alpha1.CanaryStrategy{ + StableService: stableService.Name, + CanaryService: canaryService.Name, + }, + }, + }, + } + + rollout.Status.CurrentStepHash = conditions.ComputeStepHash(rollout) + haveRoutedTraffic := false + trafficRouter := mocks.NewTrafficRoutingReconciler(t) + if tc.shouldRouteTraffic { + trafficRouter.On("Type").Return("mock") + trafficRouter.On("RemoveManagedRoutes").Return(nil) + trafficRouter.On("UpdateHash", "new-rs-xxx", "").Return(nil) + trafficRouter.On("SetWeight", int32(0)).Return(nil) + trafficRouter.On("VerifyWeight", int32(0)).Return(nil, nil) + } + mocks.NewTrafficRoutingReconciler(t) + rc := rolloutContext{ + log: logutil.WithRollout(rollout), + reconcilerBase: reconcilerBase{ + argoprojclientset: &fake, + servicesLister: servicesLister, + kubeclientset: kubeclient, + recorder: record.NewFakeEventRecorder(), + + newTrafficRoutingReconciler: func(roCtx *rolloutContext) ([]trafficrouting.TrafficRoutingReconciler, error) { + haveRoutedTraffic = true + return []trafficrouting.TrafficRoutingReconciler{ + trafficRouter, + }, nil + }, + }, + + rollout: rollout, + pauseContext: &pauseContext{ + rollout: rollout, + }, + newRS: canaryReplicaset, + stableRS: stableReplicaset, + } + stopchan := make(chan struct{}) + defer close(stopchan) + informers.Start(stopchan) + informers.WaitForCacheSync(stopchan) + err := rc.rolloutCanary() + assert.NoError(t, err) + assert.Equal(t, tc.shouldRouteTraffic, haveRoutedTraffic, " the traffic routing reconciler was called even though we are not ready to route traffic") + } +} + // TestCanaryRolloutBumpVersion verifies we correctly bump revision of Rollout and new ReplicaSet func TestCanaryRolloutBumpVersion(t *testing.T) { f := newFixture(t) From 6f40ca03d2c59764a9e6b308fc12c58fdfc22593 Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Tue, 8 Nov 2022 17:36:32 -0500 Subject: [PATCH 05/10] Use a different method of propogating information about delayed service swaps Signed-off-by: Jack Andersen --- rollout/canary.go | 6 ------ rollout/context.go | 4 ++++ rollout/service.go | 4 +++- rollout/service_test.go | 13 ++++++------- rollout/trafficrouting.go | 4 ++++ 5 files changed, 17 insertions(+), 14 deletions(-) diff --git a/rollout/canary.go b/rollout/canary.go index c2300fc02d..7c31506a82 100644 --- a/rollout/canary.go +++ b/rollout/canary.go @@ -1,7 +1,6 @@ package rollout import ( - "errors" "sort" appsv1 "k8s.io/api/apps/v1" @@ -51,11 +50,6 @@ func (c *rolloutContext) rolloutCanary() error { } if err := c.reconcileStableAndCanaryService(); err != nil { - // if we cannot reconcile the stable and canary services then - // we should not continue to adjust traffic routing - if errors.Is(err, DelayServiceSelectorSwapError) { - return nil - } return err } diff --git a/rollout/context.go b/rollout/context.go index f8fa5b5f03..14283e3a83 100644 --- a/rollout/context.go +++ b/rollout/context.go @@ -49,6 +49,10 @@ type rolloutContext struct { // (e.g. a setWeight step, after a blue-green active switch, after stable service switch), // since we do not want to continually verify weight in case it could incur rate-limiting or other expenses. targetsVerified *bool + + // skippedSelectorSwap indicates that we have skipped swapping selectors + // and are not ready to perform any final actions of moving to 100% + skippedSelectorSwap *bool } func (c *rolloutContext) reconcile() error { diff --git a/rollout/service.go b/rollout/service.go index 8a657efb8d..0337c8352b 100644 --- a/rollout/service.go +++ b/rollout/service.go @@ -290,8 +290,10 @@ func (c *rolloutContext) ensureSVCTargets(svcName string, rs *appsv1.ReplicaSet, if checkRsAvailability && !replicasetutil.IsReplicaSetAvailable(rs) { logCtx := c.log.WithField(logutil.ServiceKey, svc.Name) logCtx.Infof("delaying service switch from %s to %s: ReplicaSet not fully available", currSelector, desiredSelector) - return DelayServiceSelectorSwapError + c.skippedSelectorSwap = pointer.BoolPtr(true) + return nil } + c.skippedSelectorSwap = pointer.BoolPtr(false) err = c.switchServiceSelector(svc, desiredSelector, c.rollout) if err != nil { return err diff --git a/rollout/service_test.go b/rollout/service_test.go index ba34b3c31a..c14506ab3c 100644 --- a/rollout/service_test.go +++ b/rollout/service_test.go @@ -781,12 +781,16 @@ func TestDelayCanaryStableServiceLabelInjection(t *testing.T) { roCtx.stableRS = newReplicaSetWithStatus(ro2, 3, 0) err = roCtx.reconcileStableAndCanaryService() - // an error is returned because we are delaying - assert.Equal(t, err, DelayServiceSelectorSwapError) + assert.NoError(t, err) _, canaryInjected := canarySvc.Spec.Selector[v1alpha1.DefaultRolloutUniqueLabelKey] assert.False(t, canaryInjected) _, stableInjected := stableSvc.Spec.Selector[v1alpha1.DefaultRolloutUniqueLabelKey] assert.False(t, stableInjected) + + assert.NotNil(t, roCtx.skippedSelectorSwap) + if roCtx.skippedSelectorSwap != nil { + assert.True(t, *roCtx.skippedSelectorSwap) + } } { // next ensure we do update service because new/stable are now available @@ -806,8 +810,3 @@ func TestDelayCanaryStableServiceLabelInjection(t *testing.T) { } } - -func TestDelayServiceSelectorSwapError(t *testing.T) { - assert.Error(t, DelayServiceSelectorSwapError) - assert.NotEqual(t, DelayServiceSelectorSwapError.Error(), "") -} diff --git a/rollout/trafficrouting.go b/rollout/trafficrouting.go index ebc22b9704..f539b44c8c 100644 --- a/rollout/trafficrouting.go +++ b/rollout/trafficrouting.go @@ -220,6 +220,10 @@ func (c *rolloutContext) reconcileTrafficRouting() error { return err } + if c.skippedSelectorSwap != nil && *c.skippedSelectorSwap { + continue + } + err = reconciler.SetWeight(desiredWeight, weightDestinations...) if err != nil { c.recorder.Warnf(c.rollout, record.EventOptions{EventReason: "TrafficRoutingError"}, err.Error()) From 49e9246c606e7a81d72dc6ca79c6d488839f533f Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Tue, 8 Nov 2022 17:40:24 -0500 Subject: [PATCH 06/10] Avoid code smell Signed-off-by: Jack Andersen --- rollout/canary_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rollout/canary_test.go b/rollout/canary_test.go index 1e0192bd62..c2a59de4a4 100644 --- a/rollout/canary_test.go +++ b/rollout/canary_test.go @@ -63,6 +63,7 @@ func bumpVersion(rollout *v1alpha1.Rollout) *v1alpha1.Rollout { } func TestCanaryRollout(t *testing.T) { + newrsVal := "new-rs-xxx" for _, tc := range []struct { canaryReplicas int32 canaryAvailReplicas int32 @@ -75,7 +76,7 @@ func TestCanaryRollout(t *testing.T) { {2, 2, true}, } { namespace := "namespace" - selectorNewRSVal := "new-rs-xxx" + selectorNewRSVal := newrsVal stableService := &corev1.Service{ ObjectMeta: metav1.ObjectMeta{ Name: "stable", @@ -141,7 +142,7 @@ func TestCanaryRollout(t *testing.T) { if tc.shouldRouteTraffic { trafficRouter.On("Type").Return("mock") trafficRouter.On("RemoveManagedRoutes").Return(nil) - trafficRouter.On("UpdateHash", "new-rs-xxx", "").Return(nil) + trafficRouter.On("UpdateHash", newrsVal, "").Return(nil) trafficRouter.On("SetWeight", int32(0)).Return(nil) trafficRouter.On("VerifyWeight", int32(0)).Return(nil, nil) } From 9c81d772164056b3689af76cdbaf12b9c919150b Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Tue, 8 Nov 2022 18:43:53 -0500 Subject: [PATCH 07/10] Fix the tests to get them to cooperate Signed-off-by: Jack Andersen --- rollout/canary_test.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/rollout/canary_test.go b/rollout/canary_test.go index c2a59de4a4..d45d2e6cf6 100644 --- a/rollout/canary_test.go +++ b/rollout/canary_test.go @@ -137,12 +137,11 @@ func TestCanaryRollout(t *testing.T) { } rollout.Status.CurrentStepHash = conditions.ComputeStepHash(rollout) - haveRoutedTraffic := false trafficRouter := mocks.NewTrafficRoutingReconciler(t) + trafficRouter.On("Type").Return("mock") + trafficRouter.On("RemoveManagedRoutes").Return(nil) + trafficRouter.On("UpdateHash", newrsVal, "").Return(nil) if tc.shouldRouteTraffic { - trafficRouter.On("Type").Return("mock") - trafficRouter.On("RemoveManagedRoutes").Return(nil) - trafficRouter.On("UpdateHash", newrsVal, "").Return(nil) trafficRouter.On("SetWeight", int32(0)).Return(nil) trafficRouter.On("VerifyWeight", int32(0)).Return(nil, nil) } @@ -156,7 +155,6 @@ func TestCanaryRollout(t *testing.T) { recorder: record.NewFakeEventRecorder(), newTrafficRoutingReconciler: func(roCtx *rolloutContext) ([]trafficrouting.TrafficRoutingReconciler, error) { - haveRoutedTraffic = true return []trafficrouting.TrafficRoutingReconciler{ trafficRouter, }, nil @@ -176,7 +174,9 @@ func TestCanaryRollout(t *testing.T) { informers.WaitForCacheSync(stopchan) err := rc.rolloutCanary() assert.NoError(t, err) - assert.Equal(t, tc.shouldRouteTraffic, haveRoutedTraffic, " the traffic routing reconciler was called even though we are not ready to route traffic") + if !tc.shouldRouteTraffic { + trafficRouter.AssertNotCalled(t, "SetWeight", int32(0)) + } } } From 002f75c7bf7482edb336fcd1798da5ed4560c47a Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Fri, 25 Nov 2022 18:39:41 -0800 Subject: [PATCH 08/10] Remove unused error and return on selector skip with error Signed-off-by: Jack Andersen --- rollout/service.go | 8 -------- rollout/trafficrouting.go | 3 ++- 2 files changed, 2 insertions(+), 9 deletions(-) diff --git a/rollout/service.go b/rollout/service.go index 0337c8352b..1f929049be 100644 --- a/rollout/service.go +++ b/rollout/service.go @@ -45,14 +45,6 @@ const ( }` ) -type delayServiceSelectorSwapError struct{} - -func (e delayServiceSelectorSwapError) Error() string { - return "Selectors cannot be swapped yet because not all pods are ready" -} - -var DelayServiceSelectorSwapError = delayServiceSelectorSwapError{} - func generatePatch(service *corev1.Service, newRolloutUniqueLabelValue string, r *v1alpha1.Rollout) string { if _, ok := service.Annotations[v1alpha1.ManagedByRolloutsKey]; !ok { return fmt.Sprintf(switchSelectorAndAddManagedByPatch, r.Name, newRolloutUniqueLabelValue) diff --git a/rollout/trafficrouting.go b/rollout/trafficrouting.go index f539b44c8c..33237ebc89 100644 --- a/rollout/trafficrouting.go +++ b/rollout/trafficrouting.go @@ -221,7 +221,8 @@ func (c *rolloutContext) reconcileTrafficRouting() error { } if c.skippedSelectorSwap != nil && *c.skippedSelectorSwap { - continue + c.log.Infof("Skipping selector swap (%v)", c.skippedSelectorSwap) + return nil } err = reconciler.SetWeight(desiredWeight, weightDestinations...) From 0db6d11279a56388b7061488a24973230ee37f6c Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Fri, 25 Nov 2022 19:59:52 -0800 Subject: [PATCH 09/10] Only delay changing weights when DynamicStableScale is set Signed-off-by: Jack Andersen --- rollout/canary_test.go | 8 +------- rollout/trafficrouting.go | 7 +++++-- 2 files changed, 6 insertions(+), 9 deletions(-) diff --git a/rollout/canary_test.go b/rollout/canary_test.go index d45d2e6cf6..1f306027dc 100644 --- a/rollout/canary_test.go +++ b/rollout/canary_test.go @@ -1391,13 +1391,7 @@ func TestCanarySVCSelectors(t *testing.T) { informers.Start(stopchan) informers.WaitForCacheSync(stopchan) err := rc.reconcileStableAndCanaryService() - // There is an error returned here because we could not reconcile - // unhealthy services. - if tc.shouldTargetNewRS { - assert.NoError(t, err, "unable to reconcileStableAndCanaryService") - } else { - assert.Error(t, err, "able to reconcileStableAndCanaryService for unhealthy replicas") - } + assert.NoError(t, err, "unable to reconcileStableAndCanaryService") updatedCanarySVC, err := servicesLister.Services(rc.rollout.Namespace).Get(canaryService.Name) assert.NoError(t, err, "unable to get updated canary service") if tc.shouldTargetNewRS { diff --git a/rollout/trafficrouting.go b/rollout/trafficrouting.go index 33237ebc89..163d41767c 100644 --- a/rollout/trafficrouting.go +++ b/rollout/trafficrouting.go @@ -220,8 +220,11 @@ func (c *rolloutContext) reconcileTrafficRouting() error { return err } - if c.skippedSelectorSwap != nil && *c.skippedSelectorSwap { - c.log.Infof("Skipping selector swap (%v)", c.skippedSelectorSwap) + // if we haven't swapped selectors and dynamic stable scale is set then + // we might switch weights to a set of pods that are not ready (importantly this is the case + // where canary: 100, stable: 0 -> canary: 0, stable: 100). + if c.skippedSelectorSwap != nil && *c.skippedSelectorSwap && c.rollout.Spec.Strategy.Canary.DynamicStableScale { + c.log.Infof("Skipping weight changes since replica set is not ready and dynamicStableScale is set") return nil } From c5ffcf2819216b4d6b216bacf37b2bc0b067abcf Mon Sep 17 00:00:00 2001 From: Jack Andersen Date: Fri, 25 Nov 2022 21:02:59 -0800 Subject: [PATCH 10/10] Add the right flag to the canary strategy for dynamic stable scale Signed-off-by: Jack Andersen --- rollout/canary_test.go | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/rollout/canary_test.go b/rollout/canary_test.go index 1f306027dc..29cdd298bb 100644 --- a/rollout/canary_test.go +++ b/rollout/canary_test.go @@ -129,8 +129,9 @@ func TestCanaryRollout(t *testing.T) { Spec: v1alpha1.RolloutSpec{ Strategy: v1alpha1.RolloutStrategy{ Canary: &v1alpha1.CanaryStrategy{ - StableService: stableService.Name, - CanaryService: canaryService.Name, + DynamicStableScale: true, + StableService: stableService.Name, + CanaryService: canaryService.Name, }, }, },