Skip to content

Commit

Permalink
fix(controller): don't timeout rollout when still waiting for scale d…
Browse files Browse the repository at this point in the history
…own delay (#3417)

Signed-off-by: Yohan Belval <[email protected]>
  • Loading branch information
yohanb authored Mar 6, 2024
1 parent ad0b7fe commit e184957
Show file tree
Hide file tree
Showing 2 changed files with 40 additions and 1 deletion.
16 changes: 15 additions & 1 deletion rollout/sync.go
Original file line number Diff line number Diff line change
Expand Up @@ -578,6 +578,19 @@ func isIndefiniteStep(r *v1alpha1.Rollout) bool {
return false
}

// isWaitingForReplicaSetScaleDown returns whether or not the rollout still has other replica sets with a scale down deadline annotation
func isWaitingForReplicaSetScaleDown(r *v1alpha1.Rollout, newRS, stableRS *appsv1.ReplicaSet, allRSs []*appsv1.ReplicaSet) bool {
otherRSs := replicasetutil.GetOtherRSs(r, newRS, stableRS, allRSs)

for _, rs := range otherRSs {
if replicasetutil.HasScaleDownDeadline(rs) {
return true
}
}

return false
}

func (c *rolloutContext) calculateRolloutConditions(newStatus v1alpha1.RolloutStatus) v1alpha1.RolloutStatus {
isPaused := len(c.rollout.Status.PauseConditions) > 0 || c.rollout.Spec.Paused
isAborted := c.pauseContext.IsAborted()
Expand Down Expand Up @@ -663,7 +676,8 @@ func (c *rolloutContext) calculateRolloutConditions(newStatus v1alpha1.RolloutSt
conditions.RemoveRolloutCondition(&newStatus, v1alpha1.RolloutProgressing)
}
conditions.SetRolloutCondition(&newStatus, *condition)
case !isIndefiniteStep(c.rollout) && conditions.RolloutTimedOut(c.rollout, &newStatus):
case !isIndefiniteStep(c.rollout) && !isWaitingForReplicaSetScaleDown(c.rollout, c.newRS, c.stableRS, c.allRSs) && conditions.RolloutTimedOut(c.rollout, &newStatus):

// Update the rollout with a timeout condition. If the condition already exists,
// we ignore this update.
msg := fmt.Sprintf(conditions.RolloutTimeOutMessage, c.rollout.Name)
Expand Down
25 changes: 25 additions & 0 deletions test/e2e/canary_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -465,6 +465,31 @@ spec:
WaitForRolloutStatus("Healthy")
}

// TestCanaryScaleDownDelayWithProgressDeadline verifies that a rollout with a pending scale down doesn't trigger a ProgressDeadlineExceeded event and renders the rollout degraded
func (s *CanarySuite) TestCanaryScaleDownDelayWithProgressDeadline() {
s.Given().
HealthyRollout(`@functional/canary-scaledowndelay.yaml`).
When().
UpdateSpec(`
spec:
progressDeadlineSeconds: 5`).
Then().
When().
UpdateSpec(`
spec:
template:
metadata:
annotations:
rev: two`). // update to revision 2
WaitForRolloutStatus("Healthy").
Sleep(10 * time.Second). // sleep > progressDeadlineSeconds
Then().
Assert(func(t *fixtures.Then) {
status := string(t.GetRollout().Status.Phase)
assert.Equal(s.T(), "Healthy", status)
})
}

// TestCanaryScaleDownDelay verifies canary uses a scaleDownDelay when traffic routing is used,
// and verifies the annotation is properly managed
func (s *CanarySuite) TestCanaryScaleDownDelay() {
Expand Down

0 comments on commit e184957

Please sign in to comment.