Add bluegreen service selector management

argoproj · Dec 27, 2018 · ddc7368 · ddc7368
1 parent e92ffb0
commit ddc7368
Show file tree

Hide file tree

Showing 7 changed files with 509 additions and 16 deletions.
diff --git a/controller/bluegreen.go b/controller/bluegreen.go
@@ -5,6 +5,7 @@ import (
 	"sort"
 
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	"k8s.io/klog"
 	"k8s.io/kubernetes/pkg/controller"
 
@@ -29,6 +30,32 @@ func (c *Controller) rolloutBlueGreen(r *v1alpha1.Rollout, rsList []*appsv1.Repl
 	if scaledUp {
 		return nil
 	}
+	previewSvc, activeSvc, err := c.getPreviewAndActiveServices(r)
+	if err != nil {
+		return err
+	}
+	if previewSvc != nil {
+		switchPreviewSvc, err := c.reconcilePreviewService(r, newRS, previewSvc, activeSvc)
+		if err != nil {
+			return err
+		}
+		if switchPreviewSvc {
+			return nil
+		}
+
+		verfyingPreview := c.reconcileVerifyingPreview(activeSvc, r)
+		if verfyingPreview {
+			return nil
+		}
+	}
+
+	switchActiveSvc, err := c.reconcileActiveService(r, newRS, previewSvc, activeSvc)
+	if err != nil {
+		return err
+	}
+	if switchActiveSvc {
+		return nil
+	}
 	// Scale down, if we can.
 	scaledDown, err := c.reconcileOldReplicaSets(allRSs, controller.FilterActiveReplicaSets(oldRSs), newRS, r)
 	if err != nil {
@@ -41,6 +68,21 @@ func (c *Controller) rolloutBlueGreen(r *v1alpha1.Rollout, rsList []*appsv1.Repl
 	return nil
 }
 
+func (c *Controller) reconcileVerifyingPreview(activeSvc *corev1.Service, rollout *v1alpha1.Rollout) bool {
+	if rollout.Spec.Strategy.BlueGreenStrategy.PreviewService == "" {
+		return false
+	}
+	if _, ok := activeSvc.Spec.Selector[v1alpha1.DefaultRolloutUniqueLabelKey]; !ok {
+		return false
+	}
+
+	if rollout.Status.VerifyingPreview == nil {
+		return false
+	}
+
+	return *rollout.Status.VerifyingPreview
+}
+
 func (c *Controller) reconcileNewReplicaSet(allRSs []*appsv1.ReplicaSet, newRS *appsv1.ReplicaSet, rollout *v1alpha1.Rollout) (bool, error) {
 	if *(newRS.Spec.Replicas) == *(rollout.Spec.Replicas) {
 		// Scaling not required.
@@ -70,6 +112,7 @@ func (c *Controller) reconcileOldReplicaSets(allRSs []*appsv1.ReplicaSet, oldRSs
 	if !annotations.IsSaturated(rollout, newRS) {
 		return false, nil
 	}
+	// Add check for active service
 
 	// Clean up unhealthy replicas first, otherwise unhealthy replicas will block deployment
 	// and cause timeout. See https://github.com/kubernetes/kubernetes/issues/16737
@@ -154,3 +197,10 @@ func (c *Controller) scaleDownOldReplicaSetsForBlueGreen(allRSs []*appsv1.Replic
 
 	return totalScaledDown, nil
 }
+
+func (c *Controller) setVerifyingPreview(r *v1alpha1.Rollout) error {
+	verifyPreprod := true
+	r.Status.VerifyingPreview = &verifyPreprod
+	_, err := c.rolloutsclientset.ArgoprojV1alpha1().Rollouts(r.Namespace).Update(r)
+	return err
+}
diff --git a/controller/bluegreen_test.go b/controller/bluegreen_test.go
@@ -4,12 +4,15 @@ import (
 	"strconv"
 	"testing"
 
+	"github.com/stretchr/testify/assert"
 	appsv1 "k8s.io/api/apps/v1"
+	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	k8sfake "k8s.io/client-go/kubernetes/fake"
 	core "k8s.io/client-go/testing"
 	"k8s.io/client-go/tools/record"
 
+	"github.com/argoproj/argo-rollouts/pkg/apis/rollouts/v1alpha1"
 	"github.com/argoproj/argo-rollouts/pkg/client/clientset/versioned/fake"
 	"github.com/argoproj/argo-rollouts/utils/annotations"
 )
@@ -18,6 +21,69 @@ var (
 	noTimestamp = metav1.Time{}
 )
 
+func TestController_reconcileVerifyingPreview(t *testing.T) {
+	boolPtr := func(boolean bool) *bool { return &boolean }
+	tests := []struct {
+		name                 string
+		activeSvc            *corev1.Service
+		previewSvcName       string
+		verifyingPreviewFlag *bool
+		notFinishedVerifying bool
+	}{
+		{
+			name:                 "Continue if preview Service isn't specificed",
+			activeSvc:            newService("active", 80, nil),
+			verifyingPreviewFlag: boolPtr(true),
+			notFinishedVerifying: false,
+		},
+		{
+			name:                 "Continue if active service doesn't have a selector from the rollout",
+			previewSvcName:       "previewSvc",
+			activeSvc:            newService("active", 80, nil),
+			verifyingPreviewFlag: boolPtr(true),
+			notFinishedVerifying: false,
+		},
+		{
+			name:                 "Do not continue if verifyingPreview flag is true",
+			previewSvcName:       "previewSvc",
+			activeSvc:            newService("active", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: "test"}),
+			verifyingPreviewFlag: boolPtr(true),
+			notFinishedVerifying: true,
+		},
+		{
+			name:                 "Continue if verifyingPreview flag is false",
+			previewSvcName:       "previewSvc",
+			activeSvc:            newService("active", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: "test"}),
+			verifyingPreviewFlag: boolPtr(false),
+			notFinishedVerifying: false,
+		},
+		{
+			name:                 "Continue if verifyingPreview flag is not set",
+			previewSvcName:       "previewSvc",
+			activeSvc:            newService("active", 80, map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: "test"}),
+			notFinishedVerifying: false,
+		},
+	}
+	for i := range tests {
+		test := tests[i]
+		t.Run(test.name, func(t *testing.T) {
+			rollout := newRollout("foo", 1, nil, map[string]string{"foo": "bar"}, "", test.previewSvcName)
+			rollout.Status = v1alpha1.RolloutStatus{
+				VerifyingPreview: test.verifyingPreviewFlag,
+			}
+			fake := fake.Clientset{}
+			k8sfake := k8sfake.Clientset{}
+			controller := &Controller{
+				rolloutsclientset: &fake,
+				kubeclientset:     &k8sfake,
+				recorder:          &record.FakeRecorder{},
+			}
+			finishedVerifying := controller.reconcileVerifyingPreview(test.activeSvc, rollout)
+			assert.Equal(t, test.notFinishedVerifying, finishedVerifying)
+		})
+	}
+}
+
 func TestController_reconcileNewReplicaSet(t *testing.T) {
 	tests := []struct {
 		name                string

diff --git a/controller/controller_test.go b/controller/controller_test.go
@@ -314,10 +314,81 @@ func TestSyncRolloutCreatesReplicaSet(t *testing.T) {
 	r := newRollout("foo", 1, nil, map[string]string{"foo": "bar"}, "bar", "")
 	f.rolloutLister = append(f.rolloutLister, r)
 	f.objects = append(f.objects, r)
+	s := newService("bar", 80, nil)
+	f.kubeobjects = append(f.kubeobjects, s)
 
 	rs := newReplicaSet(r, "foo-895c6c4f9", 1)
 
 	f.expectCreateReplicaSetAction(rs)
+	f.expectGetServiceAction(s)
+	f.run(getKey(r, t))
+}
+
+func TestSyncRolloutSetPreviewService(t *testing.T) {
+	f := newFixture(t)
+
+	r := newRollout("foo", 1, nil, map[string]string{"foo": "bar"}, "active", "preview")
+	f.rolloutLister = append(f.rolloutLister, r)
+	f.objects = append(f.objects, r)
+
+	rs := newReplicaSetWithStatus(r, "foo-895c6c4f9", 1, 1)
+	f.kubeobjects = append(f.kubeobjects, rs)
+	f.replicaSetLister = append(f.replicaSetLister, rs)
+
+	previewSvc := newService("preview", 80, nil)
+	selector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: "test"}
+	activeSvc := newService("active", 80, selector)
+	f.kubeobjects = append(f.kubeobjects, previewSvc, activeSvc)
+
+	f.expectGetServiceAction(activeSvc)
+	f.expectGetServiceAction(previewSvc)
+	f.expectPatchServiceAction(previewSvc, rs)
+	f.expectUpdateRolloutAction(r)
+	f.run(getKey(r, t))
+}
+
+func TestSyncRolloutVerifyPreviewNoActions(t *testing.T) {
+	f := newFixture(t)
+
+	r := newRollout("foo", 1, nil, map[string]string{"foo": "bar"}, "active", "preview")
+	r.Status.VerifyingPreview = func(boolean bool) *bool { return &boolean }(true)
+	f.rolloutLister = append(f.rolloutLister, r)
+	f.objects = append(f.objects, r)
+
+	rs := newReplicaSetWithStatus(r, "foo-895c6c4f9", 1, 1)
+	rs2 := newImage(rs, "foo/bar2.0")
+	f.kubeobjects = append(f.kubeobjects, rs, rs2)
+	f.replicaSetLister = append(f.replicaSetLister, rs, rs2)
+
+	previewSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: "895c6c4f9"}
+	previewSvc := newService("preview", 80, previewSelector)
+	activeSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: rs2.Name}
+	activeSvc := newService("active", 80, activeSelector)
+	f.kubeobjects = append(f.kubeobjects, previewSvc, activeSvc)
+
+	f.expectGetServiceAction(activeSvc)
+	f.expectGetServiceAction(previewSvc)
+	f.run(getKey(r, t))
+}
+
+func TestSyncRolloutSkipPreviewUpdateActive(t *testing.T) {
+	f := newFixture(t)
+
+	r := newRollout("foo", 1, nil, map[string]string{"foo": "bar"}, "active", "preview")
+	f.rolloutLister = append(f.rolloutLister, r)
+	f.objects = append(f.objects, r)
+
+	rs := newReplicaSetWithStatus(r, "foo-895c6c4f9", 1, 1)
+	f.kubeobjects = append(f.kubeobjects, rs)
+	f.replicaSetLister = append(f.replicaSetLister, rs)
+
+	previewSvc := newService("preview", 80, nil)
+	activeSvc := newService("active", 80, nil)
+	f.kubeobjects = append(f.kubeobjects, previewSvc, activeSvc)
+
+	f.expectGetServiceAction(activeSvc)
+	f.expectGetServiceAction(previewSvc)
+	f.expectPatchServiceAction(activeSvc, rs)
 	f.run(getKey(r, t))
 }
 
@@ -350,8 +421,13 @@ func TestSyncRolloutsScaleDownOldRS(t *testing.T) {
 	f.kubeobjects = append(f.kubeobjects, rs2)
 	f.replicaSetLister = append(f.replicaSetLister, rs2)
 
+	serviceSelector := map[string]string{v1alpha1.DefaultRolloutUniqueLabelKey: "6479c8f85c"}
+	s := newService("bar", 80, serviceSelector)
+	f.kubeobjects = append(f.kubeobjects, s)
+
 	expRS := rs2.DeepCopy()
 	expRS.Annotations[annotations.DesiredReplicasAnnotation] = "0"
+	f.expectGetServiceAction(s)
 	f.expectUpdateReplicaSetAction(expRS)
 
 	f.run(getKey(r2, t))