Merge pull request #307 from weaveworks/confirm-promotion

Implement confirm-promotion hook

Makefile

@@ -7,15 +7,8 @@ LT_VERSION?=$(shell grep 'VERSION' cmd/loadtester/main.go | awk '{ print $$4 }'
 TS=$(shell date +%Y-%m-%d_%H-%M-%S)
 
 run:
-	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test \
-	-metrics-server=https://prometheus.istio.weavedx.com \
-	-enable-leader-election=true
-
-run2:
-	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test \
-	-metrics-server=https://prometheus.istio.weavedx.com \
-	-enable-leader-election=true \
-	-port=9092
+	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=istio -namespace=test-istio \
+	-metrics-server=https://prometheus.istio.flagger.dev
 
 run-appmesh:
 	GO111MODULE=on go run cmd/flagger/* -kubeconfig=$$HOME/.kube/config -log-level=info -mesh-provider=appmesh \

artifacts/flagger/crd.yaml

@@ -215,7 +215,7 @@ spec:
                   properties:
                     items:
                       type: object
-                      required: ["name", "url", "timeout"]
+                      required: ["name", "url"]
                       properties:
                         name:
                           description: Name of the webhook
@@ -228,6 +228,7 @@ spec:
                             - confirm-rollout
                             - pre-rollout
                             - rollout
+                            - confirm-promotion
                             - post-rollout
                         url:
                           description: URL address of this webhook

charts/flagger/templates/crd.yaml

@@ -216,7 +216,7 @@ spec:
                   properties:
                     items:
                       type: object
-                      required: ['name', 'url', 'timeout']
+                      required: ["name", "url"]
                       properties:
                         name:
                           description: Name of the webhook
@@ -229,6 +229,7 @@ spec:
                             - confirm-rollout
                             - pre-rollout
                             - rollout
+                            - confirm-promotion
                             - post-rollout
                         url:
                           description: URL address of this webhook

docs/gitbook/how-it-works.md

@@ -663,6 +663,9 @@ The canary advancement is paused if a pre-rollout hook fails and if the number o
 threshold the canary will be rollback.
 * Rollout hooks are executed during the analysis on each iteration before the metric checks. 
 If a rollout hook call fails the canary advancement is paused and eventfully rolled back.
+* Confirm-promotion hooks are executed before the promotion step.
+The canary promotion is paused until the hooks return HTTP 200.
+While the promotion is paused, Flagger will continue to run the metrics checks and rollout hooks.
 * Post-rollout hooks are executed after the canary has been promoted or rolled back. 
 If a post rollout hook fails the error is logged.
 
@@ -687,6 +690,9 @@ Spec:
         timeout: 15s
         metadata:
           cmd: "hey -z 1m -q 5 -c 2 http://podinfo-canary.test:9898/"
+      - name: "promotion gate"
+        type: confirm-promotion
+        url: http://flagger-loadtester.test/gate/approve
       - name: "notify"
         type: post-rollout
         url: http://telegram.bot:8080/
@@ -914,8 +920,8 @@ Note that you should create a ConfigMap with your Bats tests and mount it inside
 
 ### Manual Gating
 
-For manual approval of a canary deployment you can use the `confirm-rollout` webhook. 
-The confirmation hooks are executed before the pre-rollout hooks. 
+For manual approval of a canary deployment you can use the `confirm-rollout` and `confirm-promotion` webhooks. 
+The confirmation rollout hooks are executed before the pre-rollout hooks. 
 Flagger will halt the canary traffic shifting and analysis until the confirm webhook returns HTTP status 200.
 
 Manual gating with Flagger's tester:
@@ -974,3 +980,16 @@ kubectl get canary/podinfo
 NAME      STATUS        WEIGHT
 podinfo   Waiting       0
 ```
+
+The `confirm-promotion` hook type can be used to manually approve the canary promotion.
+While the promotion is paused, Flagger will continue to run the metrics checks and load tests.
+
+```yaml
+  canaryAnalysis:
+    webhooks:
+      - name: "promotion gate"
+        type: confirm-promotion
+        url: http://flagger-loadtester.test/gate/halt
+```
+
+If you have notifications enabled, Flagger will post a message to Slack or MS Teams if a canary promotion is waiting for approval.

kustomize/base/flagger/crd.yaml

@@ -215,7 +215,7 @@ spec:
                   properties:
                     items:
                       type: object
-                      required: ["name", "url", "timeout"]
+                      required: ["name", "url"]
                       properties:
                         name:
                           description: Name of the webhook
@@ -228,6 +228,7 @@ spec:
                             - confirm-rollout
                             - pre-rollout
                             - rollout
+                            - confirm-promotion
                             - post-rollout
                         url:
                           description: URL address of this webhook

pkg/apis/flagger/v1alpha3/types.go

@@ -139,6 +139,8 @@ const (
 	PostRolloutHook HookType = "post-rollout"
 	// ConfirmRolloutHook halt canary analysis until webhook returns HTTP 200
 	ConfirmRolloutHook HookType = "confirm-rollout"
+	// ConfirmPromotionHook halt canary promotion until webhook returns HTTP 200
+	ConfirmPromotionHook HookType = "confirm-promotion"
 )
 
 // CanaryWebhook holds the reference to external checks used for canary analysis

pkg/controller/scheduler.go

@@ -323,6 +323,11 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 			return
 		}
 
+		// check promotion gate
+		if promote := c.runConfirmPromotionHooks(cd); !promote {
+			return
+		}
+
 		// promote canary - max iterations reached
 		if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations {
 			c.recordEventInfof(cd, "Copying %s.%s template spec to %s.%s",
@@ -375,6 +380,11 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 			return
 		}
 
+		// check promotion gate
+		if promote := c.runConfirmPromotionHooks(cd); !promote {
+			return
+		}
+
 		// route all traffic to canary - max iterations reached
 		if cd.Spec.CanaryAnalysis.Iterations == cd.Status.Iterations {
 			if provider != "kubernetes" {
@@ -445,6 +455,13 @@ func (c *Controller) advanceCanary(name string, namespace string, skipLivenessCh
 			primaryWeight = 100
 		}
 
+		// check promotion gate
+		if canaryWeight >= maxWeight {
+			if promote := c.runConfirmPromotionHooks(cd); !promote {
+				return
+			}
+		}
+
 		if err := meshRouter.SetRoutes(cd, primaryWeight, canaryWeight); err != nil {
 			c.recordEventWarningf(cd, "%v", err)
 			return
@@ -637,6 +654,23 @@ func (c *Controller) runConfirmRolloutHooks(canary *flaggerv1.Canary) bool {
 	return true
 }
 
+func (c *Controller) runConfirmPromotionHooks(canary *flaggerv1.Canary) bool {
+	for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
+		if webhook.Type == flaggerv1.ConfirmPromotionHook {
+			err := CallWebhook(canary.Name, canary.Namespace, flaggerv1.CanaryPhaseProgressing, webhook)
+			if err != nil {
+				c.recordEventWarningf(canary, "Halt %s.%s advancement waiting for promotion approval %s",
+					canary.Name, canary.Namespace, webhook.Name)
+				c.sendNotification(canary, "Canary promotion is waiting for approval.", false, false)
+				return false
+			} else {
+				c.recordEventInfof(canary, "Confirm-promotion check %s passed", webhook.Name)
+			}
+		}
+	}
+	return true
+}
+
 func (c *Controller) runPreRolloutHooks(canary *flaggerv1.Canary) bool {
 	for _, webhook := range canary.Spec.CanaryAnalysis.Webhooks {
 		if webhook.Type == flaggerv1.PreRolloutHook {

test/e2e-tests.sh

@@ -257,6 +257,9 @@ spec:
           type: cmd
           cmd: "hey -z 10m -q 10 -c 2 -H 'Cookie: type=insider' http://podinfo-canary.test:9898/"
           logCmdOutput: "true"
+      - name: promote-gate
+        type: confirm-promotion
+        url: http://flagger-loadtester.test/gate/approve
       - name: post
         type: post-rollout
         url: http://flagger-loadtester.test/