Enable session affinity for canaries (#7371)

build/run-in-docker.sh

@@ -25,6 +25,9 @@ set -o pipefail
 # temporal directory for the /etc/ingress-controller directory
 INGRESS_VOLUME=$(mktemp -d)
 
+# make sure directory for SSL cert storage exists under ingress volume
+mkdir "${INGRESS_VOLUME}/ssl"
+
 if [[ "$OSTYPE" == darwin* ]]; then
   INGRESS_VOLUME=/private$INGRESS_VOLUME
 fi

docs/examples/affinity/cookie/README.md

@@ -10,6 +10,7 @@ Session affinity can be configured using the following annotations:
 | --- | --- | --- |
 |nginx.ingress.kubernetes.io/affinity|Type of the affinity, set this to `cookie` to enable session affinity|string (NGINX only supports `cookie`)|
 |nginx.ingress.kubernetes.io/affinity-mode|The affinity mode defines how sticky a session is. Use `balanced` to redistribute some sessions when scaling pods or `persistent` for maximum stickiness.|`balanced` (default) or `persistent`|
+|nginx.ingress.kubernetes.io/affinity-canary-behavior|Defines session affinity behavior of canaries. By default the behavior is `sticky`, and canaries respect session affinity configuration. Set this to `legacy` to restore original canary behavior, when session affinity parameters were not respected.|`sticky` (default) or `legacy`|
 |nginx.ingress.kubernetes.io/session-cookie-name|Name of the cookie that will be created|string (defaults to `INGRESSCOOKIE`)|
 |nginx.ingress.kubernetes.io/session-cookie-path|Path that will be set on the cookie (required if your [Ingress paths][ingress-paths] use regular expressions)|string (defaults to the currently [matched path][ingress-paths])|
 |nginx.ingress.kubernetes.io/session-cookie-samesite|SameSite attribute to apply to the cookie|Browser accepted values are `None`, `Lax`, and `Strict`|

docs/user-guide/nginx-configuration/annotations.md

@@ -18,6 +18,7 @@ You can add these Kubernetes annotations to specific Ingress objects to customiz
 |[nginx.ingress.kubernetes.io/app-root](#rewrite)|string|
 |[nginx.ingress.kubernetes.io/affinity](#session-affinity)|cookie|
 |[nginx.ingress.kubernetes.io/affinity-mode](#session-affinity)|"balanced" or "persistent"|
+|[nginx.ingress.kubernetes.io/affinity-canary-behavior](#session-affinity)|"sticky" or "legacy"|
 |[nginx.ingress.kubernetes.io/auth-realm](#authentication)|string|
 |[nginx.ingress.kubernetes.io/auth-secret](#authentication)|string|
 |[nginx.ingress.kubernetes.io/auth-secret-type](#authentication)|string|
@@ -140,7 +141,7 @@ In some cases, you may want to "canary" a new set of changes by sending a small
 Canary rules are evaluated in order of precedence. Precedence is as follows:
 `canary-by-header -> canary-by-cookie -> canary-weight`
 
-**Note** that when you mark an ingress as canary, then all the other non-canary annotations will be ignored (inherited from the corresponding main ingress) except `nginx.ingress.kubernetes.io/load-balance` and `nginx.ingress.kubernetes.io/upstream-hash-by`.
+**Note** that when you mark an ingress as canary, then all the other non-canary annotations will be ignored (inherited from the corresponding main ingress) except `nginx.ingress.kubernetes.io/load-balance`, `nginx.ingress.kubernetes.io/upstream-hash-by`, and [annotations related to session affinity](#session-affinity). If you want to restore the original behavior of canaries when session affinity was ignored, set `nginx.ingress.kubernetes.io/affinity-canary-behavior` annotation with value `legacy` on the non-canary ingress definition.
 
 **Known Limitations**
 
@@ -163,6 +164,8 @@ The only affinity type available for NGINX is `cookie`.
 
 The annotation `nginx.ingress.kubernetes.io/affinity-mode` defines the stickiness of a session. Setting this to `balanced` (default) will redistribute some sessions if a deployment gets scaled up, therefore rebalancing the load on the servers. Setting this to `persistent` will not rebalance sessions to new servers, therefore providing maximum stickiness.
 
+The annotation `nginx.ingress.kubernetes.io/affinity-canary-behavior` defines the behavior of canaries when session affinity is enabled. Setting this to `sticky` (default) will ensure that users that were served by canaries, will continue to be served by canaries. Setting this to `legacy` will restore original canary behavior, when session affinity was ignored.
+
 !!! attention
     If more than one Ingress is defined for a host and at least one Ingress uses `nginx.ingress.kubernetes.io/affinity: cookie`, then only paths on the Ingress using `nginx.ingress.kubernetes.io/affinity` will use session cookie affinity. All paths defined on other Ingresses for the host will be load balanced through the random selection of a backend server.
 
@@ -342,7 +345,7 @@ CORS can be controlled with the following annotations:
   - Example: `nginx.ingress.kubernetes.io/cors-allow-headers: "X-Forwarded-For, X-app123-XPTO"`
 
 * `nginx.ingress.kubernetes.io/cors-expose-headers`
-  controls which headers are exposed to response. This is a multi-valued field, separated by ',' and accepts 
+  controls which headers are exposed to response. This is a multi-valued field, separated by ',' and accepts
   letters, numbers, _, - and *.
   - Default: *empty*
   - Example: `nginx.ingress.kubernetes.io/cors-expose-headers: "*, X-CustomResponseHeader"`

internal/ingress/annotations/annotations_test.go

@@ -30,19 +30,20 @@ import (
 )
 
 var (
-	annotationPassthrough          = parser.GetAnnotationWithPrefix("ssl-passthrough")
-	annotationAffinityType         = parser.GetAnnotationWithPrefix("affinity")
-	annotationAffinityMode         = parser.GetAnnotationWithPrefix("affinity-mode")
-	annotationCorsEnabled          = parser.GetAnnotationWithPrefix("enable-cors")
-	annotationCorsAllowMethods     = parser.GetAnnotationWithPrefix("cors-allow-methods")
-	annotationCorsAllowHeaders     = parser.GetAnnotationWithPrefix("cors-allow-headers")
-	annotationCorsExposeHeaders    = parser.GetAnnotationWithPrefix("cors-expose-headers")
-	annotationCorsAllowCredentials = parser.GetAnnotationWithPrefix("cors-allow-credentials")
-	defaultCorsMethods             = "GET, PUT, POST, DELETE, PATCH, OPTIONS"
-	defaultCorsHeaders             = "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization"
-	annotationAffinityCookieName   = parser.GetAnnotationWithPrefix("session-cookie-name")
-	annotationUpstreamHashBy       = parser.GetAnnotationWithPrefix("upstream-hash-by")
-	annotationCustomHTTPErrors     = parser.GetAnnotationWithPrefix("custom-http-errors")
+	annotationPassthrough            = parser.GetAnnotationWithPrefix("ssl-passthrough")
+	annotationAffinityType           = parser.GetAnnotationWithPrefix("affinity")
+	annotationAffinityMode           = parser.GetAnnotationWithPrefix("affinity-mode")
+	annotationAffinityCanaryBehavior = parser.GetAnnotationWithPrefix("affinity-canary-behavior")
+	annotationCorsEnabled            = parser.GetAnnotationWithPrefix("enable-cors")
+	annotationCorsAllowMethods       = parser.GetAnnotationWithPrefix("cors-allow-methods")
+	annotationCorsAllowHeaders       = parser.GetAnnotationWithPrefix("cors-allow-headers")
+	annotationCorsExposeHeaders      = parser.GetAnnotationWithPrefix("cors-expose-headers")
+	annotationCorsAllowCredentials   = parser.GetAnnotationWithPrefix("cors-allow-credentials")
+	defaultCorsMethods               = "GET, PUT, POST, DELETE, PATCH, OPTIONS"
+	defaultCorsHeaders               = "DNT,X-CustomHeader,Keep-Alive,User-Agent,X-Requested-With,If-Modified-Since,Cache-Control,Content-Type,Authorization"
+	annotationAffinityCookieName     = parser.GetAnnotationWithPrefix("session-cookie-name")
+	annotationUpstreamHashBy         = parser.GetAnnotationWithPrefix("upstream-hash-by")
+	annotationCustomHTTPErrors       = parser.GetAnnotationWithPrefix("custom-http-errors")
 )
 
 type mockCfg struct {
@@ -162,29 +163,38 @@ func TestAffinitySession(t *testing.T) {
 	ing := buildIngress()
 
 	fooAnns := []struct {
-		annotations  map[string]string
-		affinitytype string
-		affinitymode string
-		name         string
+		annotations    map[string]string
+		affinitytype   string
+		affinitymode   string
+		cookiename     string
+		canarybehavior string
 	}{
-		{map[string]string{annotationAffinityType: "cookie", annotationAffinityMode: "balanced", annotationAffinityCookieName: "route"}, "cookie", "balanced", "route"},
-		{map[string]string{annotationAffinityType: "cookie", annotationAffinityMode: "persistent", annotationAffinityCookieName: "route1"}, "cookie", "persistent", "route1"},
-		{map[string]string{annotationAffinityType: "cookie", annotationAffinityMode: "balanced", annotationAffinityCookieName: ""}, "cookie", "balanced", "INGRESSCOOKIE"},
-		{map[string]string{}, "", "", ""},
-		{nil, "", "", ""},
+		{map[string]string{annotationAffinityType: "cookie", annotationAffinityMode: "balanced", annotationAffinityCookieName: "route", annotationAffinityCanaryBehavior: ""}, "cookie", "balanced", "route", ""},
+		{map[string]string{annotationAffinityType: "cookie", annotationAffinityMode: "persistent", annotationAffinityCookieName: "route1", annotationAffinityCanaryBehavior: "sticky"}, "cookie", "persistent", "route1", "sticky"},
+		{map[string]string{annotationAffinityType: "cookie", annotationAffinityMode: "balanced", annotationAffinityCookieName: "", annotationAffinityCanaryBehavior: "legacy"}, "cookie", "balanced", "INGRESSCOOKIE", "legacy"},
+		{map[string]string{}, "", "", "", ""},
+		{nil, "", "", "", ""},
 	}
 
 	for _, foo := range fooAnns {
 		ing.SetAnnotations(foo.annotations)
 		r := ec.Extract(ing).SessionAffinity
-		t.Logf("Testing pass %v %v", foo.affinitytype, foo.name)
+		t.Logf("Testing pass %v %v", foo.affinitytype, foo.cookiename)
+
+		if r.Type != foo.affinitytype {
+			t.Errorf("Returned %v but expected %v for Type", r.Type, foo.affinitytype)
+		}
 
 		if r.Mode != foo.affinitymode {
-			t.Errorf("Returned %v but expected %v for Name", r.Mode, foo.affinitymode)
+			t.Errorf("Returned %v but expected %v for Mode", r.Mode, foo.affinitymode)
+		}
+
+		if r.CanaryBehavior != foo.canarybehavior {
+			t.Errorf("Returned %v but expected %v for CanaryBehavior", r.CanaryBehavior, foo.canarybehavior)
 		}
 
-		if r.Cookie.Name != foo.name {
-			t.Errorf("Returned %v but expected %v for Name", r.Cookie.Name, foo.name)
+		if r.Cookie.Name != foo.cookiename {
+			t.Errorf("Returned %v but expected %v for Cookie.Name", r.Cookie.Name, foo.cookiename)
 		}
 	}
 }

internal/ingress/annotations/sessionaffinity/main.go

@@ -27,8 +27,10 @@ import (
 )
 
 const (
-	annotationAffinityType = "affinity"
-	annotationAffinityMode = "affinity-mode"
+	annotationAffinityType           = "affinity"
+	annotationAffinityMode           = "affinity-mode"
+	annotationAffinityCanaryBehavior = "affinity-canary-behavior"
+
 	// If a cookie with this name exists,
 	// its value is used as an index into the list of available backends.
 	annotationAffinityCookieName = "session-cookie-name"
@@ -66,6 +68,8 @@ type Config struct {
 	Type string `json:"type"`
 	// The affinity mode, i.e. how sticky a session is
 	Mode string `json:"mode"`
+	// Affinity behavior for canaries (sticky or legacy)
+	CanaryBehavior string `json:"canaryBehavior"`
 	Cookie
 }
 
@@ -160,6 +164,11 @@ func (a affinity) Parse(ing *networking.Ingress) (interface{}, error) {
 		am = ""
 	}
 
+	cb, err := parser.GetStringAnnotation(annotationAffinityCanaryBehavior, ing)
+	if err != nil {
+		cb = ""
+	}
+
 	switch at {
 	case "cookie":
 		cookie = a.cookieAffinityParse(ing)
@@ -169,8 +178,9 @@ func (a affinity) Parse(ing *networking.Ingress) (interface{}, error) {
 	}
 
 	return &Config{
-		Type:   at,
-		Mode:   am,
-		Cookie: *cookie,
+		Type:           at,
+		Mode:           am,
+		CanaryBehavior: cb,
+		Cookie:         *cookie,
 	}, nil
 }

internal/ingress/controller/controller.go

@@ -1315,7 +1315,7 @@ func canMergeBackend(primary *ingress.Backend, alternative *ingress.Backend) boo
 }
 
 // Performs the merge action and checks to ensure that one two alternative backends do not merge into each other
-func mergeAlternativeBackend(priUps *ingress.Backend, altUps *ingress.Backend) bool {
+func mergeAlternativeBackend(ing *ingress.Ingress, priUps *ingress.Backend, altUps *ingress.Backend) bool {
 	if priUps.NoServer {
 		klog.Warningf("unable to merge alternative backend %v into primary backend %v because %v is a primary backend",
 			altUps.Name, priUps.Name, priUps.Name)
@@ -1329,6 +1329,10 @@ func mergeAlternativeBackend(priUps *ingress.Backend, altUps *ingress.Backend) b
 		}
 	}
 
+	if ing.ParsedAnnotations != nil && ing.ParsedAnnotations.SessionAffinity.CanaryBehavior != "legacy" {
+		priUps.SessionAffinity.DeepCopyInto(&altUps.SessionAffinity)
+	}
+
 	priUps.AlternativeBackends =
 		append(priUps.AlternativeBackends, altUps.Name)
 
@@ -1368,7 +1372,7 @@ func mergeAlternativeBackends(ing *ingress.Ingress, upstreams map[string]*ingres
 					klog.V(2).Infof("matching backend %v found for alternative backend %v",
 						priUps.Name, altUps.Name)
 
-					merged = mergeAlternativeBackend(priUps, altUps)
+					merged = mergeAlternativeBackend(ing, priUps, altUps)
 				}
 			}
 
@@ -1421,7 +1425,7 @@ func mergeAlternativeBackends(ing *ingress.Ingress, upstreams map[string]*ingres
 					klog.V(2).Infof("matching backend %v found for alternative backend %v",
 						priUps.Name, altUps.Name)
 
-					merged = mergeAlternativeBackend(priUps, altUps)
+					merged = mergeAlternativeBackend(ing, priUps, altUps)
 				}
 			}