Experimental feature: User-specified node labels for worker nodes

This complements Node Pools(kubernetes-retired#46) and Spot Fleet support(kubernetes-retired#112) The former `experimental.nodeLabel` configuration key is renamed to `experimental.awsNodeLabels` to avoid collision with newly added `experimental.nodeLabels` and consistency with `experimental.awsEnvironment`.
HotelsDotCom · Dec 14, 2016 · 0c980e3 · 0c980e3
1 parent 99c3e50
commit 0c980e3
Show file tree

Hide file tree

Showing 10 changed files with 84 additions and 47 deletions.
diff --git a/config/config.go b/config/config.go
@@ -37,6 +37,9 @@ func NewDefaultCluster() *Cluster {
 		AwsEnvironment{
 			Enabled: false,
 		},
+		AwsNodeLabels{
+			Enabled: false,
+		},
 		EphemeralImageStorage{
 			Enabled:    false,
 			Disk:       "xvdb",
@@ -48,9 +51,7 @@ func NewDefaultCluster() *Cluster {
 		NodeDrainer{
 			Enabled: false,
 		},
-		NodeLabel{
-			Enabled: false,
-		},
+		NodeLabels{},
 		Plugins{
 			Rbac{
 				Enabled: false,
@@ -296,10 +297,11 @@ type Subnet struct {
 type Experimental struct {
 	AuditLog              AuditLog              `yaml:"auditLog"`
 	AwsEnvironment        AwsEnvironment        `yaml:"awsEnvironment"`
+	AwsNodeLabels         AwsNodeLabels         `yaml:"awsNodeLabels"`
 	EphemeralImageStorage EphemeralImageStorage `yaml:"ephemeralImageStorage"`
 	LoadBalancer          LoadBalancer          `yaml:"loadBalancer"`
 	NodeDrainer           NodeDrainer           `yaml:"nodeDrainer"`
-	NodeLabel             NodeLabel             `yaml:"nodeLabel"`
+	NodeLabels            NodeLabels            `yaml:"nodeLabels"`
 	Plugins               Plugins               `yaml:"plugins"`
 	Taints                []Taint               `yaml:"taints"`
 	WaitSignal            WaitSignal            `yaml:"waitSignal"`
@@ -316,6 +318,10 @@ type AuditLog struct {
 	LogPath string `yaml:"logpath"`
 }
 
+type AwsNodeLabels struct {
+	Enabled bool `yaml:"enabled"`
+}
+
 type EphemeralImageStorage struct {
 	Enabled    bool   `yaml:"enabled"`
 	Disk       string `yaml:"disk"`
@@ -326,8 +332,19 @@ type NodeDrainer struct {
 	Enabled bool `yaml:"enabled"`
 }
 
-type NodeLabel struct {
-	Enabled bool `yaml:"enabled"`
+type NodeLabels map[string]string
+
+func (l NodeLabels) Enabled() bool {
+	return len(l) > 0
+}
+
+// Returns key=value pairs separated by ',' to be passed to kubelet's `--node-labels` flag
+func (l NodeLabels) String() string {
+	labels := []string{}
+	for k, v := range l {
+		labels = append(labels, fmt.Sprintf("%s=%s", k, v))
+	}
+	return strings.Join(labels, ",")
 }
 
 type LoadBalancer struct {

diff --git a/config/config_test.go b/config/config_test.go
@@ -1010,6 +1010,9 @@ func TestConfig(t *testing.T) {
 			AwsEnvironment: AwsEnvironment{
 				Enabled: false,
 			},
+			AwsNodeLabels: AwsNodeLabels{
+				Enabled: false,
+			},
 			EphemeralImageStorage: EphemeralImageStorage{
 				Enabled:    false,
 				Disk:       "xvdb",
@@ -1021,10 +1024,8 @@ func TestConfig(t *testing.T) {
 			NodeDrainer: NodeDrainer{
 				Enabled: false,
 			},
-			NodeLabel: NodeLabel{
-				Enabled: false,
-			},
-			Taints: []Taint{},
+			NodeLabels: NodeLabels{},
+			Taints:     []Taint{},
 			WaitSignal: WaitSignal{
 				Enabled:      false,
 				MaxBatchSize: 1,
@@ -1058,6 +1059,8 @@ experimental:
     enabled: true
     environment:
       CFNSTACK: '{ "Ref" : "AWS::StackId" }'
+  awsNodeLabels:
+    enabled: true
   ephemeralImageStorage:
     enabled: true
   loadBalancer:
@@ -1068,8 +1071,8 @@ experimental:
       - sg-12345678
   nodeDrainer:
     enabled: true
-  nodeLabel:
-    enabled: true
+  nodeLabels:
+    kube-aws.coreos.com/role: worker
   plugins:
     rbac:
       enabled: true
@@ -1095,6 +1098,9 @@ experimental:
 								"CFNSTACK": `{ "Ref" : "AWS::StackId" }`,
 							},
 						},
+						AwsNodeLabels: AwsNodeLabels{
+							Enabled: true,
+						},
 						EphemeralImageStorage: EphemeralImageStorage{
 							Enabled:    true,
 							Disk:       "xvdb",
@@ -1108,8 +1114,8 @@ experimental:
 						NodeDrainer: NodeDrainer{
 							Enabled: true,
 						},
-						NodeLabel: NodeLabel{
-							Enabled: true,
+						NodeLabels: NodeLabels{
+							"kube-aws.coreos.com/role": "worker",
 						},
 						Plugins: Plugins{
 							Rbac: Rbac{

diff --git a/config/templates/cloud-config-controller b/config/templates/cloud-config-controller
@@ -260,7 +260,7 @@ coreos:
               --resource AutoScaleController \
               --stack {{.ClusterName}}
 {{end}}
-{{if .Experimental.NodeLabel.Enabled }}
+{{if .Experimental.AwsNodeLabels.Enabled }}
     - name: kube-node-label.service
       enable: true
       command: start

diff --git a/config/templates/cloud-config-worker b/config/templates/cloud-config-worker
@@ -70,7 +70,8 @@ coreos:
         --container-runtime={{.ContainerRuntime}} \
         --rkt-path=/usr/bin/rkt \
         --rkt-stage1-image=coreos.com/rkt/stage1-coreos \
-        --register-node=true \
+        {{if .Experimental.NodeLabels.Enabled}}--node-labels {{.Experimental.NodeLabels.String}} \
+        {{end}}--register-node=true \
         {{if .Experimental.Taints}}--register-schedulable=false \
         {{end}}--allow-privileged=true \
         --pod-manifest-path=/etc/kubernetes/manifests \
@@ -292,7 +293,7 @@ coreos:
               --stack {{.ClusterName}}
 {{end}}
 
-{{if .Experimental.NodeLabel.Enabled }}
+{{if .Experimental.AwsNodeLabels.Enabled }}
     - name: kube-node-label.service
       enable: true
       command: start

diff --git a/config/templates/cluster.yaml b/config/templates/cluster.yaml
@@ -210,26 +210,15 @@ worker:
 
 # Experimental features will change in backward-incompatible ways
 # experimental:
-#   nodeDrainer:
-#     enabled: true
-#   nodeLabel:
-#     enabled: true
 #   awsEnvironment:
 #     enabled: true
 #     environment:
 #       CFNSTACK: '{ "Ref" : "AWS::StackId" }'
-#   plugins:
-#     rbac:
-#       enabled: true
 #   auditLog:
 #     enabled: true
 #     maxage: 30
 #     logpath: /dev/stdout
-#   taints:
-#     - key: dedicated
-#       value: search
-#       effect: NoSchedule
-#   waitSignal:
+#   awsNodeLabels:
 #     enabled: true
 #   # This option has not yet been tested with rkt as container runtime
 #   ephemeralImageStorage:
@@ -238,6 +227,19 @@ worker:
 #     enabled: true
 #     names: [ "manuallymanagedelb" ]
 #     securityGroupIds: [ "sg-87654321" ]
+#   nodeDrainer:
+#     enabled: true
+#   nodeLabels:
+#     kube-aws.coreos.com/role: worker
+#   taints:
+#     - key: dedicated
+#       value: search
+#       effect: NoSchedule
+#   waitSignal:
+#     enabled: true
+#   plugins:
+#     rbac:
+#       enabled: true
 
 # AWS Tags for cloudformation stack resources
 #stackTags:

diff --git a/config/templates/stack-template.json b/config/templates/stack-template.json
@@ -255,7 +255,7 @@
                     ] }
                 },
                 {{end}}
-                {{if .Experimental.NodeLabel.Enabled}}
+                {{if .Experimental.AwsNodeLabels.Enabled}}
                 {
                   "Action": "autoscaling:Describe*",
                   "Effect": "Allow",
@@ -348,7 +348,7 @@
                     ] }
                 },
                 {{end}}
-                {{if .Experimental.NodeLabel.Enabled}}
+                {{if .Experimental.AwsNodeLabels.Enabled}}
                 {
                   "Action": "autoscaling:Describe*",
                   "Effect": "Allow",

diff --git a/e2e/run b/e2e/run
@@ -88,8 +88,11 @@ customize_worker() {
   if [ "${KUBE_AWS_WAIT_SIGNAL_ENABLED}" != "" ]; then
     echo -e '  waitSignal:\n    enabled: true' >> cluster.yaml
   fi
-  if [ "${KUBE_AWS_NODE_LABEL_ENABLED}" != "" ]; then
-    echo -e '  nodeLabel:\n    enabled: true' >> cluster.yaml
+  if [ "${KUBE_AWS_AWS_NODE_LABELS_ENABLED}" != "" ]; then
+    echo -e '  awsNodeLabels:\n    enabled: true' >> cluster.yaml
+  fi
+  if [ "${KUBE_AWS_NODE_LABELS_ENABLED}" != "" ]; then
+    echo -e '  nodeLabels:\n    kube-aws.coreos.com/role: worker' >> cluster.yaml
   fi
   if [ "${KUBE_AWS_AWS_ENV_ENABLED}" != "" ]; then
     echo -e "  awsEnvironment:\n    enabled: true\n    environment:\n      CFNSTACK: '{\"Ref\":\"AWS::StackId\"}'" >> cluster.yaml

diff --git a/nodepool/config/config_test.go b/nodepool/config/config_test.go
@@ -84,6 +84,9 @@ etcdEndpoints: "10.0.0.1"
 			AwsEnvironment: cfg.AwsEnvironment{
 				Enabled: false,
 			},
+			AwsNodeLabels: cfg.AwsNodeLabels{
+				Enabled: false,
+			},
 			EphemeralImageStorage: cfg.EphemeralImageStorage{
 				Enabled:    false,
 				Disk:       "xvdb",
@@ -95,10 +98,8 @@ etcdEndpoints: "10.0.0.1"
 			NodeDrainer: cfg.NodeDrainer{
 				Enabled: false,
 			},
-			NodeLabel: cfg.NodeLabel{
-				Enabled: false,
-			},
-			Taints: []cfg.Taint{},
+			NodeLabels: cfg.NodeLabels{},
+			Taints:     []cfg.Taint{},
 			WaitSignal: cfg.WaitSignal{
 				Enabled:      false,
 				MaxBatchSize: 1,
@@ -125,6 +126,8 @@ experimental:
     enabled: true
     environment:
       CFNSTACK: '{ "Ref" : "AWS::StackId" }'
+  awsNodeLabels:
+    enabled: true
   ephemeralImageStorage:
     enabled: true
   loadBalancer:
@@ -135,8 +138,8 @@ experimental:
       - sg-12345678
   nodeDrainer:
     enabled: true
-  nodeLabel:
-    enabled: true
+  nodeLabels:
+    kube-aws.coreos.com/role: worker
   taints:
     - key: reservation
       value: spot
@@ -159,6 +162,9 @@ experimental:
 								"CFNSTACK": `{ "Ref" : "AWS::StackId" }`,
 							},
 						},
+						AwsNodeLabels: cfg.AwsNodeLabels{
+							Enabled: true,
+						},
 						EphemeralImageStorage: cfg.EphemeralImageStorage{
 							Enabled:    true,
 							Disk:       "xvdb",
@@ -172,8 +178,8 @@ experimental:
 						NodeDrainer: cfg.NodeDrainer{
 							Enabled: true,
 						},
-						NodeLabel: cfg.NodeLabel{
-							Enabled: true,
+						NodeLabels: cfg.NodeLabels{
+							"kube-aws.coreos.com/role": "worker",
 						},
 						Taints: []cfg.Taint{
 							{Key: "reservation", Value: "spot", Effect: "NoSchedule"},

diff --git a/nodepool/config/templates/cluster.yaml b/nodepool/config/templates/cluster.yaml
@@ -180,20 +180,22 @@ useCalico: {{.UseCalico}}
 
 # Experimental features will change in backward-incompatible ways
 # experimental:
-#   nodeDrainer:
-#     enabled: true
-#   nodeLabel:
-#     enabled: true
 #   awsEnvironment:
 #     enabled: true
 #     environment:
 #       CFNSTACK: '{ "Ref" : "AWS::StackId" }'
-#   waitSignal:
+#   awsNodeLabels:
 #     enabled: true
 #   loadBalancer:
 #     enabled: true
 #     names: [ "manuallymanagedelb" ]
 #     securityGroupIds: [ "sg-87654321" ]
+#   nodeDrainer:
+#     enabled: true
+#   nodeLabels:
+#     kube-aws.coreos.com/role: worker
+#   waitSignal:
+#     enabled: true
 
 # AWS Tags for cloudformation stack resources 
 #stackTags:

diff --git a/nodepool/config/templates/stack-template.json b/nodepool/config/templates/stack-template.json
@@ -272,7 +272,7 @@
                     ] }
                 },
                 {{end}}
-                {{if .Experimental.NodeLabel.Enabled}}
+                {{if .Experimental.AwsNodeLabels.Enabled}}
                 {
                   "Action": "autoscaling:Describe*",
                   "Effect": "Allow",