From 0c980e3e3a848476744d9eab57d0b50b1985e79b Mon Sep 17 00:00:00 2001 From: Yusuke Kuoka Date: Sun, 11 Dec 2016 00:33:59 +0900 Subject: [PATCH] Experimental feature: User-specified node labels for worker nodes This complements Node Pools(#46) and Spot Fleet support(#112) The former `experimental.nodeLabel` configuration key is renamed to `experimental.awsNodeLabels` to avoid collision with newly added `experimental.nodeLabels` and consistency with `experimental.awsEnvironment`. --- config/config.go | 29 +++++++++++++++---- config/config_test.go | 22 +++++++++----- config/templates/cloud-config-controller | 2 +- config/templates/cloud-config-worker | 5 ++-- config/templates/cluster.yaml | 26 +++++++++-------- config/templates/stack-template.json | 4 +-- e2e/run | 7 +++-- nodepool/config/config_test.go | 22 +++++++++----- nodepool/config/templates/cluster.yaml | 12 ++++---- nodepool/config/templates/stack-template.json | 2 +- 10 files changed, 84 insertions(+), 47 deletions(-) diff --git a/config/config.go b/config/config.go index 84dc15d22..548e25a77 100644 --- a/config/config.go +++ b/config/config.go @@ -37,6 +37,9 @@ func NewDefaultCluster() *Cluster { AwsEnvironment{ Enabled: false, }, + AwsNodeLabels{ + Enabled: false, + }, EphemeralImageStorage{ Enabled: false, Disk: "xvdb", @@ -48,9 +51,7 @@ func NewDefaultCluster() *Cluster { NodeDrainer{ Enabled: false, }, - NodeLabel{ - Enabled: false, - }, + NodeLabels{}, Plugins{ Rbac{ Enabled: false, @@ -296,10 +297,11 @@ type Subnet struct { type Experimental struct { AuditLog AuditLog `yaml:"auditLog"` AwsEnvironment AwsEnvironment `yaml:"awsEnvironment"` + AwsNodeLabels AwsNodeLabels `yaml:"awsNodeLabels"` EphemeralImageStorage EphemeralImageStorage `yaml:"ephemeralImageStorage"` LoadBalancer LoadBalancer `yaml:"loadBalancer"` NodeDrainer NodeDrainer `yaml:"nodeDrainer"` - NodeLabel NodeLabel `yaml:"nodeLabel"` + NodeLabels NodeLabels `yaml:"nodeLabels"` Plugins Plugins `yaml:"plugins"` Taints []Taint `yaml:"taints"` WaitSignal WaitSignal `yaml:"waitSignal"` @@ -316,6 +318,10 @@ type AuditLog struct { LogPath string `yaml:"logpath"` } +type AwsNodeLabels struct { + Enabled bool `yaml:"enabled"` +} + type EphemeralImageStorage struct { Enabled bool `yaml:"enabled"` Disk string `yaml:"disk"` @@ -326,8 +332,19 @@ type NodeDrainer struct { Enabled bool `yaml:"enabled"` } -type NodeLabel struct { - Enabled bool `yaml:"enabled"` +type NodeLabels map[string]string + +func (l NodeLabels) Enabled() bool { + return len(l) > 0 +} + +// Returns key=value pairs separated by ',' to be passed to kubelet's `--node-labels` flag +func (l NodeLabels) String() string { + labels := []string{} + for k, v := range l { + labels = append(labels, fmt.Sprintf("%s=%s", k, v)) + } + return strings.Join(labels, ",") } type LoadBalancer struct { diff --git a/config/config_test.go b/config/config_test.go index 745fa3b22..acae40d85 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1010,6 +1010,9 @@ func TestConfig(t *testing.T) { AwsEnvironment: AwsEnvironment{ Enabled: false, }, + AwsNodeLabels: AwsNodeLabels{ + Enabled: false, + }, EphemeralImageStorage: EphemeralImageStorage{ Enabled: false, Disk: "xvdb", @@ -1021,10 +1024,8 @@ func TestConfig(t *testing.T) { NodeDrainer: NodeDrainer{ Enabled: false, }, - NodeLabel: NodeLabel{ - Enabled: false, - }, - Taints: []Taint{}, + NodeLabels: NodeLabels{}, + Taints: []Taint{}, WaitSignal: WaitSignal{ Enabled: false, MaxBatchSize: 1, @@ -1058,6 +1059,8 @@ experimental: enabled: true environment: CFNSTACK: '{ "Ref" : "AWS::StackId" }' + awsNodeLabels: + enabled: true ephemeralImageStorage: enabled: true loadBalancer: @@ -1068,8 +1071,8 @@ experimental: - sg-12345678 nodeDrainer: enabled: true - nodeLabel: - enabled: true + nodeLabels: + kube-aws.coreos.com/role: worker plugins: rbac: enabled: true @@ -1095,6 +1098,9 @@ experimental: "CFNSTACK": `{ "Ref" : "AWS::StackId" }`, }, }, + AwsNodeLabels: AwsNodeLabels{ + Enabled: true, + }, EphemeralImageStorage: EphemeralImageStorage{ Enabled: true, Disk: "xvdb", @@ -1108,8 +1114,8 @@ experimental: NodeDrainer: NodeDrainer{ Enabled: true, }, - NodeLabel: NodeLabel{ - Enabled: true, + NodeLabels: NodeLabels{ + "kube-aws.coreos.com/role": "worker", }, Plugins: Plugins{ Rbac: Rbac{ diff --git a/config/templates/cloud-config-controller b/config/templates/cloud-config-controller index 47e0d9537..829574f0e 100644 --- a/config/templates/cloud-config-controller +++ b/config/templates/cloud-config-controller @@ -260,7 +260,7 @@ coreos: --resource AutoScaleController \ --stack {{.ClusterName}} {{end}} -{{if .Experimental.NodeLabel.Enabled }} +{{if .Experimental.AwsNodeLabels.Enabled }} - name: kube-node-label.service enable: true command: start diff --git a/config/templates/cloud-config-worker b/config/templates/cloud-config-worker index c99b5df0c..487493dd1 100644 --- a/config/templates/cloud-config-worker +++ b/config/templates/cloud-config-worker @@ -70,7 +70,8 @@ coreos: --container-runtime={{.ContainerRuntime}} \ --rkt-path=/usr/bin/rkt \ --rkt-stage1-image=coreos.com/rkt/stage1-coreos \ - --register-node=true \ + {{if .Experimental.NodeLabels.Enabled}}--node-labels {{.Experimental.NodeLabels.String}} \ + {{end}}--register-node=true \ {{if .Experimental.Taints}}--register-schedulable=false \ {{end}}--allow-privileged=true \ --pod-manifest-path=/etc/kubernetes/manifests \ @@ -292,7 +293,7 @@ coreos: --stack {{.ClusterName}} {{end}} -{{if .Experimental.NodeLabel.Enabled }} +{{if .Experimental.AwsNodeLabels.Enabled }} - name: kube-node-label.service enable: true command: start diff --git a/config/templates/cluster.yaml b/config/templates/cluster.yaml index abdfa185c..7ee0ff6d9 100644 --- a/config/templates/cluster.yaml +++ b/config/templates/cluster.yaml @@ -210,26 +210,15 @@ worker: # Experimental features will change in backward-incompatible ways # experimental: -# nodeDrainer: -# enabled: true -# nodeLabel: -# enabled: true # awsEnvironment: # enabled: true # environment: # CFNSTACK: '{ "Ref" : "AWS::StackId" }' -# plugins: -# rbac: -# enabled: true # auditLog: # enabled: true # maxage: 30 # logpath: /dev/stdout -# taints: -# - key: dedicated -# value: search -# effect: NoSchedule -# waitSignal: +# awsNodeLabels: # enabled: true # # This option has not yet been tested with rkt as container runtime # ephemeralImageStorage: @@ -238,6 +227,19 @@ worker: # enabled: true # names: [ "manuallymanagedelb" ] # securityGroupIds: [ "sg-87654321" ] +# nodeDrainer: +# enabled: true +# nodeLabels: +# kube-aws.coreos.com/role: worker +# taints: +# - key: dedicated +# value: search +# effect: NoSchedule +# waitSignal: +# enabled: true +# plugins: +# rbac: +# enabled: true # AWS Tags for cloudformation stack resources #stackTags: diff --git a/config/templates/stack-template.json b/config/templates/stack-template.json index 565975e0a..8627d013b 100644 --- a/config/templates/stack-template.json +++ b/config/templates/stack-template.json @@ -255,7 +255,7 @@ ] } }, {{end}} - {{if .Experimental.NodeLabel.Enabled}} + {{if .Experimental.AwsNodeLabels.Enabled}} { "Action": "autoscaling:Describe*", "Effect": "Allow", @@ -348,7 +348,7 @@ ] } }, {{end}} - {{if .Experimental.NodeLabel.Enabled}} + {{if .Experimental.AwsNodeLabels.Enabled}} { "Action": "autoscaling:Describe*", "Effect": "Allow", diff --git a/e2e/run b/e2e/run index 96d7ef022..9a9e33508 100755 --- a/e2e/run +++ b/e2e/run @@ -88,8 +88,11 @@ customize_worker() { if [ "${KUBE_AWS_WAIT_SIGNAL_ENABLED}" != "" ]; then echo -e ' waitSignal:\n enabled: true' >> cluster.yaml fi - if [ "${KUBE_AWS_NODE_LABEL_ENABLED}" != "" ]; then - echo -e ' nodeLabel:\n enabled: true' >> cluster.yaml + if [ "${KUBE_AWS_AWS_NODE_LABELS_ENABLED}" != "" ]; then + echo -e ' awsNodeLabels:\n enabled: true' >> cluster.yaml + fi + if [ "${KUBE_AWS_NODE_LABELS_ENABLED}" != "" ]; then + echo -e ' nodeLabels:\n kube-aws.coreos.com/role: worker' >> cluster.yaml fi if [ "${KUBE_AWS_AWS_ENV_ENABLED}" != "" ]; then echo -e " awsEnvironment:\n enabled: true\n environment:\n CFNSTACK: '{\"Ref\":\"AWS::StackId\"}'" >> cluster.yaml diff --git a/nodepool/config/config_test.go b/nodepool/config/config_test.go index bfe85795a..7571e0d1e 100644 --- a/nodepool/config/config_test.go +++ b/nodepool/config/config_test.go @@ -84,6 +84,9 @@ etcdEndpoints: "10.0.0.1" AwsEnvironment: cfg.AwsEnvironment{ Enabled: false, }, + AwsNodeLabels: cfg.AwsNodeLabels{ + Enabled: false, + }, EphemeralImageStorage: cfg.EphemeralImageStorage{ Enabled: false, Disk: "xvdb", @@ -95,10 +98,8 @@ etcdEndpoints: "10.0.0.1" NodeDrainer: cfg.NodeDrainer{ Enabled: false, }, - NodeLabel: cfg.NodeLabel{ - Enabled: false, - }, - Taints: []cfg.Taint{}, + NodeLabels: cfg.NodeLabels{}, + Taints: []cfg.Taint{}, WaitSignal: cfg.WaitSignal{ Enabled: false, MaxBatchSize: 1, @@ -125,6 +126,8 @@ experimental: enabled: true environment: CFNSTACK: '{ "Ref" : "AWS::StackId" }' + awsNodeLabels: + enabled: true ephemeralImageStorage: enabled: true loadBalancer: @@ -135,8 +138,8 @@ experimental: - sg-12345678 nodeDrainer: enabled: true - nodeLabel: - enabled: true + nodeLabels: + kube-aws.coreos.com/role: worker taints: - key: reservation value: spot @@ -159,6 +162,9 @@ experimental: "CFNSTACK": `{ "Ref" : "AWS::StackId" }`, }, }, + AwsNodeLabels: cfg.AwsNodeLabels{ + Enabled: true, + }, EphemeralImageStorage: cfg.EphemeralImageStorage{ Enabled: true, Disk: "xvdb", @@ -172,8 +178,8 @@ experimental: NodeDrainer: cfg.NodeDrainer{ Enabled: true, }, - NodeLabel: cfg.NodeLabel{ - Enabled: true, + NodeLabels: cfg.NodeLabels{ + "kube-aws.coreos.com/role": "worker", }, Taints: []cfg.Taint{ {Key: "reservation", Value: "spot", Effect: "NoSchedule"}, diff --git a/nodepool/config/templates/cluster.yaml b/nodepool/config/templates/cluster.yaml index 181bbbf9d..fc5690536 100644 --- a/nodepool/config/templates/cluster.yaml +++ b/nodepool/config/templates/cluster.yaml @@ -180,20 +180,22 @@ useCalico: {{.UseCalico}} # Experimental features will change in backward-incompatible ways # experimental: -# nodeDrainer: -# enabled: true -# nodeLabel: -# enabled: true # awsEnvironment: # enabled: true # environment: # CFNSTACK: '{ "Ref" : "AWS::StackId" }' -# waitSignal: +# awsNodeLabels: # enabled: true # loadBalancer: # enabled: true # names: [ "manuallymanagedelb" ] # securityGroupIds: [ "sg-87654321" ] +# nodeDrainer: +# enabled: true +# nodeLabels: +# kube-aws.coreos.com/role: worker +# waitSignal: +# enabled: true # AWS Tags for cloudformation stack resources #stackTags: diff --git a/nodepool/config/templates/stack-template.json b/nodepool/config/templates/stack-template.json index 780445e73..808dd6bdc 100644 --- a/nodepool/config/templates/stack-template.json +++ b/nodepool/config/templates/stack-template.json @@ -272,7 +272,7 @@ ] } }, {{end}} - {{if .Experimental.NodeLabel.Enabled}} + {{if .Experimental.AwsNodeLabels.Enabled}} { "Action": "autoscaling:Describe*", "Effect": "Allow",