Experimental feature to taint worker nodes with user-provided `key=va…

…lue:effect`

ref kubernetes-retired#112

config/config.go

@@ -55,6 +55,7 @@ func NewDefaultCluster() *Cluster {
 				Enabled: false,
 			},
 		},
+		[]Taint{},
 		WaitSignal{
 			Enabled:      false,
 			MaxBatchSize: 1,
@@ -297,6 +298,7 @@ type Experimental struct {
 	NodeDrainer           NodeDrainer           `yaml:"nodeDrainer"`
 	NodeLabel             NodeLabel             `yaml:"nodeLabel"`
 	Plugins               Plugins               `yaml:"plugins"`
+	Taints                []Taint               `yaml:"taints"`
 	WaitSignal            WaitSignal            `yaml:"waitSignal"`
 }
 
@@ -339,6 +341,16 @@ type Rbac struct {
 	Enabled bool `yaml:"enabled"`
 }
 
+type Taint struct {
+	Key    string `yaml:"key"`
+	Value  string `yaml:"value"`
+	Effect string `yaml:"effect"`
+}
+
+func (t Taint) String() string {
+	return fmt.Sprintf("%s=%s:%s", t.Key, t.Value, t.Effect)
+}
+
 type WaitSignal struct {
 	Enabled      bool `yaml:"enabled"`
 	MaxBatchSize int  `yaml:"maxBatchSize"`
@@ -810,6 +822,10 @@ func (c DeploymentSettings) Valid() (*DeploymentValidationResult, error) {
 		}
 	}
 
+	if err := c.Experimental.Valid(); err != nil {
+		return nil, err
+	}
+
 	return &DeploymentValidationResult{vpcNet: vpcNet}, nil
 }
 
@@ -849,6 +865,16 @@ func (c ControllerSettings) Valid() error {
 	return nil
 }
 
+func (c Experimental) Valid() error {
+	for _, taint := range c.Taints {
+		if taint.Effect != "NoSchedule" && taint.Effect != "PreferNoSchedule" {
+			return fmt.Errorf("Effect must be NoSchdule or PreferNoSchedule, but was %s", taint.Effect)
+		}
+	}
+
+	return nil
+}
+
 /*
 Returns the availability zones referenced by the cluster configuration
 */

config/config_test.go

@@ -1024,6 +1024,7 @@ func TestConfig(t *testing.T) {
 			NodeLabel: NodeLabel{
 				Enabled: false,
 			},
+			Taints: []Taint{},
 			WaitSignal: WaitSignal{
 				Enabled:      false,
 				MaxBatchSize: 1,
@@ -1072,6 +1073,10 @@ experimental:
   plugins:
     rbac:
       enabled: true
+  taints:
+    - key: reservation
+      value: spot
+      effect: NoSchedule
   waitSignal:
     enabled: true
 `,
@@ -1111,6 +1116,9 @@ experimental:
 								Enabled: true,
 							},
 						},
+						Taints: []Taint{
+							{Key: "reservation", Value: "spot", Effect: "NoSchedule"},
+						},
 						WaitSignal: WaitSignal{
 							Enabled:      true,
 							MaxBatchSize: 1,
@@ -1307,6 +1315,17 @@ etcdDataVolumeIOPS: 104
 		configYaml           string
 		expectedErrorMessage string
 	}{
+		{
+			context: "WithInvalidTaint",
+			configYaml: minimalValidConfigYaml + `
+experimental:
+  taints:
+    - key: foo
+      value: bar
+      effect: UnknownEffect
+`,
+			expectedErrorMessage: "Effect must be NoSchdule or PreferNoSchedule, but was UnknownEffect",
+		},
 		{
 			context: "WithVpcIdAndVPCCIDRSpecified",
 			configYaml: minimalValidConfigYaml + `

config/templates/cloud-config-worker

@@ -71,7 +71,8 @@ coreos:
         --rkt-path=/usr/bin/rkt \
         --rkt-stage1-image=coreos.com/rkt/stage1-coreos \
         --register-node=true \
-        --allow-privileged=true \
+        {{if .Experimental.Taints}}--register-schedulable=false \
+        {{end}}--allow-privileged=true \
         --pod-manifest-path=/etc/kubernetes/manifests \
         --cluster_dns={{.DNSServiceIP}} \
         --cluster_domain=cluster.local \
@@ -229,6 +230,27 @@ coreos:
         WantedBy=kubelet.service
 {{ end }}
 
+{{if .Experimental.Taints }}
+    - name: kube-node-taint-and-uncordon.service
+      command: start
+      runtime: true
+      content: |
+        [Unit]
+        Description=Taint this kubernetes node with user-provided taints and then uncordon it
+        Wants=kubelet.service
+        After=kubelet.service
+        Before=cfn-signal.service
+
+        [Service]
+        Type=simple
+        StartLimitInterval=0
+        RestartSec=10
+        Restart=on-failure
+        ExecStartPre=/usr/bin/systemctl is-active kubelet.service
+        ExecStartPre=/usr/bin/bash -c "while sleep 1; do if /usr/bin/curl  --insecure -s -m 20 -f  https://127.0.0.1:10250/healthz > /dev/null ; then break ; fi;  done"
+        ExecStart=/opt/bin/taint-and-uncordon
+{{end}}
+
 {{ if .Experimental.WaitSignal.Enabled }}
     - name: cfn-signal.service
       command: start
@@ -415,6 +437,42 @@ write_files:
       done
       echo done.
 
+  - path: /opt/bin/taint-and-uncordon
+    owner: root:root
+    permissions: 0700
+    content: |
+      #!/bin/bash -e
+
+      hostname=$(hostname)
+
+      sudo rkt run \
+        --volume=kube,kind=host,source=/etc/kubernetes,readOnly=true \
+        --mount=volume=kube,target=/etc/kubernetes \
+        --uuid-file-save=/var/run/coreos/taint-and-uncordon.uuid \
+        --volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true --mount volume=dns,target=/etc/resolv.conf \
+        --net=host \
+        --trust-keys-from-https \
+        {{.HyperkubeImageRepo}}:{{.K8sVer}} --exec=/bin/bash -- \
+          -vxc \
+          'echo tainting this node; \
+           hostname="'${hostname}'"; \
+           taints=({{range $i, $taint := .Experimental.Taints}}"{{$taint.String}}" {{end}}); \
+           kubectl="/kubectl --server=https://{{.ExternalDNSName}}:443 --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"; \
+           taint="$kubectl taint node $hostname"; \
+           for t in ${taints[@]}; do \
+             $taint "$t"; \
+           done; \
+           echo done. ;\
+           echo uncordoning this node; \
+           $kubectl uncordon $hostname;\
+           echo done.'
+
+      echo cleaning pod resources.
+
+      sudo rkt rm --uuid-file=/var/run/coreos/taint-and-uncordon.uuid
+
+      echo done.
+
   - path: /etc/kubernetes/manifests/kube-proxy.yaml
     content: |
         apiVersion: v1

config/templates/cluster.yaml

@@ -211,6 +211,10 @@ kmsKeyArn: "{{.KMSKeyARN}}"
 #     enabled: true
 #     maxage: 30
 #     logpath: /dev/stdout
+#   taints:
+#     - key: dedicated
+#       value: search
+#       effect: NoSchedule
 #   waitSignal:
 #     enabled: true
 #   # This option has not yet been tested with rkt as container runtime

nodepool/config/config_test.go

@@ -97,6 +97,7 @@ etcdEndpoints: "10.0.0.1"
 			NodeLabel: cfg.NodeLabel{
 				Enabled: false,
 			},
+			Taints: []cfg.Taint{},
 			WaitSignal: cfg.WaitSignal{
 				Enabled:      false,
 				MaxBatchSize: 1,
@@ -135,6 +136,10 @@ experimental:
     enabled: true
   nodeLabel:
     enabled: true
+  taints:
+    - key: reservation
+      value: spot
+      effect: NoSchedule
   waitSignal:
     enabled: true
 `,
@@ -169,6 +174,9 @@ experimental:
 						NodeLabel: cfg.NodeLabel{
 							Enabled: true,
 						},
+						Taints: []cfg.Taint{
+							{Key: "reservation", Value: "spot", Effect: "NoSchedule"},
+						},
 						WaitSignal: cfg.WaitSignal{
 							Enabled:      true,
 							MaxBatchSize: 1,
@@ -432,6 +440,17 @@ experimental:
 		configYaml           string
 		expectedErrorMessage string
 	}{
+		{
+			context: "WithInvalidTaint",
+			configYaml: minimalValidConfigYaml + `
+experimental:
+  taints:
+    - key: foo
+      value: bar
+      effect: UnknownEffect
+`,
+			expectedErrorMessage: "Effect must be NoSchdule or PreferNoSchedule, but was UnknownEffect",
+		},
 		{
 			context: "WithVpcIdAndVPCCIDRSpecified",
 			configYaml: minimalValidConfigYaml + `

test/integration/aws_test.go

@@ -105,6 +105,10 @@ experimental:
     enabled: true
   nodeLabel:
     enabled: true
+  taints:
+    - key: reservation
+      value: spot
+      effect: NoSchedule
   waitSignal:
     enabled: true
 `,