diff --git a/config/config.go b/config/config.go index 5c2a55627..0503ab3d5 100644 --- a/config/config.go +++ b/config/config.go @@ -55,6 +55,7 @@ func NewDefaultCluster() *Cluster { Enabled: false, }, }, + []Taint{}, WaitSignal{ Enabled: false, MaxBatchSize: 1, @@ -297,6 +298,7 @@ type Experimental struct { NodeDrainer NodeDrainer `yaml:"nodeDrainer"` NodeLabel NodeLabel `yaml:"nodeLabel"` Plugins Plugins `yaml:"plugins"` + Taints []Taint `yaml:"taints"` WaitSignal WaitSignal `yaml:"waitSignal"` } @@ -339,6 +341,16 @@ type Rbac struct { Enabled bool `yaml:"enabled"` } +type Taint struct { + Key string `yaml:"key"` + Value string `yaml:"value"` + Effect string `yaml:"effect"` +} + +func (t Taint) String() string { + return fmt.Sprintf("%s=%s:%s", t.Key, t.Value, t.Effect) +} + type WaitSignal struct { Enabled bool `yaml:"enabled"` MaxBatchSize int `yaml:"maxBatchSize"` @@ -810,6 +822,10 @@ func (c DeploymentSettings) Valid() (*DeploymentValidationResult, error) { } } + if err := c.Experimental.Valid(); err != nil { + return nil, err + } + return &DeploymentValidationResult{vpcNet: vpcNet}, nil } @@ -849,6 +865,16 @@ func (c ControllerSettings) Valid() error { return nil } +func (c Experimental) Valid() error { + for _, taint := range c.Taints { + if taint.Effect != "NoSchedule" && taint.Effect != "PreferNoSchedule" { + return fmt.Errorf("Effect must be NoSchdule or PreferNoSchedule, but was %s", taint.Effect) + } + } + + return nil +} + /* Returns the availability zones referenced by the cluster configuration */ diff --git a/config/config_test.go b/config/config_test.go index 9a444699d..b8c6e5e66 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1024,6 +1024,7 @@ func TestConfig(t *testing.T) { NodeLabel: NodeLabel{ Enabled: false, }, + Taints: []Taint{}, WaitSignal: WaitSignal{ Enabled: false, MaxBatchSize: 1, @@ -1072,6 +1073,10 @@ experimental: plugins: rbac: enabled: true + taints: + - key: reservation + value: spot + effect: NoSchedule waitSignal: enabled: true `, @@ -1111,6 +1116,9 @@ experimental: Enabled: true, }, }, + Taints: []Taint{ + {Key: "reservation", Value: "spot", Effect: "NoSchedule"}, + }, WaitSignal: WaitSignal{ Enabled: true, MaxBatchSize: 1, @@ -1307,6 +1315,17 @@ etcdDataVolumeIOPS: 104 configYaml string expectedErrorMessage string }{ + { + context: "WithInvalidTaint", + configYaml: minimalValidConfigYaml + ` +experimental: + taints: + - key: foo + value: bar + effect: UnknownEffect +`, + expectedErrorMessage: "Effect must be NoSchdule or PreferNoSchedule, but was UnknownEffect", + }, { context: "WithVpcIdAndVPCCIDRSpecified", configYaml: minimalValidConfigYaml + ` diff --git a/config/templates/cloud-config-worker b/config/templates/cloud-config-worker index c0c22fded..20989ae31 100644 --- a/config/templates/cloud-config-worker +++ b/config/templates/cloud-config-worker @@ -71,7 +71,8 @@ coreos: --rkt-path=/usr/bin/rkt \ --rkt-stage1-image=coreos.com/rkt/stage1-coreos \ --register-node=true \ - --allow-privileged=true \ + {{if .Experimental.Taints}}--register-schedulable=false \ + {{end}}--allow-privileged=true \ --pod-manifest-path=/etc/kubernetes/manifests \ --cluster_dns={{.DNSServiceIP}} \ --cluster_domain=cluster.local \ @@ -229,6 +230,27 @@ coreos: WantedBy=kubelet.service {{ end }} +{{if .Experimental.Taints }} + - name: kube-node-taint-and-uncordon.service + command: start + runtime: true + content: | + [Unit] + Description=Taint this kubernetes node with user-provided taints and then uncordon it + Wants=kubelet.service + After=kubelet.service + Before=cfn-signal.service + + [Service] + Type=simple + StartLimitInterval=0 + RestartSec=10 + Restart=on-failure + ExecStartPre=/usr/bin/systemctl is-active kubelet.service + ExecStartPre=/usr/bin/bash -c "while sleep 1; do if /usr/bin/curl --insecure -s -m 20 -f https://127.0.0.1:10250/healthz > /dev/null ; then break ; fi; done" + ExecStart=/opt/bin/taint-and-uncordon +{{end}} + {{ if .Experimental.WaitSignal.Enabled }} - name: cfn-signal.service command: start @@ -415,6 +437,42 @@ write_files: done echo done. + - path: /opt/bin/taint-and-uncordon + owner: root:root + permissions: 0700 + content: | + #!/bin/bash -e + + hostname=$(hostname) + + sudo rkt run \ + --volume=kube,kind=host,source=/etc/kubernetes,readOnly=true \ + --mount=volume=kube,target=/etc/kubernetes \ + --uuid-file-save=/var/run/coreos/taint-and-uncordon.uuid \ + --volume=dns,kind=host,source=/etc/resolv.conf,readOnly=true --mount volume=dns,target=/etc/resolv.conf \ + --net=host \ + --trust-keys-from-https \ + {{.HyperkubeImageRepo}}:{{.K8sVer}} --exec=/bin/bash -- \ + -vxc \ + 'echo tainting this node; \ + hostname="'${hostname}'"; \ + taints=({{range $i, $taint := .Experimental.Taints}}"{{$taint.String}}" {{end}}); \ + kubectl="/kubectl --server=https://{{.ExternalDNSName}}:443 --kubeconfig=/etc/kubernetes/worker-kubeconfig.yaml"; \ + taint="$kubectl taint node $hostname"; \ + for t in ${taints[@]}; do \ + $taint "$t"; \ + done; \ + echo done. ;\ + echo uncordoning this node; \ + $kubectl uncordon $hostname;\ + echo done.' + + echo cleaning pod resources. + + sudo rkt rm --uuid-file=/var/run/coreos/taint-and-uncordon.uuid + + echo done. + - path: /etc/kubernetes/manifests/kube-proxy.yaml content: | apiVersion: v1 diff --git a/config/templates/cluster.yaml b/config/templates/cluster.yaml index fc726494c..5c3a18a7e 100644 --- a/config/templates/cluster.yaml +++ b/config/templates/cluster.yaml @@ -211,6 +211,10 @@ kmsKeyArn: "{{.KMSKeyARN}}" # enabled: true # maxage: 30 # logpath: /dev/stdout +# taints: +# - key: dedicated +# value: search +# effect: NoSchedule # waitSignal: # enabled: true # # This option has not yet been tested with rkt as container runtime diff --git a/nodepool/config/config_test.go b/nodepool/config/config_test.go index c1e68c175..ef4087348 100644 --- a/nodepool/config/config_test.go +++ b/nodepool/config/config_test.go @@ -97,6 +97,7 @@ etcdEndpoints: "10.0.0.1" NodeLabel: cfg.NodeLabel{ Enabled: false, }, + Taints: []cfg.Taint{}, WaitSignal: cfg.WaitSignal{ Enabled: false, MaxBatchSize: 1, @@ -135,6 +136,10 @@ experimental: enabled: true nodeLabel: enabled: true + taints: + - key: reservation + value: spot + effect: NoSchedule waitSignal: enabled: true `, @@ -169,6 +174,9 @@ experimental: NodeLabel: cfg.NodeLabel{ Enabled: true, }, + Taints: []cfg.Taint{ + {Key: "reservation", Value: "spot", Effect: "NoSchedule"}, + }, WaitSignal: cfg.WaitSignal{ Enabled: true, MaxBatchSize: 1, @@ -432,6 +440,17 @@ experimental: configYaml string expectedErrorMessage string }{ + { + context: "WithInvalidTaint", + configYaml: minimalValidConfigYaml + ` +experimental: + taints: + - key: foo + value: bar + effect: UnknownEffect +`, + expectedErrorMessage: "Effect must be NoSchdule or PreferNoSchedule, but was UnknownEffect", + }, { context: "WithVpcIdAndVPCCIDRSpecified", configYaml: minimalValidConfigYaml + ` diff --git a/test/integration/aws_test.go b/test/integration/aws_test.go index 2a35d83e2..ff3fe5cc8 100644 --- a/test/integration/aws_test.go +++ b/test/integration/aws_test.go @@ -105,6 +105,10 @@ experimental: enabled: true nodeLabel: enabled: true + taints: + - key: reservation + value: spot + effect: NoSchedule waitSignal: enabled: true `,