diff --git a/config/config_test.go b/config/config_test.go index 75a898d79..9a444699d 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -1000,6 +1000,43 @@ func TestConfig(t *testing.T) { } } + hasDefaultExperimentalFeatures := func(c *Cluster, t *testing.T) { + expected := Experimental{ + AuditLog: AuditLog{ + Enabled: false, + MaxAge: 30, + LogPath: "/dev/stdout", + }, + AwsEnvironment: AwsEnvironment{ + Enabled: false, + }, + EphemeralImageStorage: EphemeralImageStorage{ + Enabled: false, + Disk: "xvdb", + Filesystem: "xfs", + }, + LoadBalancer: LoadBalancer{ + Enabled: false, + }, + NodeDrainer: NodeDrainer{ + Enabled: false, + }, + NodeLabel: NodeLabel{ + Enabled: false, + }, + WaitSignal: WaitSignal{ + Enabled: false, + MaxBatchSize: 1, + }, + } + + actual := c.Experimental + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("experimental settings didn't match :\nexpected=%v\nactual=%v", expected, actual) + } + } + minimalValidConfigYaml := minimalConfigYaml + ` availabilityZone: us-west-1c ` @@ -1008,11 +1045,92 @@ availabilityZone: us-west-1c configYaml string assertConfig []ConfigTester }{ + { + context: "WithExperimentalFeatures", + configYaml: minimalValidConfigYaml + ` +experimental: + auditLog: + enabled: true + maxage: 100 + logpath: "/var/log/audit.log" + awsEnvironment: + enabled: true + environment: + CFNSTACK: '{ "Ref" : "AWS::StackId" }' + ephemeralImageStorage: + enabled: true + loadBalancer: + enabled: true + names: + - manuallymanagedlb + securityGroupIds: + - sg-12345678 + nodeDrainer: + enabled: true + nodeLabel: + enabled: true + plugins: + rbac: + enabled: true + waitSignal: + enabled: true +`, + assertConfig: []ConfigTester{ + hasDefaultEtcdSettings, + func(c *Cluster, t *testing.T) { + expected := Experimental{ + AuditLog: AuditLog{ + Enabled: true, + MaxAge: 100, + LogPath: "/var/log/audit.log", + }, + AwsEnvironment: AwsEnvironment{ + Enabled: true, + Environment: map[string]string{ + "CFNSTACK": `{ "Ref" : "AWS::StackId" }`, + }, + }, + EphemeralImageStorage: EphemeralImageStorage{ + Enabled: true, + Disk: "xvdb", + Filesystem: "xfs", + }, + LoadBalancer: LoadBalancer{ + Enabled: true, + Names: []string{"manuallymanagedlb"}, + SecurityGroupIds: []string{"sg-12345678"}, + }, + NodeDrainer: NodeDrainer{ + Enabled: true, + }, + NodeLabel: NodeLabel{ + Enabled: true, + }, + Plugins: Plugins{ + Rbac: Rbac{ + Enabled: true, + }, + }, + WaitSignal: WaitSignal{ + Enabled: true, + MaxBatchSize: 1, + }, + } + + actual := c.Experimental + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("experimental settings didn't match : expected=%v actual=%v", expected, actual) + } + }, + }, + }, { context: "WithMinimalValidConfig", configYaml: minimalValidConfigYaml, assertConfig: []ConfigTester{ hasDefaultEtcdSettings, + hasDefaultExperimentalFeatures, }, }, { @@ -1022,6 +1140,7 @@ vpcId: vpc-1a2b3c4d `, assertConfig: []ConfigTester{ hasDefaultEtcdSettings, + hasDefaultExperimentalFeatures, }, }, { @@ -1032,6 +1151,7 @@ routeTableId: rtb-1a2b3c4d `, assertConfig: []ConfigTester{ hasDefaultEtcdSettings, + hasDefaultExperimentalFeatures, }, }, { @@ -1045,6 +1165,7 @@ workerSecurityGroupIds: `, assertConfig: []ConfigTester{ hasDefaultEtcdSettings, + hasDefaultExperimentalFeatures, func(c *Cluster, t *testing.T) { expectedWorkerSecurityGroupIds := []string{ `sg-12345678`, `sg-abcdefab`, `sg-23456789`, `sg-bcdefabc`, @@ -1115,6 +1236,7 @@ etcdDataVolumeType: io1 etcdDataVolumeIOPS: 104 `, assertConfig: []ConfigTester{ + hasDefaultExperimentalFeatures, func(c *Cluster, t *testing.T) { expected := EtcdSettings{ EtcdCount: 2, diff --git a/nodepool/config/config_test.go b/nodepool/config/config_test.go index b22c4c299..c1e68c175 100644 --- a/nodepool/config/config_test.go +++ b/nodepool/config/config_test.go @@ -3,6 +3,7 @@ package config import ( "fmt" "github.com/aws/aws-sdk-go/service/kms" + cfg "github.com/coreos/kube-aws/config" "github.com/coreos/kube-aws/test/helper" "reflect" "strings" @@ -72,15 +73,121 @@ etcdEndpoints: "10.0.0.1" } } + hasDefaultExperimentalFeatures := func(c *ProvidedConfig, t *testing.T) { + expected := cfg.Experimental{ + AuditLog: cfg.AuditLog{ + Enabled: false, + MaxAge: 30, + LogPath: "/dev/stdout", + }, + AwsEnvironment: cfg.AwsEnvironment{ + Enabled: false, + }, + EphemeralImageStorage: cfg.EphemeralImageStorage{ + Enabled: false, + Disk: "xvdb", + Filesystem: "xfs", + }, + LoadBalancer: cfg.LoadBalancer{ + Enabled: false, + }, + NodeDrainer: cfg.NodeDrainer{ + Enabled: false, + }, + NodeLabel: cfg.NodeLabel{ + Enabled: false, + }, + WaitSignal: cfg.WaitSignal{ + Enabled: false, + MaxBatchSize: 1, + }, + } + + actual := c.Experimental + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("experimental settings didn't match :\nexpected=%v\nactual=%v", expected, actual) + } + } + validCases := []struct { context string configYaml string assertProvidedConfig []ConfigTester }{ + { + context: "WithExperimentalFeatures", + configYaml: minimalValidConfigYaml + ` +experimental: + awsEnvironment: + enabled: true + environment: + CFNSTACK: '{ "Ref" : "AWS::StackId" }' + ephemeralImageStorage: + enabled: true + loadBalancer: + enabled: true + names: + - manuallymanagedlb + securityGroupIds: + - sg-12345678 + nodeDrainer: + enabled: true + nodeLabel: + enabled: true + waitSignal: + enabled: true +`, + assertProvidedConfig: []ConfigTester{ + hasDefaultLaunchSpecifications, + func(c *ProvidedConfig, t *testing.T) { + expected := cfg.Experimental{ + AuditLog: cfg.AuditLog{ + Enabled: false, + MaxAge: 30, + LogPath: "/dev/stdout", + }, + AwsEnvironment: cfg.AwsEnvironment{ + Enabled: true, + Environment: map[string]string{ + "CFNSTACK": `{ "Ref" : "AWS::StackId" }`, + }, + }, + EphemeralImageStorage: cfg.EphemeralImageStorage{ + Enabled: true, + Disk: "xvdb", + Filesystem: "xfs", + }, + LoadBalancer: cfg.LoadBalancer{ + Enabled: true, + Names: []string{"manuallymanagedlb"}, + SecurityGroupIds: []string{"sg-12345678"}, + }, + NodeDrainer: cfg.NodeDrainer{ + Enabled: true, + }, + NodeLabel: cfg.NodeLabel{ + Enabled: true, + }, + WaitSignal: cfg.WaitSignal{ + Enabled: true, + MaxBatchSize: 1, + }, + } + + actual := c.Experimental + + if !reflect.DeepEqual(expected, actual) { + t.Errorf("experimental settings didn't match : expected=%v actual=%v", expected, actual) + } + }, + }, + }, { context: "WithMinimalValidConfig", configYaml: minimalValidConfigYaml, assertProvidedConfig: []ConfigTester{ + hasDefaultExperimentalFeatures, hasDefaultLaunchSpecifications, }}, { @@ -89,6 +196,7 @@ etcdEndpoints: "10.0.0.1" vpcId: vpc-1a2b3c4d `, assertProvidedConfig: []ConfigTester{ + hasDefaultExperimentalFeatures, hasDefaultLaunchSpecifications, }, }, @@ -99,6 +207,7 @@ vpcId: vpc-1a2b3c4d routeTableId: rtb-1a2b3c4d `, assertProvidedConfig: []ConfigTester{ + hasDefaultExperimentalFeatures, hasDefaultLaunchSpecifications, }, }, @@ -110,6 +219,7 @@ worker: targetCapacity: 10 `, assertProvidedConfig: []ConfigTester{ + hasDefaultExperimentalFeatures, hasDefaultLaunchSpecifications, }, }, @@ -128,6 +238,7 @@ worker: rootVolumeSize: 100 `, assertProvidedConfig: []ConfigTester{ + hasDefaultExperimentalFeatures, func(c *ProvidedConfig, t *testing.T) { expected := []LaunchSpecification{ { @@ -175,6 +286,7 @@ worker: rootVolumeIOPS: 500 `, assertProvidedConfig: []ConfigTester{ + hasDefaultExperimentalFeatures, func(c *ProvidedConfig, t *testing.T) { expected := []LaunchSpecification{ { @@ -217,6 +329,7 @@ workerSecurityGroupIds: - sg-bcdefabc `, assertProvidedConfig: []ConfigTester{ + hasDefaultExperimentalFeatures, hasDefaultLaunchSpecifications, func(c *ProvidedConfig, t *testing.T) { expectedWorkerSecurityGroupIds := []string{ diff --git a/nodepool/config/templates/cluster.yaml b/nodepool/config/templates/cluster.yaml index 48c98c575..5b79c5b4a 100644 --- a/nodepool/config/templates/cluster.yaml +++ b/nodepool/config/templates/cluster.yaml @@ -177,10 +177,18 @@ useCalico: {{.UseCalico}} # experimental: # nodeDrainer: # enabled: true +# nodeLabel: +# enabled: true # awsEnvironment: # enabled: true # environment: # CFNSTACK: '{ "Ref" : "AWS::StackId" }' +# waitSignal: +# enabled: true +# loadBalancer: +# enabled: true +# names: [ "manuallymanagedelb" ] +# securityGroupIds: [ "sg-87654321" ] # AWS Tags for cloudformation stack resources #stackTags: diff --git a/nodepool/config/templates/stack-template.json b/nodepool/config/templates/stack-template.json index a0de514e1..5b0f2828f 100644 --- a/nodepool/config/templates/stack-template.json +++ b/nodepool/config/templates/stack-template.json @@ -90,6 +90,14 @@ "Value": "{{.NodePoolName}}-kube-aws-worker" } ], + {{if .Experimental.LoadBalancer.Enabled}} + "LoadBalancerNames" : [ + {{range $index, $elb := .Experimental.LoadBalancer.Names}} + {{if $index}},{{end}} + "{{$elb}}" + {{end}} + ], + {{end}} "VPCZoneIdentifier": [ {{range $index, $subnet := .Subnets}} {{with $subnetLogicalName := printf "Subnet%d" $index}} @@ -102,6 +110,14 @@ ] }, "Type": "AWS::AutoScaling::AutoScalingGroup", + {{if .Experimental.WaitSignal.Enabled}} + "CreationPolicy" : { + "ResourceSignal" : { + "Count" : "{{.WorkerCount}}", + "Timeout" : "{{.WorkerCreateTimeout}}" + } + }, + {{end}} "UpdatePolicy" : { "AutoScalingRollingUpdate" : { "MinInstancesInService" : @@ -110,8 +126,14 @@ {{else}} "{{.WorkerCount}}" {{end}}, + {{if .Experimental.WaitSignal.Enabled}} + "WaitOnResourceSignals" : "true", + "MaxBatchSize" : "{{.Experimental.WaitSignal.MaxBatchSize}}", + "PauseTime": "{{.WorkerCreateTimeout}}" + {{else}} "MaxBatchSize" : "1", - "PauseTime" : "PT2M" + "PauseTime": "PT2M" + {{end}} } } }, @@ -147,6 +169,23 @@ {{end}} "UserData": "{{ .UserDataWorker }}" }, + {{ if .Experimental.AwsEnvironment.Enabled }} + "Metadata" : { + "AWS::CloudFormation::Init" : { + "config" : { + "commands": { + "write-environment": { + "command": { "Fn::Join" : ["", [ "echo '", + {{range $variable, $function := .Experimental.AwsEnvironment.Environment}} + "{{$variable}}=", {{$function}} , "\n", + {{end}} + "' > /etc/aws-environment" ] ] } + } + } + } + } + }, + {{end}} "Type": "AWS::AutoScaling::LaunchConfiguration" }, {{end}} @@ -213,6 +252,29 @@ "Effect" : "Allow", "Resource" : "{{.KMSKeyARN}}" }, + {{if .Experimental.WaitSignal.Enabled}} + { + "Action": "cloudformation:SignalResource", + "Effect": "Allow", + "Resource": + { "Fn::Join": [ "", [ + "arn:aws:cloudformation:", + { "Ref": "AWS::Region" }, + ":", + { "Ref": "AWS::AccountId" }, + ":stack/", + { "Ref": "AWS::StackName" }, + "/*" ] + ] } + }, + {{end}} + {{if .Experimental.NodeLabel.Enabled}} + { + "Action": "autoscaling:Describe*", + "Effect": "Allow", + "Resource": [ "*" ] + }, + {{end}} { "Action": [ "ecr:GetAuthorizationToken", diff --git a/test/integration/aws_test.go b/test/integration/aws_test.go index 85896304b..2a35d83e2 100644 --- a/test/integration/aws_test.go +++ b/test/integration/aws_test.go @@ -85,6 +85,30 @@ etcdEndpoints: "10.0.0.1" context string configYaml string }{ + { + context: "WithExperimentalFeatures", + configYaml: minimalValidConfigYaml + ` +experimental: + awsEnvironment: + enabled: true + environment: + CFNSTACK: '{ "Ref" : "AWS::StackId" }' + ephemeralImageStorage: + enabled: true + loadBalancer: + enabled: true + names: + - manuallymanagedlb + securityGroupIds: + - sg-12345678 + nodeDrainer: + enabled: true + nodeLabel: + enabled: true + waitSignal: + enabled: true +`, + }, { context: "WithMinimalValidConfig", configYaml: minimalValidConfigYaml,