Switch to Application Load Balancers

CHANGELOG.md

@@ -11,6 +11,7 @@
 * Empire now supports sending internal metrics to statsd or dogstatsd [#953](https://github.com/remind101/empire/pull/953)
 * Attached and detached runs now have an `empire.user` label attached to them [#965](https://github.com/remind101/empire/pull/965)
 * You can now provide the name of a process defined in the Procfile when calling `emp run` [#967](https://github.com/remind101/empire/pull/967)
+* Empire now includes experimental support for the new [Application Load Balancers](https://aws.amazon.com/blogs/aws/new-aws-application-load-balancer/) by setting the `LOAD_BALANCER_TYPE=alb` environment variable. [#969](https://github.com/remind101/empire/pull/969)
 
 **Improvements**
 

cmd/empire/factories.go

@@ -149,6 +149,7 @@ func newCloudFormationScheduler(db *empire.DB, c *Context) (*cloudformation.Sche
 	}
 
 	t := &cloudformation.EmpireTemplate{
+		VpcId:                   c.String(FlagELBVpcId),
 		Cluster:                 c.String(FlagECSCluster),
 		InternalSecurityGroupID: c.String(FlagELBSGPrivate),
 		ExternalSecurityGroupID: c.String(FlagELBSGPublic),

cmd/empire/main.go

@@ -53,6 +53,7 @@ const (
 
 	FlagELBSGPrivate = "elb.sg.private"
 	FlagELBSGPublic  = "elb.sg.public"
+	FlagELBVpcId     = "elb.vpc.id"
 
 	FlagEC2SubnetsPrivate = "ec2.subnets.private"
 	FlagEC2SubnetsPublic  = "ec2.subnets.public"
@@ -278,6 +279,11 @@ var EmpireFlags = []cli.Flag{
 		Usage:  "The ELB security group to assign public load balancers",
 		EnvVar: "EMPIRE_ELB_SG_PUBLIC",
 	},
+	cli.StringFlag{
+		Name:   FlagELBVpcId,
+		Usage:  "The comma separated private subnet ids",
+		EnvVar: "EMPIRE_ELB_VPC_ID",
+	},
 	cli.StringSliceFlag{
 		Name:   FlagEC2SubnetsPrivate,
 		Value:  &cli.StringSlice{},

docs/cloudformation.json

@@ -507,16 +507,7 @@
             {
               "Effect": "Allow",
               "Action": [
-                "elasticloadbalancing:DeleteLoadBalancer",
-                "elasticloadbalancing:CreateLoadBalancer",
-                "elasticloadbalancing:DescribeLoadBalancers",
-                "elasticloadbalancing:DescribeTags",
-                "elasticloadbalancing:ConfigureHealthCheck",
-                "elasticloadbalancing:ModifyLoadBalancerAttributes",
-                "elasticloadbalancing:SetLoadBalancerListenerSSLCertificate",
-                "elasticloadbalancing:CreateLoadBalancerListeners",
-                "elasticloadbalancing:DeleteLoadBalancerListeners",
-                "elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
+                "elasticloadbalancing:*"
               ],
               "Resource": ["*"]
             },

pkg/troposphere/troposphere.go

@@ -45,6 +45,7 @@ type Output struct {
 // Resource represents a CloudFormation Resource.
 type Resource struct {
 	Condition  interface{} `json:"Condition,omitempty"`
+	DependsOn  interface{} `json:"DependsOn,omitempty"`
 	Properties interface{} `json:"Properties,omitempty"`
 	Type       interface{} `json:"Type,omitempty"`
 	Version    interface{} `json:"Version,omitempty"`

scheduler/cloudformation/template.go

@@ -28,6 +28,12 @@ var (
 	Join   = troposphere.Join
 )
 
+// Load balancer types
+var (
+	classicLoadBalancer     = "elb"
+	applicationLoadBalancer = "alb"
+)
+
 const (
 	// For HTTP/HTTPS/TCP services, we allocate an ELB and map it's instance port to
 	// the container port. This is the port that processes within the container
@@ -59,6 +65,10 @@ type EmpireTemplate struct {
 	// The ECS cluster to run the services in.
 	Cluster string
 
+	// The VPC to create ALB target groups within. Should be the same VPC
+	// that ECS services will run within.
+	VpcId string
+
 	// The hosted zone to add CNAME's to.
 	HostedZone *route53.HostedZone
 
@@ -93,6 +103,9 @@ func (t *EmpireTemplate) Validate() error {
 		return errors.New(fmt.Sprintf("%s is required", n))
 	}
 
+	if t.VpcId == "" {
+		return r("VpcId")
+	}
 	if t.Cluster == "" {
 		return r("Cluster")
 	}
@@ -321,6 +334,7 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A
 
 	var portMappings []*PortMappingProperties
 
+	var serviceDependencies []string
 	loadBalancers := []map[string]interface{}{}
 	if p.Exposure != nil {
 		scheme := schemeInternal
@@ -333,72 +347,167 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A
 			subnets = t.ExternalSubnetIDs
 		}
 
-		instancePort := fmt.Sprintf("%s%dInstancePort", key, ContainerPort)
-		tmpl.Resources[instancePort] = troposphere.Resource{
-			Type:    "Custom::InstancePort",
-			Version: "1.0",
-			Properties: map[string]interface{}{
-				"ServiceToken": t.CustomResourcesTopic,
-			},
-		}
+		p.Env["PORT"] = fmt.Sprintf("%d", ContainerPort)
 
-		listeners := []map[string]interface{}{
-			map[string]interface{}{
-				"LoadBalancerPort": 80,
-				"Protocol":         "http",
-				"InstancePort":     GetAtt(instancePort, "InstancePort"),
-				"InstanceProtocol": "http",
-			},
+		loadBalancerType := classicLoadBalancer
+		if v, ok := app.Env["LOAD_BALANCER_TYPE"]; ok {
+			loadBalancerType = v
 		}
 
-		if e, ok := p.Exposure.Type.(*scheduler.HTTPSExposure); ok {
-			var cert interface{}
-			if _, err := arn.Parse(e.Cert); err == nil {
-				cert = e.Cert
-			} else {
-				cert = Join("", "arn:aws:iam::", Ref("AWS::AccountId"), ":server-certificate/", e.Cert)
+		var loadBalancer string
+		switch loadBalancerType {
+		case applicationLoadBalancer:
+			loadBalancer = fmt.Sprintf("%sApplicationLoadBalancer", key)
+			tmpl.Resources[loadBalancer] = troposphere.Resource{
+				Type: "AWS::ElasticLoadBalancingV2::LoadBalancer",
+				Properties: map[string]interface{}{
+					"Scheme":         scheme,
+					"SecurityGroups": []string{sg},
+					"Subnets":        subnets,
+					"Tags": []map[string]string{
+						map[string]string{
+							"Key":   "empire.app.process",
+							"Value": p.Type,
+						},
+					},
+				},
+			}
+
+			targetGroup := fmt.Sprintf("%sTargetGroup", key)
+			tmpl.Resources[targetGroup] = troposphere.Resource{
+				Type: "AWS::ElasticLoadBalancingV2::TargetGroup",
+				Properties: map[string]interface{}{
+					"Port":     65535, // Not used. ECS sets a port override when registering targets.
+					"Protocol": "HTTP",
+					"VpcId":    t.VpcId,
+				},
 			}
 
-			listeners = append(listeners, map[string]interface{}{
-				"LoadBalancerPort": 443,
-				"Protocol":         "https",
-				"InstancePort":     GetAtt(instancePort, "InstancePort"),
-				"SSLCertificateId": cert,
-				"InstanceProtocol": "http",
+			httpListener := fmt.Sprintf("%sPort%dListener", loadBalancer, 80)
+			tmpl.Resources[httpListener] = troposphere.Resource{
+				Type: "AWS::ElasticLoadBalancingV2::Listener",
+				Properties: map[string]interface{}{
+					"LoadBalancerArn": Ref(loadBalancer),
+					"Port":            80,
+					"Protocol":        "HTTP",
+					"DefaultActions": []interface{}{
+						map[string]interface{}{
+							"TargetGroupArn": Ref(targetGroup),
+							"Type":           "forward",
+						},
+					},
+				},
+			}
+			serviceDependencies = append(serviceDependencies, httpListener)
+
+			if e, ok := p.Exposure.Type.(*scheduler.HTTPSExposure); ok {
+				var cert interface{}
+				if _, err := arn.Parse(e.Cert); err == nil {
+					cert = e.Cert
+				} else {
+					cert = Join("", "arn:aws:iam::", Ref("AWS::AccountId"), ":server-certificate/", e.Cert)
+				}
+
+				httpsListener := fmt.Sprintf("%sPort%dListener", loadBalancer, 443)
+				tmpl.Resources[httpsListener] = troposphere.Resource{
+					Type: "AWS::ElasticLoadBalancingV2::Listener",
+					Properties: map[string]interface{}{
+						"Certificates": []interface{}{
+							map[string]interface{}{
+								"CertificateArn": cert,
+							},
+						},
+						"LoadBalancerArn": GetAtt(loadBalancer, "Arn"),
+						"Port":            443,
+						"Protocol":        "HTTPS",
+						"DefaultActions": []interface{}{
+							map[string]interface{}{
+								"TargetGroupArn": Ref(targetGroup),
+								"Type":           "forward",
+							},
+						},
+					},
+				}
+				serviceDependencies = append(serviceDependencies, httpsListener)
+			}
+
+			loadBalancers = append(loadBalancers, map[string]interface{}{
+				"ContainerName":  p.Type,
+				"ContainerPort":  ContainerPort,
+				"TargetGroupArn": Ref(targetGroup),
 			})
-		}
+			portMappings = append(portMappings, &PortMappingProperties{
+				ContainerPort: ContainerPort,
+				HostPort:      0,
+			})
+		default:
+			loadBalancer = fmt.Sprintf("%sLoadBalancer", key)
 
-		portMappings = append(portMappings, &PortMappingProperties{
-			ContainerPort: ContainerPort,
-			HostPort:      GetAtt(instancePort, "InstancePort"),
-		})
-		p.Env["PORT"] = fmt.Sprintf("%d", ContainerPort)
+			instancePort := fmt.Sprintf("%s%dInstancePort", key, ContainerPort)
+			tmpl.Resources[instancePort] = troposphere.Resource{
+				Type:    "Custom::InstancePort",
+				Version: "1.0",
+				Properties: map[string]interface{}{
+					"ServiceToken": t.CustomResourcesTopic,
+				},
+			}
 
-		loadBalancer := fmt.Sprintf("%sLoadBalancer", key)
-		loadBalancers = append(loadBalancers, map[string]interface{}{
-			"ContainerName":    p.Type,
-			"ContainerPort":    ContainerPort,
-			"LoadBalancerName": Ref(loadBalancer),
-		})
-		tmpl.Resources[loadBalancer] = troposphere.Resource{
-			Type: "AWS::ElasticLoadBalancing::LoadBalancer",
-			Properties: map[string]interface{}{
-				"Scheme":         scheme,
-				"SecurityGroups": []string{sg},
-				"Subnets":        subnets,
-				"Listeners":      listeners,
-				"CrossZone":      true,
-				"Tags": []map[string]string{
-					map[string]string{
-						"Key":   "empire.app.process",
-						"Value": p.Type,
-					},
+			listeners := []map[string]interface{}{
+				map[string]interface{}{
+					"LoadBalancerPort": 80,
+					"Protocol":         "http",
+					"InstancePort":     GetAtt(instancePort, "InstancePort"),
+					"InstanceProtocol": "http",
 				},
-				"ConnectionDrainingPolicy": map[string]interface{}{
-					"Enabled": true,
-					"Timeout": defaultConnectionDrainingTimeout,
+			}
+
+			if e, ok := p.Exposure.Type.(*scheduler.HTTPSExposure); ok {
+				var cert interface{}
+				if _, err := arn.Parse(e.Cert); err == nil {
+					cert = e.Cert
+				} else {
+					cert = Join("", "arn:aws:iam::", Ref("AWS::AccountId"), ":server-certificate/", e.Cert)
+				}
+
+				listeners = append(listeners, map[string]interface{}{
+					"LoadBalancerPort": 443,
+					"Protocol":         "https",
+					"InstancePort":     GetAtt(instancePort, "InstancePort"),
+					"SSLCertificateId": cert,
+					"InstanceProtocol": "http",
+				})
+			}
+
+			tmpl.Resources[loadBalancer] = troposphere.Resource{
+				Type: "AWS::ElasticLoadBalancing::LoadBalancer",
+				Properties: map[string]interface{}{
+					"Scheme":         scheme,
+					"SecurityGroups": []string{sg},
+					"Subnets":        subnets,
+					"Listeners":      listeners,
+					"CrossZone":      true,
+					"Tags": []map[string]string{
+						map[string]string{
+							"Key":   "empire.app.process",
+							"Value": p.Type,
+						},
+					},
+					"ConnectionDrainingPolicy": map[string]interface{}{
+						"Enabled": true,
+						"Timeout": defaultConnectionDrainingTimeout,
+					},
 				},
-			},
+			}
+
+			loadBalancers = append(loadBalancers, map[string]interface{}{
+				"ContainerName":    p.Type,
+				"ContainerPort":    ContainerPort,
+				"LoadBalancerName": Ref(loadBalancer),
+			})
+			portMappings = append(portMappings, &PortMappingProperties{
+				ContainerPort: ContainerPort,
+				HostPort:      GetAtt(instancePort, "InstancePort"),
+			})
 		}
 
 		if p.Type == "web" {
@@ -421,7 +530,6 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A
 	containerDefinition.DockerLabels[restartLabel] = Ref(restartParameter)
 	containerDefinition.PortMappings = portMappings
 
-	service := fmt.Sprintf("%sService", key)
 	serviceProperties := map[string]interface{}{
 		"Cluster":        t.Cluster,
 		"DesiredCount":   Ref(scaleParameter(p.Type)),
@@ -433,11 +541,18 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A
 	if len(loadBalancers) > 0 {
 		serviceProperties["Role"] = t.ServiceRole
 	}
-	tmpl.Resources[service] = troposphere.Resource{
-		Type:       ecsServiceType,
-		Properties: serviceProperties,
+	service := troposphere.NamedResource{
+		Name: fmt.Sprintf("%sService", key),
+		Resource: troposphere.Resource{
+			Type:       ecsServiceType,
+			Properties: serviceProperties,
+		},
+	}
+	if len(serviceDependencies) > 0 {
+		service.Resource.DependsOn = serviceDependencies
 	}
-	return service
+	tmpl.AddResource(service)
+	return service.Name
 }
 
 // If the ServiceRole option is not an ARN, it will return a CloudFormation