Skip to content

Commit

Permalink
Switch to Application Load Balancers.
Browse files Browse the repository at this point in the history
  • Loading branch information
@ejholmes
ejholmes committed Aug 11, 2016
1 parent b7ecf89 commit b405ade
Show file tree
Hide file tree
Showing 9 changed files with 1,000 additions and 71 deletions.
1 change: 1 addition & 0 deletions cmd/empire/factories.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -149,6 +149,7 @@ func newCloudFormationScheduler(db *empire.DB, c *Context) (*cloudformation.Sche
}

t := &cloudformation.EmpireTemplate{
VpcId: c.String(FlagELBVpcId),
Cluster: c.String(FlagECSCluster),
InternalSecurityGroupID: c.String(FlagELBSGPrivate),
ExternalSecurityGroupID: c.String(FlagELBSGPublic),
Expand Down
6 changes: 6 additions & 0 deletions cmd/empire/main.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,7 @@ const (

FlagELBSGPrivate = "elb.sg.private"
FlagELBSGPublic = "elb.sg.public"
FlagELBVpcId = "elb.vpc.id"

FlagEC2SubnetsPrivate = "ec2.subnets.private"
FlagEC2SubnetsPublic = "ec2.subnets.public"
Expand Down Expand Up @@ -278,6 +279,11 @@ var EmpireFlags = []cli.Flag{
Usage: "The ELB security group to assign public load balancers",
EnvVar: "EMPIRE_ELB_SG_PUBLIC",
},
cli.StringFlag{
Name: FlagELBVpcId,
Usage: "The comma separated private subnet ids",
EnvVar: "EMPIRE_ELB_VPC_ID",
},
cli.StringSliceFlag{
Name: FlagEC2SubnetsPrivate,
Value: &cli.StringSlice{},
Expand Down
11 changes: 1 addition & 10 deletions docs/cloudformation.json
View file
Original file line number Diff line number Diff line change
Expand Up @@ -507,16 +507,7 @@
{
"Effect": "Allow",
"Action": [
"elasticloadbalancing:DeleteLoadBalancer",
"elasticloadbalancing:CreateLoadBalancer",
"elasticloadbalancing:DescribeLoadBalancers",
"elasticloadbalancing:DescribeTags",
"elasticloadbalancing:ConfigureHealthCheck",
"elasticloadbalancing:ModifyLoadBalancerAttributes",
"elasticloadbalancing:SetLoadBalancerListenerSSLCertificate",
"elasticloadbalancing:CreateLoadBalancerListeners",
"elasticloadbalancing:DeleteLoadBalancerListeners",
"elasticloadbalancing:SetLoadBalancerPoliciesOfListener"
"elasticloadbalancing:*"
],
"Resource": ["*"]
},
Expand Down
1 change: 1 addition & 0 deletions pkg/troposphere/troposphere.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -45,6 +45,7 @@ type Output struct {
// Resource represents a CloudFormation Resource.
type Resource struct {
Condition interface{} `json:"Condition,omitempty"`
DependsOn interface{} `json:"DependsOn,omitempty"`
Properties interface{} `json:"Properties,omitempty"`
Type interface{} `json:"Type,omitempty"`
Version interface{} `json:"Version,omitempty"`
Expand Down
237 changes: 176 additions & 61 deletions scheduler/cloudformation/template.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,12 @@ var (
Join = troposphere.Join
)

// Load balancer types
var (
classicLoadBalancer = "elb"
applicationLoadBalancer = "alb"
)

const (
// For HTTP/HTTPS/TCP services, we allocate an ELB and map it's instance port to
// the container port. This is the port that processes within the container
Expand Down Expand Up @@ -59,6 +65,10 @@ type EmpireTemplate struct {
// The ECS cluster to run the services in.
Cluster string

// The VPC to create ALB target groups within. Should be the same VPC
// that ECS services will run within.
VpcId string

// The hosted zone to add CNAME's to.
HostedZone *route53.HostedZone

Expand Down Expand Up @@ -93,6 +103,9 @@ func (t *EmpireTemplate) Validate() error {
return errors.New(fmt.Sprintf("%s is required", n))
}

if t.VpcId == "" {
return r("VpcId")
}
if t.Cluster == "" {
return r("Cluster")
}
Expand Down Expand Up @@ -321,6 +334,7 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A

var portMappings []*PortMappingProperties

var serviceDependencies []string
loadBalancers := []map[string]interface{}{}
if p.Exposure != nil {
scheme := schemeInternal
Expand All @@ -333,72 +347,167 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A
subnets = t.ExternalSubnetIDs
}

instancePort := fmt.Sprintf("%s%dInstancePort", key, ContainerPort)
tmpl.Resources[instancePort] = troposphere.Resource{
Type: "Custom::InstancePort",
Version: "1.0",
Properties: map[string]interface{}{
"ServiceToken": t.CustomResourcesTopic,
},
}
p.Env["PORT"] = fmt.Sprintf("%d", ContainerPort)

listeners := []map[string]interface{}{
map[string]interface{}{
"LoadBalancerPort": 80,
"Protocol": "http",
"InstancePort": GetAtt(instancePort, "InstancePort"),
"InstanceProtocol": "http",
},
loadBalancerType := classicLoadBalancer
if v, ok := app.Env["LOAD_BALANCER_TYPE"]; ok {
loadBalancerType = v
}

if e, ok := p.Exposure.Type.(*scheduler.HTTPSExposure); ok {
var cert interface{}
if _, err := arn.Parse(e.Cert); err == nil {
cert = e.Cert
} else {
cert = Join("", "arn:aws:iam::", Ref("AWS::AccountId"), ":server-certificate/", e.Cert)
var loadBalancer string
switch loadBalancerType {
case applicationLoadBalancer:
loadBalancer = fmt.Sprintf("%sApplicationLoadBalancer", key)
tmpl.Resources[loadBalancer] = troposphere.Resource{
Type: "AWS::ElasticLoadBalancingV2::LoadBalancer",
Properties: map[string]interface{}{
"Scheme": scheme,
"SecurityGroups": []string{sg},
"Subnets": subnets,
"Tags": []map[string]string{
map[string]string{
"Key": "empire.app.process",
"Value": p.Type,
},
},
},
}

targetGroup := fmt.Sprintf("%sTargetGroup", key)
tmpl.Resources[targetGroup] = troposphere.Resource{
Type: "AWS::ElasticLoadBalancingV2::TargetGroup",
Properties: map[string]interface{}{
"Port": 65535, // Not used. ECS sets a port override when registering targets.
"Protocol": "HTTP",
"VpcId": t.VpcId,
},
}

listeners = append(listeners, map[string]interface{}{
"LoadBalancerPort": 443,
"Protocol": "https",
"InstancePort": GetAtt(instancePort, "InstancePort"),
"SSLCertificateId": cert,
"InstanceProtocol": "http",
httpListener := fmt.Sprintf("%sPort%dListener", loadBalancer, 80)
tmpl.Resources[httpListener] = troposphere.Resource{
Type: "AWS::ElasticLoadBalancingV2::Listener",
Properties: map[string]interface{}{
"LoadBalancerArn": Ref(loadBalancer),
"Port": 80,
"Protocol": "HTTP",
"DefaultActions": []interface{}{
map[string]interface{}{
"TargetGroupArn": Ref(targetGroup),
"Type": "forward",
},
},
},
}
serviceDependencies = append(serviceDependencies, httpListener)

if e, ok := p.Exposure.Type.(*scheduler.HTTPSExposure); ok {
var cert interface{}
if _, err := arn.Parse(e.Cert); err == nil {
cert = e.Cert
} else {
cert = Join("", "arn:aws:iam::", Ref("AWS::AccountId"), ":server-certificate/", e.Cert)
}

httpsListener := fmt.Sprintf("%sPort%dListener", loadBalancer, 443)
tmpl.Resources[httpsListener] = troposphere.Resource{
Type: "AWS::ElasticLoadBalancingV2::Listener",
Properties: map[string]interface{}{
"Certificates": []interface{}{
map[string]interface{}{
"CertificateArn": cert,
},
},
"LoadBalancerArn": GetAtt(loadBalancer, "Arn"),
"Port": 443,
"Protocol": "HTTPS",
"DefaultActions": []interface{}{
map[string]interface{}{
"TargetGroupArn": Ref(targetGroup),
"Type": "forward",
},
},
},
}
serviceDependencies = append(serviceDependencies, httpsListener)
}

loadBalancers = append(loadBalancers, map[string]interface{}{
"ContainerName": p.Type,
"ContainerPort": ContainerPort,
"TargetGroupArn": Ref(targetGroup),
})
}
portMappings = append(portMappings, &PortMappingProperties{
ContainerPort: ContainerPort,
HostPort: 0,
})
default:
loadBalancer = fmt.Sprintf("%sLoadBalancer", key)

portMappings = append(portMappings, &PortMappingProperties{
ContainerPort: ContainerPort,
HostPort: GetAtt(instancePort, "InstancePort"),
})
p.Env["PORT"] = fmt.Sprintf("%d", ContainerPort)
instancePort := fmt.Sprintf("%s%dInstancePort", key, ContainerPort)
tmpl.Resources[instancePort] = troposphere.Resource{
Type: "Custom::InstancePort",
Version: "1.0",
Properties: map[string]interface{}{
"ServiceToken": t.CustomResourcesTopic,
},
}

loadBalancer := fmt.Sprintf("%sLoadBalancer", key)
loadBalancers = append(loadBalancers, map[string]interface{}{
"ContainerName": p.Type,
"ContainerPort": ContainerPort,
"LoadBalancerName": Ref(loadBalancer),
})
tmpl.Resources[loadBalancer] = troposphere.Resource{
Type: "AWS::ElasticLoadBalancing::LoadBalancer",
Properties: map[string]interface{}{
"Scheme": scheme,
"SecurityGroups": []string{sg},
"Subnets": subnets,
"Listeners": listeners,
"CrossZone": true,
"Tags": []map[string]string{
map[string]string{
"Key": "empire.app.process",
"Value": p.Type,
},
listeners := []map[string]interface{}{
map[string]interface{}{
"LoadBalancerPort": 80,
"Protocol": "http",
"InstancePort": GetAtt(instancePort, "InstancePort"),
"InstanceProtocol": "http",
},
"ConnectionDrainingPolicy": map[string]interface{}{
"Enabled": true,
"Timeout": defaultConnectionDrainingTimeout,
}

if e, ok := p.Exposure.Type.(*scheduler.HTTPSExposure); ok {
var cert interface{}
if _, err := arn.Parse(e.Cert); err == nil {
cert = e.Cert
} else {
cert = Join("", "arn:aws:iam::", Ref("AWS::AccountId"), ":server-certificate/", e.Cert)
}

listeners = append(listeners, map[string]interface{}{
"LoadBalancerPort": 443,
"Protocol": "https",
"InstancePort": GetAtt(instancePort, "InstancePort"),
"SSLCertificateId": cert,
"InstanceProtocol": "http",
})
}

tmpl.Resources[loadBalancer] = troposphere.Resource{
Type: "AWS::ElasticLoadBalancing::LoadBalancer",
Properties: map[string]interface{}{
"Scheme": scheme,
"SecurityGroups": []string{sg},
"Subnets": subnets,
"Listeners": listeners,
"CrossZone": true,
"Tags": []map[string]string{
map[string]string{
"Key": "empire.app.process",
"Value": p.Type,
},
},
"ConnectionDrainingPolicy": map[string]interface{}{
"Enabled": true,
"Timeout": defaultConnectionDrainingTimeout,
},
},
},
}

loadBalancers = append(loadBalancers, map[string]interface{}{
"ContainerName": p.Type,
"ContainerPort": ContainerPort,
"LoadBalancerName": Ref(loadBalancer),
})
portMappings = append(portMappings, &PortMappingProperties{
ContainerPort: ContainerPort,
HostPort: GetAtt(instancePort, "InstancePort"),
})
}

if p.Type == "web" {
Expand All @@ -421,7 +530,6 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A
containerDefinition.DockerLabels[restartLabel] = Ref(restartParameter)
containerDefinition.PortMappings = portMappings

service := fmt.Sprintf("%sService", key)
serviceProperties := map[string]interface{}{
"Cluster": t.Cluster,
"DesiredCount": Ref(scaleParameter(p.Type)),
Expand All @@ -433,11 +541,18 @@ func (t *EmpireTemplate) addService(tmpl *troposphere.Template, app *scheduler.A
if len(loadBalancers) > 0 {
serviceProperties["Role"] = t.ServiceRole
}
tmpl.Resources[service] = troposphere.Resource{
Type: ecsServiceType,
Properties: serviceProperties,
service := troposphere.NamedResource{
Name: fmt.Sprintf("%sService", key),
Resource: troposphere.Resource{
Type: ecsServiceType,
Properties: serviceProperties,
},
}
if len(serviceDependencies) > 0 {
service.Resource.DependsOn = serviceDependencies
}
return service
tmpl.AddResource(service)
return service.Name
}

// If the ServiceRole option is not an ARN, it will return a CloudFormation
Expand Down
Loading

0 comments on commit b405ade

Please sign in to comment.