From e55ac4695ad211284dc20ba1ac74471a8e699baf Mon Sep 17 00:00:00 2001 From: "Eric J. Holmes" Date: Sun, 13 Dec 2015 08:13:57 +0700 Subject: [PATCH 1/3] Simplify Empire core --- apps.go | 16 +- cmd/empire/factories.go | 219 ++++++++++++ cmd/empire/main.go | 79 ----- configs.go | 3 +- db.go | 3 +- deployments.go | 19 +- domains.go | 2 +- empire.go | 322 ++++-------------- empiretest/test.go | 57 +++- extractor.go | 202 ----------- logs.go | 10 +- processes.go | 13 +- procfile/extractor.go | 191 +++++++++++ .../extractor_test.go | 42 +-- procfile/procfile.go | 21 ++ releases.go | 8 +- resolver.go | 90 ----- runner.go | 7 +- slugs.go | 21 +- ssl.go | 13 +- 20 files changed, 606 insertions(+), 732 deletions(-) create mode 100644 cmd/empire/factories.go delete mode 100644 extractor.go create mode 100644 procfile/extractor.go rename extractor_test.go => procfile/extractor_test.go (82%) create mode 100644 procfile/procfile.go delete mode 100644 resolver.go diff --git a/apps.go b/apps.go index 3620963c6..785f507e6 100644 --- a/apps.go +++ b/apps.go @@ -8,7 +8,6 @@ import ( "time" "github.com/jinzhu/gorm" - "github.com/remind101/empire/scheduler" "github.com/remind101/pkg/timex" "golang.org/x/net/context" ) @@ -142,12 +141,11 @@ func AppID(id string) func(*gorm.DB) *gorm.DB { } type appsService struct { - store *store - scheduler scheduler.Scheduler + *Empire } func (s *appsService) AppsDestroy(ctx context.Context, app *App) error { - if err := s.scheduler.Remove(ctx, app.ID); err != nil { + if err := s.Scheduler.Remove(ctx, app.ID); err != nil { return err } @@ -204,8 +202,7 @@ func appsDestroy(db *gorm.DB, app *App) error { // scaler is a small service for scaling an apps process. type scaler struct { - store *store - scheduler scheduler.Scheduler + *Empire } func (s *scaler) Scale(ctx context.Context, app *App, t ProcessType, quantity int, c *Constraints) (*Process, error) { @@ -228,7 +225,7 @@ func (s *scaler) Scale(ctx context.Context, app *App, t ProcessType, quantity in return nil, &ValidationError{Err: fmt.Errorf("no %s process type in release", t)} } - if err := s.scheduler.Scale(ctx, release.AppID, string(p.Type), uint(quantity)); err != nil { + if err := s.Scheduler.Scale(ctx, release.AppID, string(p.Type), uint(quantity)); err != nil { return nil, err } @@ -243,13 +240,12 @@ func (s *scaler) Scale(ctx context.Context, app *App, t ProcessType, quantity in // restarter is a small service for restarting an apps processes. type restarter struct { - scheduler scheduler.Scheduler - releaser *releaser + *Empire } func (s *restarter) Restart(ctx context.Context, app *App, id string) error { if id != "" { - return s.scheduler.Stop(ctx, id) + return s.Scheduler.Stop(ctx, id) } return s.releaser.ReleaseApp(ctx, app) diff --git a/cmd/empire/factories.go b/cmd/empire/factories.go new file mode 100644 index 000000000..671a1a863 --- /dev/null +++ b/cmd/empire/factories.go @@ -0,0 +1,219 @@ +package main + +import ( + "fmt" + "log" + "net/url" + "os" + + "github.com/aws/aws-sdk-go/aws" + "github.com/aws/aws-sdk-go/aws/client" + "github.com/aws/aws-sdk-go/aws/session" + "github.com/codegangsta/cli" + "github.com/inconshreveable/log15" + "github.com/jinzhu/gorm" + "github.com/remind101/empire" + "github.com/remind101/empire/pkg/dockerutil" + "github.com/remind101/empire/pkg/runner" + "github.com/remind101/empire/pkg/sslcert" + "github.com/remind101/empire/scheduler" + "github.com/remind101/empire/scheduler/ecs" + "github.com/remind101/pkg/reporter" + "github.com/remind101/pkg/reporter/hb" +) + +// DB =================================== + +func newDB(c *cli.Context) (*gorm.DB, error) { + return empire.NewDB(c.String(FlagDB)) +} + +// Empire =============================== + +func newEmpire(c *cli.Context) (*empire.Empire, error) { + db, err := newDB(c) + if err != nil { + return nil, err + } + + docker, err := newDockerClient(c) + if err != nil { + return nil, err + } + + reporter, err := newReporter(c) + if err != nil { + return nil, err + } + + scheduler, err := newScheduler(c) + if err != nil { + return nil, err + } + + certManager, err := newCertManager(c) + if err != nil { + return nil, err + } + + logs, err := newLogsStreamer(c) + if err != nil { + return nil, err + } + + e := empire.New(db, empire.Options{ + Secret: c.String(FlagSecret), + }) + e.Reporter = reporter + e.Scheduler = scheduler + e.CertManager = certManager + e.LogsStreamer = logs + e.ExtractProcfile = empire.PullAndExtract(docker) + e.Logger = newLogger() + + return e, nil +} + +// Scheduler ============================ + +func newScheduler(c *cli.Context) (scheduler.Scheduler, error) { + return newECSScheduler(c) +} + +func newECSScheduler(c *cli.Context) (scheduler.Scheduler, error) { + config := ecs.Config{ + AWS: newConfigProvider(c), + Cluster: c.String(FlagECSCluster), + ServiceRole: c.String(FlagECSServiceRole), + InternalSecurityGroupID: c.String(FlagELBSGPrivate), + ExternalSecurityGroupID: c.String(FlagELBSGPublic), + InternalSubnetIDs: c.StringSlice(FlagEC2SubnetsPrivate), + ExternalSubnetIDs: c.StringSlice(FlagEC2SubnetsPublic), + ZoneID: c.String(FlagRoute53InternalZoneID), + } + + s, err := ecs.NewLoadBalancedScheduler(config) + if err != nil { + return nil, err + } + + r, err := newDockerRunner(c) + if err != nil { + return nil, err + } + + log.Println("Using ECS backend with the following configuration:") + log.Println(fmt.Sprintf(" Cluster: %v", config.Cluster)) + log.Println(fmt.Sprintf(" ServiceRole: %v", config.ServiceRole)) + log.Println(fmt.Sprintf(" InternalSecurityGroupID: %v", config.InternalSecurityGroupID)) + log.Println(fmt.Sprintf(" ExternalSecurityGroupID: %v", config.ExternalSecurityGroupID)) + log.Println(fmt.Sprintf(" InternalSubnetIDs: %v", config.InternalSubnetIDs)) + log.Println(fmt.Sprintf(" ExternalSubnetIDs: %v", config.ExternalSubnetIDs)) + log.Println(fmt.Sprintf(" ZoneID: %v", config.ZoneID)) + + return &scheduler.AttachedRunner{ + Scheduler: s, + Runner: r, + }, nil +} + +func newConfigProvider(c *cli.Context) client.ConfigProvider { + p := session.New() + + if c.Bool(FlagAWSDebug) { + config := &aws.Config{} + config.WithLogLevel(1) + p = session.New(config) + } + + return p +} + +func newDockerRunner(c *cli.Context) (*runner.Runner, error) { + client, err := newDockerClient(c) + if err != nil { + return nil, err + } + return runner.NewRunner(client), nil +} + +// DockerClient ======================== + +func newDockerClient(c *cli.Context) (*dockerutil.Client, error) { + socket := c.String(FlagDockerSocket) + certPath := c.String(FlagDockerCert) + auth, err := dockerAuth(c.String(FlagDockerAuth)) + if err != nil { + return nil, err + } + + return dockerutil.NewClient(auth, socket, certPath) +} + +// CertManager ========================= + +func newCertManager(c *cli.Context) (sslcert.Manager, error) { + return newIAMCertManager(c) +} + +func newIAMCertManager(c *cli.Context) (sslcert.Manager, error) { + return sslcert.NewIAMManager(newConfigProvider(c), "/empire/certs/"), nil +} + +// LogStreamer ========================= + +func newLogsStreamer(c *cli.Context) (empire.LogsStreamer, error) { + switch c.String(FlagLogsStreamer) { + case "kinesis": + return newKinesisLogsStreamer(c) + default: + log.Println("Streaming logs are disabled") + return nil, nil + } +} + +func newKinesisLogsStreamer(c *cli.Context) (empire.LogsStreamer, error) { + log.Println("Using Kinesis backend for log streaming") + return empire.NewKinesisLogsStreamer(), nil +} + +// Logger ============================== + +func newLogger() log15.Logger { + l := log15.New() + h := log15.StreamHandler(os.Stdout, log15.LogfmtFormat()) + l.SetHandler(log15.LazyHandler(h)) + return l +} + +// Reporter ============================ + +func newReporter(c *cli.Context) (reporter.Reporter, error) { + u := c.String(FlagReporter) + if u == "" { + return empire.DefaultReporter, nil + } + + uri, err := url.Parse(u) + if err != nil { + return nil, err + } + + switch uri.Scheme { + case "hb": + log.Println("Using Honeybadger to report errors") + q := uri.Query() + return newHBReporter(q.Get("key"), q.Get("environment")) + default: + panic(fmt.Errorf("unknown reporter: %s", u)) + } +} + +func newHBReporter(key, env string) (reporter.Reporter, error) { + r := hb.NewReporter(key) + r.Environment = env + + // Append here because `go vet` will complain about unkeyed fields, + // since it thinks MultiReporter is a struct literal. + return append(reporter.MultiReporter{}, empire.DefaultReporter, r), nil +} diff --git a/cmd/empire/main.go b/cmd/empire/main.go index f7591b019..d5b39b42f 100644 --- a/cmd/empire/main.go +++ b/cmd/empire/main.go @@ -1,19 +1,12 @@ package main import ( - "fmt" - "net/url" "os" "path" - "github.com/aws/aws-sdk-go/aws" - "github.com/aws/aws-sdk-go/aws/session" "github.com/codegangsta/cli" "github.com/fsouza/go-dockerclient" - "github.com/remind101/empire" "github.com/remind101/empire/server/github" - "github.com/remind101/pkg/reporter" - "github.com/remind101/pkg/reporter/hb" ) const ( @@ -247,78 +240,6 @@ func main() { app.Run(os.Args) } -func newEmpire(c *cli.Context) (*empire.Empire, error) { - opts := empire.Options{} - - opts.Docker.Socket = c.String(FlagDockerSocket) - opts.Docker.CertPath = c.String(FlagDockerCert) - opts.AWSConfig = session.New() - if c.Bool(FlagAWSDebug) { - config := &aws.Config{} - config.WithLogLevel(1) - opts.AWSConfig = session.New(config) - } - opts.ECS.Cluster = c.String(FlagECSCluster) - opts.ECS.ServiceRole = c.String(FlagECSServiceRole) - opts.ELB.InternalSecurityGroupID = c.String(FlagELBSGPrivate) - opts.ELB.ExternalSecurityGroupID = c.String(FlagELBSGPublic) - opts.ELB.InternalSubnetIDs = c.StringSlice(FlagEC2SubnetsPrivate) - opts.ELB.ExternalSubnetIDs = c.StringSlice(FlagEC2SubnetsPublic) - opts.ELB.InternalZoneID = c.String(FlagRoute53InternalZoneID) - opts.DB = c.String(FlagDB) - opts.Secret = c.String(FlagSecret) - opts.LogsStreamer = c.String(FlagLogsStreamer) - - auth, err := dockerAuth(c.String(FlagDockerAuth)) - if err != nil { - return nil, err - } - - opts.Docker.Auth = auth - - e, err := empire.New(opts) - if err != nil { - return e, err - } - - reporter, err := newReporter(c.String(FlagReporter)) - if err != nil { - return e, err - } - - e.Reporter = reporter - - return e, nil -} - -func newReporter(u string) (reporter.Reporter, error) { - if u == "" { - return empire.DefaultReporter, nil - } - - uri, err := url.Parse(u) - if err != nil { - return nil, err - } - - switch uri.Scheme { - case "hb": - q := uri.Query() - return newHBReporter(q.Get("key"), q.Get("environment")) - default: - panic(fmt.Errorf("unknown reporter: %s", u)) - } -} - -func newHBReporter(key, env string) (reporter.Reporter, error) { - r := hb.NewReporter(key) - r.Environment = env - - // Append here because `go vet` will complain about unkeyed fields, - // since it thinks MultiReporter is a struct literal. - return append(reporter.MultiReporter{}, empire.DefaultReporter, r), nil -} - func dockerAuth(path string) (*docker.AuthConfigurations, error) { f, err := os.Open(path) if err != nil { diff --git a/configs.go b/configs.go index 7914a1366..23e43bf65 100644 --- a/configs.go +++ b/configs.go @@ -120,8 +120,7 @@ func configsCreate(db *gorm.DB, config *Config) (*Config, error) { } type configsService struct { - store *store - releases *releasesService + *Empire } func (s *configsService) ConfigsApply(ctx context.Context, app *App, vars Vars) (*Config, error) { diff --git a/db.go b/db.go index 0a476354f..72e464c29 100644 --- a/db.go +++ b/db.go @@ -7,7 +7,8 @@ import ( "github.com/jinzhu/gorm" ) -func newDB(uri string) (*gorm.DB, error) { +// NewDB returns a new gorm.DB instance. +func NewDB(uri string) (*gorm.DB, error) { u, err := url.Parse(uri) if err != nil { return nil, err diff --git a/deployments.go b/deployments.go index b243cbce8..b045ff3df 100644 --- a/deployments.go +++ b/deployments.go @@ -34,19 +34,10 @@ type DeploymentsCreateOpts struct { Output io.Writer } -// deployer is an interface that represents something that can perform a -// deployment. -type deployer interface { - Deploy(context.Context, DeploymentsCreateOpts) (*Release, error) -} - // deployerService is an implementation of the deployer interface that performs // the core business logic to deploy. type deployerService struct { - *appsService - *configsService - *slugsService - *releasesService + *Empire } // doDeploy does the actual deployment @@ -57,14 +48,14 @@ func (s *deployerService) doDeploy(ctx context.Context, opts DeploymentsCreateOp // images repository, or create it if not found. if app == nil { var err error - app, err = s.appsService.AppsFindOrCreateByRepo(img.Repository) + app, err = s.apps.AppsFindOrCreateByRepo(img.Repository) if err != nil { return nil, err } } else { // If the app doesn't already have a repo attached to it, we'll attach // this image's repo. - if err := s.appsService.AppsEnsureRepo(app, img.Repository); err != nil { + if err := s.apps.AppsEnsureRepo(app, img.Repository); err != nil { return nil, err } } @@ -76,7 +67,7 @@ func (s *deployerService) doDeploy(ctx context.Context, opts DeploymentsCreateOp } // Create a new slug for the docker image. - slug, err := s.SlugsCreateByImage(ctx, img, opts.Output) + slug, err := s.slugs.SlugsCreateByImage(ctx, img, opts.Output) if err != nil { return nil, err } @@ -85,7 +76,7 @@ func (s *deployerService) doDeploy(ctx context.Context, opts DeploymentsCreateOp // and Slug. desc := fmt.Sprintf("Deploy %s", img.String()) - r, err := s.ReleasesCreate(ctx, &Release{ + r, err := s.releases.ReleasesCreate(ctx, &Release{ App: app, Config: config, Slug: slug, diff --git a/domains.go b/domains.go index dde3e3e5d..979a00a8a 100644 --- a/domains.go +++ b/domains.go @@ -31,7 +31,7 @@ func (d *Domain) BeforeCreate() error { } type domainsService struct { - store *store + *Empire } func (s *domainsService) DomainsCreate(domain *Domain) (*Domain, error) { diff --git a/empire.go b/empire.go index 35d883650..be578b4a3 100644 --- a/empire.go +++ b/empire.go @@ -2,19 +2,18 @@ package empire // import "github.com/remind101/empire" import ( "io" - "log" - "os" + "io/ioutil" - "github.com/aws/aws-sdk-go/aws/client" "github.com/docker/docker/pkg/jsonmessage" "github.com/fsouza/go-dockerclient" "github.com/inconshreveable/log15" + "github.com/jinzhu/gorm" "github.com/mattes/migrate/migrate" "github.com/remind101/empire/pkg/dockerutil" - "github.com/remind101/empire/pkg/runner" + "github.com/remind101/empire/pkg/image" "github.com/remind101/empire/pkg/sslcert" + "github.com/remind101/empire/procfile" "github.com/remind101/empire/scheduler" - "github.com/remind101/empire/scheduler/ecs" "github.com/remind101/pkg/reporter" "golang.org/x/net/context" ) @@ -36,58 +35,12 @@ const ( WebProcessType = "web" ) -// DockerOptions is a set of options to configure a docker api client. -type DockerOptions struct { - // The unix socket to connect to the docker api. - Socket string - - // Path to a certificate to use for TLS connections. - CertPath string - - // A set of docker registry credentials. - Auth *docker.AuthConfigurations -} - -// ECSOptions is a set of options to configure ECS. -type ECSOptions struct { - Cluster string - ServiceRole string -} - -// ELBOptions is a set of options to configure ELB. -type ELBOptions struct { - // The Security Group ID to assign when creating internal load balancers. - InternalSecurityGroupID string - - // The Security Group ID to assign when creating external load balancers. - ExternalSecurityGroupID string - - // The Subnet IDs to assign when creating internal load balancers. - InternalSubnetIDs []string - - // The Subnet IDs to assign when creating external load balancers. - ExternalSubnetIDs []string - - // Zone ID of the internal zone to add cnames for each elb - InternalZoneID string -} +// ProcfileExtractor is a function that can extract a Procfile from an image. +type ProcfileExtractor func(context.Context, image.Image, io.Writer) (procfile.Procfile, error) // Options is provided to New to configure the Empire services. type Options struct { - Docker DockerOptions - ECS ECSOptions - ELB ELBOptions - - // AWS Configuration - AWSConfig client.ConfigProvider - Secret string - - // Database connection string. - DB string - - // Location of the app logs - LogsStreamer string } // Empire is a context object that contains a collection of services. @@ -108,131 +61,49 @@ type Empire struct { domains *domainsService jobStates *processStatesService releases *releasesService - deployer deployer + releaser *releaser + deployer *deployerService scaler *scaler restarter *restarter runner *runnerService - logs LogsStreamer -} - -// New returns a new Empire instance. -func New(options Options) (*Empire, error) { - db, err := newDB(options.DB) - if err != nil { - return nil, err - } - - store := &store{db: db} - - extractor, err := newExtractor(options.Docker) - if err != nil { - return nil, err - } - - resolver, err := newResolver(options.Docker) - if err != nil { - return nil, err - } - - runner, err := newRunner(options.Docker) - if err != nil { - return nil, err - } - - scheduler, err := newManager( - runner, - options.ECS, - options.ELB, - options.AWSConfig, - ) - if err != nil { - return nil, err - } + slugs *slugsService - accessTokens := &accessTokensService{ - Secret: []byte(options.Secret), - } - - apps := &appsService{ - store: store, - scheduler: scheduler, - } + // Scheduler is the backend scheduler used to run applications. + Scheduler scheduler.Scheduler - jobStates := &processStatesService{ - scheduler: scheduler, - } - - scaler := &scaler{ - store: store, - scheduler: scheduler, - } + // CertManager is the backend used to store SSL/TLS certificates. + CertManager sslcert.Manager - releaser := &releaser{ - store: store, - scheduler: scheduler, - } - - restarter := &restarter{ - releaser: releaser, - scheduler: scheduler, - } - - releases := &releasesService{ - store: store, - releaser: releaser, - } - - configs := &configsService{ - store: store, - releases: releases, - } - - domains := &domainsService{ - store: store, - } - - slugs := &slugsService{ - store: store, - extractor: extractor, - resolver: resolver, - } + // LogsStreamer is the backend used to stream application logs. + LogsStreamer LogsStreamer - deployer := &deployerService{ - appsService: apps, - configsService: configs, - slugsService: slugs, - releasesService: releases, - } - - certs := &certificatesService{ - store: store, - manager: newCertManager(options.AWSConfig), - releaser: releaser, - } + // ExtractProcfile is called during deployments to extract the Procfile + // from the newly deployed image. + ExtractProcfile ProcfileExtractor +} - runnerService := &runnerService{ - store: store, - scheduler: scheduler, +// New returns a new Empire instance. +func New(db *gorm.DB, options Options) *Empire { + e := &Empire{ + Logger: nullLogger(), + LogsStreamer: logsDisabled, + store: &store{db: db}, } - logs := newLogStreamer(options.LogsStreamer) - - return &Empire{ - Logger: newLogger(), - store: store, - accessTokens: accessTokens, - apps: apps, - certs: certs, - configs: configs, - deployer: deployer, - domains: domains, - jobStates: jobStates, - scaler: scaler, - restarter: restarter, - runner: runnerService, - releases: releases, - logs: logs, - }, nil + e.accessTokens = &accessTokensService{Secret: []byte(options.Secret)} + e.apps = &appsService{Empire: e} + e.certs = &certificatesService{Empire: e} + e.configs = &configsService{Empire: e} + e.deployer = &deployerService{Empire: e} + e.domains = &domainsService{Empire: e} + e.slugs = &slugsService{Empire: e} + e.jobStates = &processStatesService{Empire: e} + e.scaler = &scaler{Empire: e} + e.restarter = &restarter{Empire: e} + e.runner = &runnerService{Empire: e} + e.releases = &releasesService{Empire: e} + e.releaser = &releaser{Empire: e} + return e } // AccessTokensFind finds an access token. @@ -359,15 +230,6 @@ func (e *Empire) Deploy(ctx context.Context, opts DeploymentsCreateOpts) (*Relea return e.deployer.Deploy(ctx, opts) } -func newJSONMessageError(err error) jsonmessage.JSONMessage { - return jsonmessage.JSONMessage{ - ErrorMessage: err.Error(), - Error: &jsonmessage.JSONError{ - Message: err.Error(), - }, - } -} - // AppsScale scales an apps process. func (e *Empire) AppsScale(ctx context.Context, app *App, t ProcessType, quantity int, c *Constraints) (*Process, error) { return e.scaler.Scale(ctx, app, t, quantity, c) @@ -375,7 +237,7 @@ func (e *Empire) AppsScale(ctx context.Context, app *App, t ProcessType, quantit // Streamlogs streams logs from an app. func (e *Empire) StreamLogs(app *App, w io.Writer) error { - return e.logs.StreamLogs(app, w) + return e.LogsStreamer.StreamLogs(app, w) } // Reset resets empire. @@ -410,86 +272,42 @@ const ( UserKey key = 0 ) -func newManager(r *runner.Runner, ecsOpts ECSOptions, elbOpts ELBOptions, config client.ConfigProvider) (scheduler.Scheduler, error) { - if config == nil { - log.Println("warn: AWS not configured, ECS service management disabled.") - return scheduler.NewFakeScheduler(), nil - } - - s, err := ecs.NewLoadBalancedScheduler(ecs.Config{ - Cluster: ecsOpts.Cluster, - ServiceRole: ecsOpts.ServiceRole, - InternalSecurityGroupID: elbOpts.InternalSecurityGroupID, - ExternalSecurityGroupID: elbOpts.ExternalSecurityGroupID, - InternalSubnetIDs: elbOpts.InternalSubnetIDs, - ExternalSubnetIDs: elbOpts.ExternalSubnetIDs, - AWS: config, - ZoneID: elbOpts.InternalZoneID, - }) - if err != nil { - return nil, err - } - - return &scheduler.AttachedRunner{ - Scheduler: s, - Runner: r, - }, nil -} - -func newCertManager(config client.ConfigProvider) sslcert.Manager { - if config == nil { - log.Println("warn: AWS not configured, IAM server certificate management disabled.") - return sslcert.NewFakeManager() - } - - return sslcert.NewIAMManager(config, "/empire/certs/") -} - -func newRunner(o DockerOptions) (*runner.Runner, error) { - if o.Socket == "" { - return nil, nil - } - - c, err := dockerutil.NewClient(o.Auth, o.Socket, o.CertPath) - if err != nil { - return nil, err +func newJSONMessageError(err error) jsonmessage.JSONMessage { + return jsonmessage.JSONMessage{ + ErrorMessage: err.Error(), + Error: &jsonmessage.JSONError{ + Message: err.Error(), + }, } - - return runner.NewRunner(c), nil } -func newLogger() log15.Logger { +func nullLogger() log15.Logger { l := log15.New() - h := log15.StreamHandler(os.Stdout, log15.LogfmtFormat()) - //h = log15.CallerStackHandler("%+n", h) - l.SetHandler(log15.LazyHandler(h)) + h := log15.StreamHandler(ioutil.Discard, log15.LogfmtFormat()) + l.SetHandler(h) return l } -func newExtractor(o DockerOptions) (Extractor, error) { - if o.Socket == "" { - log.Println("warn: docker socket not configured, docker command extractor disabled.") - return &fakeExtractor{}, nil - } - - c, err := dockerutil.NewDockerClient(o.Socket, o.CertPath) - return newProcfileFallbackExtractor(c), err -} - -func newResolver(o DockerOptions) (Resolver, error) { - if o.Socket == "" { - log.Println("warn: docker socket not configured, docker image puller disabled.") - return &fakeResolver{}, nil - } - - c, err := dockerutil.NewClient(o.Auth, o.Socket, o.CertPath) - return newDockerResolver(c), err -} - -func newLogStreamer(logsStreamer string) LogsStreamer { - if logsStreamer == "kinesis" { - return &kinesisLogsStreamer{} - } +// PullAndExtract returns a ProcfileExtractor that will pull the image using the +// docker client, then attempt to extract the Procfile from the WORKDIR, or +// fallback to the CMD directive in the Procfile. +func PullAndExtract(c *dockerutil.Client) ProcfileExtractor { + e := procfile.MultiExtractor( + procfile.NewFileExtractor(c.Client), + procfile.NewCMDExtractor(c.Client), + ) - return &nullLogsStreamer{} + return ProcfileExtractor(func(ctx context.Context, img image.Image, w io.Writer) (procfile.Procfile, error) { + if err := c.PullImage(ctx, docker.PullImageOptions{ + Registry: img.Registry, + Repository: img.Repository, + Tag: img.Tag, + OutputStream: w, + RawJSONStream: true, + }); err != nil { + return nil, err + } + + return e.Extract(img) + }) } diff --git a/empiretest/test.go b/empiretest/test.go index 853f13a5f..4cb35a10e 100644 --- a/empiretest/test.go +++ b/empiretest/test.go @@ -1,13 +1,22 @@ package empiretest import ( + "encoding/json" + "fmt" + "io" "net/http/httptest" "os" "testing" + "golang.org/x/net/context" + + "github.com/docker/docker/pkg/jsonmessage" "github.com/ejholmes/flock" - "github.com/fsouza/go-dockerclient" "github.com/remind101/empire" + "github.com/remind101/empire/pkg/image" + "github.com/remind101/empire/pkg/sslcert" + "github.com/remind101/empire/procfile" + "github.com/remind101/empire/scheduler" "github.com/remind101/empire/server" ) @@ -20,26 +29,16 @@ var ( // NewEmpire returns a new Empire instance suitable for testing. It ensures that // the database is clean before returning. func NewEmpire(t testing.TB) *empire.Empire { - opts := empire.Options{ - DB: DatabaseURL, - AWSConfig: nil, - Docker: empire.DockerOptions{ - Auth: &docker.AuthConfigurations{ - Configs: map[string]docker.AuthConfiguration{ - "https://index.docker.io/v1/": docker.AuthConfiguration{ - Username: "", - Password: "", - }, - }, - }, - }, - } - - e, err := empire.New(opts) + db, err := empire.NewDB(DatabaseURL) if err != nil { t.Fatal(err) } + e := empire.New(db, empire.DefaultOptions) + e.Scheduler = scheduler.NewFakeScheduler() + e.CertManager = sslcert.NewFakeManager() + e.ExtractProcfile = ExtractProcfile + if err := e.Reset(); err != nil { t.Fatal(err) } @@ -73,3 +72,27 @@ func Run(m *testing.M) { os.Exit(m.Run()) } + +// ExtractProcfile extracts a fake procfile. +func ExtractProcfile(ctx context.Context, img image.Image, w io.Writer) (procfile.Procfile, error) { + messages := []jsonmessage.JSONMessage{ + {Status: fmt.Sprintf("Pulling repository %s", img.Repository)}, + {Status: fmt.Sprintf("Pulling image (%s) from %s", img.Tag, img.Repository), Progress: &jsonmessage.JSONProgress{}, ID: "345c7524bc96"}, + {Status: fmt.Sprintf("Pulling image (%s) from %s, endpoint: https://registry-1.docker.io/v1/", img.Tag, img.Repository), Progress: &jsonmessage.JSONProgress{}, ID: "345c7524bc96"}, + {Status: "Pulling dependent layers", Progress: &jsonmessage.JSONProgress{}, ID: "345c7524bc96"}, + {Status: "Download complete", Progress: &jsonmessage.JSONProgress{}, ID: "a1dd7097a8e8"}, + {Status: fmt.Sprintf("Status: Image is up to date for %s", img)}, + } + + enc := json.NewEncoder(w) + + for _, m := range messages { + if err := enc.Encode(&m); err != nil { + return nil, err + } + } + + return procfile.Procfile{ + "web": "./bin/web", + }, nil +} diff --git a/extractor.go b/extractor.go deleted file mode 100644 index 7b935570b..000000000 --- a/extractor.go +++ /dev/null @@ -1,202 +0,0 @@ -package empire - -import ( - "archive/tar" - "bytes" - "fmt" - "io" - "path" - "strings" - - "github.com/fsouza/go-dockerclient" - "github.com/remind101/empire/pkg/image" - "gopkg.in/yaml.v2" -) - -var ( - // Procfile is the name of the Procfile file. - Procfile = "Procfile" -) - -// Extractor represents an object that can extract the process types from an -// image. -type Extractor interface { - // Extract takes a repo in the form `remind101/r101-api`, and an image - // id, and extracts the process types from the image. - Extract(image.Image) (CommandMap, error) -} - -// fakeExtractor is a fake implementation of the Extractor interface. -type fakeExtractor struct{} - -// Extract implements Extractor Extract. -func (e *fakeExtractor) Extract(img image.Image) (CommandMap, error) { - pm := make(CommandMap) - - // Just return some fake processes. - pm[ProcessType("web")] = Command("./bin/web") - - return pm, nil -} - -type cmdExtractor struct { - // Client is the docker client to use to pull the container image. - client *docker.Client -} - -func (e *cmdExtractor) Extract(img image.Image) (CommandMap, error) { - pm := make(CommandMap) - - i, err := e.client.InspectImage(img.String()) - if err != nil { - return pm, err - } - - pm[ProcessType("web")] = Command(strings.Join(i.Config.Cmd, " ")) - - return pm, nil -} - -// procfileFallbackExtractor attempts to extract commands using the procfileExtractor. -// If that fails because Procfile does not exist, it uses the cmdExtractor instead. -type procfileFallbackExtractor struct { - pe *procfileExtractor - ce *cmdExtractor -} - -func newProcfileFallbackExtractor(c *docker.Client) Extractor { - return &procfileFallbackExtractor{ - pe: &procfileExtractor{ - client: c, - }, - ce: &cmdExtractor{ - client: c, - }, - } -} - -func (e *procfileFallbackExtractor) Extract(img image.Image) (CommandMap, error) { - cm, err := e.pe.Extract(img) - // If err is a ProcfileError, Procfile doesn't exist. - if _, ok := err.(*ProcfileError); ok { - cm, err = e.ce.Extract(img) - } - - return cm, err -} - -// procfileExtractor is an implementation of the Extractor interface that can -// pull a docker image and extract its Procfile into a process.CommandMap. -type procfileExtractor struct { - // Client is the docker client to use to pull the container image. - client *docker.Client -} - -// Extract implements Extractor Extract. -func (e *procfileExtractor) Extract(img image.Image) (CommandMap, error) { - pm := make(CommandMap) - - c, err := e.createContainer(img) - if err != nil { - return pm, err - } - - defer e.removeContainer(c.ID) - - procfile, err := e.procfile(c.ID) - if err != nil { - return pm, err - } - - b, err := e.copyFile(c.ID, procfile) - if err != nil { - return pm, &ProcfileError{Err: err} - } - - return ParseProcfile(b) -} - -// procfile returns the path to the Procfile. If the container has a WORKDIR -// set, then this will return a path to the Procfile within that directory. -func (e *procfileExtractor) procfile(id string) (string, error) { - p := "" - - c, err := e.client.InspectContainer(id) - if err != nil { - return "", err - } - - if c.Config != nil { - p = c.Config.WorkingDir - } - - return path.Join(p, Procfile), nil -} - -// createContainer creates a new docker container for the given docker image. -func (e *procfileExtractor) createContainer(img image.Image) (*docker.Container, error) { - return e.client.CreateContainer(docker.CreateContainerOptions{ - Config: &docker.Config{ - Image: img.String(), - }, - }) -} - -// removeContainer removes a container by its ID. -func (e *procfileExtractor) removeContainer(containerID string) error { - return e.client.RemoveContainer(docker.RemoveContainerOptions{ - ID: containerID, - }) -} - -// copyFile copies a file from a container. -func (e *procfileExtractor) copyFile(containerID, path string) ([]byte, error) { - var buf bytes.Buffer - if err := e.client.CopyFromContainer(docker.CopyFromContainerOptions{ - Container: containerID, - Resource: path, - OutputStream: &buf, - }); err != nil { - return nil, err - } - - // Open the tar archive for reading. - r := bytes.NewReader(buf.Bytes()) - - return firstFile(tar.NewReader(r)) -} - -// Example instance: Procfile doesn't exist -type ProcfileError struct { - Err error -} - -func (e *ProcfileError) Error() string { - return fmt.Sprintf("Procfile not found: %s", e.Err) -} - -// ParseProcfile takes a byte slice representing a YAML Procfile and parses it -// into a processes.CommandMap. -func ParseProcfile(b []byte) (CommandMap, error) { - pm := make(CommandMap) - - if err := yaml.Unmarshal(b, &pm); err != nil { - return pm, err - } - - return pm, nil -} - -// firstFile extracts the first file from a tar archive. -func firstFile(tr *tar.Reader) ([]byte, error) { - if _, err := tr.Next(); err != nil { - return nil, err - } - - var buf bytes.Buffer - if _, err := io.Copy(&buf, tr); err != nil { - return nil, err - } - - return buf.Bytes(), nil -} diff --git a/logs.go b/logs.go index 9e9e91777..98149b2c6 100644 --- a/logs.go +++ b/logs.go @@ -10,6 +10,8 @@ type LogsStreamer interface { StreamLogs(*App, io.Writer) error } +var logsDisabled = &nullLogsStreamer{} + type nullLogsStreamer struct{} func (s *nullLogsStreamer) StreamLogs(app *App, w io.Writer) error { @@ -17,9 +19,13 @@ func (s *nullLogsStreamer) StreamLogs(app *App, w io.Writer) error { return nil } -type kinesisLogsStreamer struct{} +type KinesisLogsStreamer struct{} + +func NewKinesisLogsStreamer() *KinesisLogsStreamer { + return &KinesisLogsStreamer{} +} -func (s *kinesisLogsStreamer) StreamLogs(app *App, w io.Writer) error { +func (s *KinesisLogsStreamer) StreamLogs(app *App, w io.Writer) error { k, err := kinesumer.NewDefault(app.ID) if err != nil { return err diff --git a/processes.go b/processes.go index 3e8fca39e..ec4effc0c 100644 --- a/processes.go +++ b/processes.go @@ -11,6 +11,7 @@ import ( "github.com/lib/pq/hstore" . "github.com/remind101/empire/pkg/bytesize" "github.com/remind101/empire/pkg/constraints" + "github.com/remind101/empire/procfile" "github.com/remind101/empire/scheduler" "golang.org/x/net/context" ) @@ -101,6 +102,14 @@ func NewProcess(t ProcessType, cmd Command) *Process { // CommandMap maps a process ProcessType to a Command. type CommandMap map[ProcessType]Command +func commandMapFromProcfile(p procfile.Procfile) CommandMap { + cm := make(CommandMap) + for n, c := range p { + cm[ProcessType(n)] = Command(c) + } + return cm +} + // Scan implements the sql.Scanner interface. func (cm *CommandMap) Scan(src interface{}) error { h := hstore.Hstore{} @@ -300,13 +309,13 @@ type ProcessState struct { } type processStatesService struct { - scheduler scheduler.Scheduler + *Empire } func (s *processStatesService) JobStatesByApp(ctx context.Context, app *App) ([]*ProcessState, error) { var states []*ProcessState - instances, err := s.scheduler.Instances(ctx, app.ID) + instances, err := s.Scheduler.Instances(ctx, app.ID) if err != nil { return states, err } diff --git a/procfile/extractor.go b/procfile/extractor.go new file mode 100644 index 000000000..8e86d8164 --- /dev/null +++ b/procfile/extractor.go @@ -0,0 +1,191 @@ +package procfile + +import ( + "archive/tar" + "bytes" + "errors" + "fmt" + "io" + "path" + "strings" + + "github.com/remind101/empire/pkg/image" + + "github.com/fsouza/go-dockerclient" +) + +var ( + // ProcfileName is the name of the Procfile file. + ProcfileName = "Procfile" +) + +// Extract represents something that can extract a Procfile from an image. +type Extractor interface { + Extract(image.Image) (Procfile, error) +} + +type ExtractorFunc func(image.Image) (Procfile, error) + +func (fn ExtractorFunc) Extract(image image.Image) (Procfile, error) { + return fn(image) +} + +// CommandExtractor is an Extractor implementation that returns a Procfile based +// on the CMD directive in the Dockerfile. It makes the assumption that the cmd +// is a "web" process. +type CMDExtractor struct { + // Client is the docker client to use to pull the container image. + client *docker.Client +} + +func NewCMDExtractor(c *docker.Client) *CMDExtractor { + return &CMDExtractor{client: c} +} + +func (e *CMDExtractor) Extract(img image.Image) (Procfile, error) { + pm := make(Procfile) + + i, err := e.client.InspectImage(img.String()) + if err != nil { + return pm, err + } + + pm["web"] = strings.Join(i.Config.Cmd, " ") + + return pm, nil +} + +// MultiExtractor is an Extractor implementation that tries multiple Extractors +// in succession until one succeeds. +func MultiExtractor(extractors ...Extractor) Extractor { + return ExtractorFunc(func(image image.Image) (Procfile, error) { + for _, extractor := range extractors { + p, err := extractor.Extract(image) + + // Yay! + if err == nil { + return p, nil + } + + // Try the next one + if _, ok := err.(*ProcfileError); ok { + continue + } + + // Bubble up the error + return p, err + } + + return nil, &ProcfileError{ + Err: errors.New("no suitable Procfile extractor found"), + } + }) +} + +// FileExtractor is an implementation of the Extractor interface that extracts +// the Procfile from the images WORKDIR. +type FileExtractor struct { + // Client is the docker client to use to pull the container image. + client *docker.Client +} + +func NewFileExtractor(c *docker.Client) *FileExtractor { + return &FileExtractor{client: c} +} + +// Extract implements Extractor Extract. +func (e *FileExtractor) Extract(img image.Image) (Procfile, error) { + pm := make(Procfile) + + c, err := e.createContainer(img) + if err != nil { + return pm, err + } + + defer e.removeContainer(c.ID) + + procfile, err := e.procfile(c.ID) + if err != nil { + return pm, err + } + + b, err := e.copyFile(c.ID, procfile) + if err != nil { + return pm, &ProcfileError{Err: err} + } + + return ParseProcfile(b) +} + +// procfile returns the path to the Procfile. If the container has a WORKDIR +// set, then this will return a path to the Procfile within that directory. +func (e *FileExtractor) procfile(id string) (string, error) { + p := "" + + c, err := e.client.InspectContainer(id) + if err != nil { + return "", err + } + + if c.Config != nil { + p = c.Config.WorkingDir + } + + return path.Join(p, ProcfileName), nil +} + +// createContainer creates a new docker container for the given docker image. +func (e *FileExtractor) createContainer(img image.Image) (*docker.Container, error) { + return e.client.CreateContainer(docker.CreateContainerOptions{ + Config: &docker.Config{ + Image: img.String(), + }, + }) +} + +// removeContainer removes a container by its ID. +func (e *FileExtractor) removeContainer(containerID string) error { + return e.client.RemoveContainer(docker.RemoveContainerOptions{ + ID: containerID, + }) +} + +// copyFile copies a file from a container. +func (e *FileExtractor) copyFile(containerID, path string) ([]byte, error) { + var buf bytes.Buffer + if err := e.client.CopyFromContainer(docker.CopyFromContainerOptions{ + Container: containerID, + Resource: path, + OutputStream: &buf, + }); err != nil { + return nil, err + } + + // Open the tar archive for reading. + r := bytes.NewReader(buf.Bytes()) + + return firstFile(tar.NewReader(r)) +} + +// Example instance: Procfile doesn't exist +type ProcfileError struct { + Err error +} + +func (e *ProcfileError) Error() string { + return fmt.Sprintf("Procfile not found: %s", e.Err) +} + +// firstFile extracts the first file from a tar archive. +func firstFile(tr *tar.Reader) ([]byte, error) { + if _, err := tr.Next(); err != nil { + return nil, err + } + + var buf bytes.Buffer + if _, err := io.Copy(&buf, tr); err != nil { + return nil, err + } + + return buf.Bytes(), nil +} diff --git a/extractor_test.go b/procfile/extractor_test.go similarity index 82% rename from extractor_test.go rename to procfile/extractor_test.go index 893d3e1c4..346129e8e 100644 --- a/extractor_test.go +++ b/procfile/extractor_test.go @@ -1,4 +1,4 @@ -package empire +package procfile import ( "archive/tar" @@ -13,24 +13,7 @@ import ( "github.com/remind101/empire/pkg/image" ) -func TestFakeExtractor(t *testing.T) { - e := fakeExtractor{} - - got, err := e.Extract(image.Image{}) - if err != nil { - t.Fatal(err) - } - - want := CommandMap{ - ProcessType("web"): Command("./bin/web"), - } - - if !reflect.DeepEqual(got, want) { - t.Errorf("Extract() => %q; want %q", got, want) - } -} - -func TestCmdExtractor(t *testing.T) { +func TestCMDExtractor(t *testing.T) { api := httpmock.NewServeReplay(t).Add(httpmock.PathHandler(t, "GET /images/remind101:acme-inc/json", 200, `{ "Config": { "Cmd": ["/go/bin/app","server"] } }`, @@ -39,7 +22,7 @@ func TestCmdExtractor(t *testing.T) { c, s := newTestDockerClient(t, api) defer s.Close() - e := cmdExtractor{ + e := CMDExtractor{ client: c, } @@ -51,8 +34,8 @@ func TestCmdExtractor(t *testing.T) { t.Fatal(err) } - want := CommandMap{ - ProcessType("web"): Command("/go/bin/app server"), + want := Procfile{ + "web": "/go/bin/app server", } if !reflect.DeepEqual(got, want) { @@ -78,7 +61,7 @@ func TestProcfileExtractor(t *testing.T) { c, s := newTestDockerClient(t, api) defer s.Close() - e := procfileExtractor{ + e := FileExtractor{ client: c, } @@ -90,8 +73,8 @@ func TestProcfileExtractor(t *testing.T) { t.Fatal(err) } - want := CommandMap{ - ProcessType("web"): Command("rails server"), + want := Procfile{ + "web": "rails server", } if !reflect.DeepEqual(got, want) { @@ -121,7 +104,10 @@ func TestProcfileFallbackExtractor(t *testing.T) { c, s := newTestDockerClient(t, api) defer s.Close() - e := newProcfileFallbackExtractor(c) + e := MultiExtractor( + NewFileExtractor(c), + NewCMDExtractor(c), + ) got, err := e.Extract(image.Image{ Tag: "acme-inc", @@ -131,8 +117,8 @@ func TestProcfileFallbackExtractor(t *testing.T) { t.Fatal(err) } - want := CommandMap{ - ProcessType("web"): Command("/go/bin/app server"), + want := Procfile{ + "web": "/go/bin/app server", } if !reflect.DeepEqual(got, want) { diff --git a/procfile/procfile.go b/procfile/procfile.go new file mode 100644 index 000000000..28230889c --- /dev/null +++ b/procfile/procfile.go @@ -0,0 +1,21 @@ +package procfile + +import "gopkg.in/yaml.v2" + +// Procfile is a Go representation of a Procfile, which maps a named process to +// a command to run. +// +// TODO: This would be better represented as a map[string][]string. +type Procfile map[string]string + +// ParseProcfile takes a byte slice representing a YAML Procfile and parses it +// into a Procfile. +func ParseProcfile(b []byte) (Procfile, error) { + p := make(Procfile) + + if err := yaml.Unmarshal(b, &p); err != nil { + return p, err + } + + return p, nil +} diff --git a/releases.go b/releases.go index 40250bc2e..91e79ac77 100644 --- a/releases.go +++ b/releases.go @@ -140,8 +140,7 @@ func (s *store) attachPorts(r *Release) error { // releasesService is a service for creating and rolling back a Release. type releasesService struct { - store *store - releaser *releaser + *Empire } // ReleasesCreate creates the release, then sets the current process formation on the release. @@ -243,15 +242,14 @@ func releasesCreate(db *gorm.DB, release *Release) (*Release, error) { } type releaser struct { - store *store - scheduler scheduler.Scheduler + *Empire } // ScheduleRelease creates jobs for every process and instance count and // schedules them onto the cluster. func (r *releaser) Release(ctx context.Context, release *Release) error { a := newServiceApp(release) - return r.scheduler.Submit(ctx, a) + return r.Scheduler.Submit(ctx, a) } // ReleaseApp will find the last release for an app and release it. diff --git a/resolver.go b/resolver.go deleted file mode 100644 index f519c56b6..000000000 --- a/resolver.go +++ /dev/null @@ -1,90 +0,0 @@ -package empire - -import ( - "encoding/json" - "fmt" - "io" - - "golang.org/x/net/context" - - "github.com/docker/docker/pkg/jsonmessage" - "github.com/fsouza/go-dockerclient" - "github.com/remind101/empire/pkg/dockerutil" - "github.com/remind101/empire/pkg/image" -) - -type Resolver interface { - Resolve(context.Context, image.Image, io.Writer) (image.Image, error) -} - -// fakeResolver is a fake resolver that will just return the provided image. -type fakeResolver struct{} - -func (r *fakeResolver) Resolve(_ context.Context, img image.Image, out io.Writer) (image.Image, error) { - err := FakeDockerPull(img, out) - return img, err -} - -// dockerResolver is a resolver that pulls the docker image, then inspects it to -// get the canonical image id. -type dockerResolver struct { - client *dockerutil.Client -} - -func newDockerResolver(c *dockerutil.Client) Resolver { - return &dockerResolver{ - client: c, - } -} - -func (r *dockerResolver) Resolve(ctx context.Context, img image.Image, out io.Writer) (image.Image, error) { - if err := r.pullImage(ctx, img, out); err != nil { - return img, err - } - - i, err := r.client.InspectImage(img.String()) - if err != nil { - return img, err - } - - return image.Image{ - Repository: img.Repository, - Tag: i.ID, - }, nil -} - -// pullImage can pull a docker image from a repo, by its imageID. -// -// Because docker does not support pulling an image by ID, we're assuming that -// the docker image has been tagged with its own ID beforehand. -func (r *dockerResolver) pullImage(ctx context.Context, img image.Image, output io.Writer) error { - return r.client.PullImage(ctx, docker.PullImageOptions{ - Registry: img.Registry, - Repository: img.Repository, - Tag: img.Tag, - OutputStream: output, - RawJSONStream: true, - }) -} - -// FakeDockerPull returns a slice of events that look like a docker pull. -func FakeDockerPull(img image.Image, w io.Writer) error { - messages := []jsonmessage.JSONMessage{ - {Status: fmt.Sprintf("Pulling repository %s", img.Repository)}, - {Status: fmt.Sprintf("Pulling image (%s) from %s", img.Tag, img.Repository), Progress: &jsonmessage.JSONProgress{}, ID: "345c7524bc96"}, - {Status: fmt.Sprintf("Pulling image (%s) from %s, endpoint: https://registry-1.docker.io/v1/", img.Tag, img.Repository), Progress: &jsonmessage.JSONProgress{}, ID: "345c7524bc96"}, - {Status: "Pulling dependent layers", Progress: &jsonmessage.JSONProgress{}, ID: "345c7524bc96"}, - {Status: "Download complete", Progress: &jsonmessage.JSONProgress{}, ID: "a1dd7097a8e8"}, - {Status: fmt.Sprintf("Status: Image is up to date for %s", img)}, - } - - enc := json.NewEncoder(w) - - for _, m := range messages { - if err := enc.Encode(&m); err != nil { - return err - } - } - - return nil -} diff --git a/runner.go b/runner.go index 53494c2c9..12605e4b3 100644 --- a/runner.go +++ b/runner.go @@ -3,8 +3,6 @@ package empire import ( "io" - "github.com/remind101/empire/scheduler" - "golang.org/x/net/context" ) @@ -22,8 +20,7 @@ type ProcessRunOpts struct { } type runnerService struct { - store *store - scheduler scheduler.Scheduler + *Empire } func (r *runnerService) Run(ctx context.Context, app *App, opts ProcessRunOpts) error { @@ -39,5 +36,5 @@ func (r *runnerService) Run(ctx context.Context, app *App, opts ProcessRunOpts) p.Env[k] = v } - return r.scheduler.Run(ctx, a, p, opts.Input, opts.Output) + return r.Scheduler.Run(ctx, a, p, opts.Input, opts.Output) } diff --git a/slugs.go b/slugs.go index f9f770a60..e66948bfe 100644 --- a/slugs.go +++ b/slugs.go @@ -27,26 +27,19 @@ func slugsCreate(db *gorm.DB, slug *Slug) (*Slug, error) { // slugsService provides convenience methods for creating slugs. type slugsService struct { - store *store - extractor Extractor - resolver Resolver + *Empire } // SlugsCreateByImage creates a Slug for the given image. func (s *slugsService) SlugsCreateByImage(ctx context.Context, img image.Image, out io.Writer) (*Slug, error) { - return slugsCreateByImage(ctx, s.store, s.extractor, s.resolver, img, out) + return slugsCreateByImage(ctx, s.store, s.ExtractProcfile, img, out) } // SlugsCreateByImage first attempts to find a matching slug for the image. If // it's not found, it will fallback to extracting the process types using the // provided extractor, then create a slug. -func slugsCreateByImage(ctx context.Context, store *store, e Extractor, r Resolver, img image.Image, out io.Writer) (*Slug, error) { - _, err := r.Resolve(ctx, img, out) - if err != nil { - return nil, err - } - - slug, err := slugsExtract(e, img) +func slugsCreateByImage(ctx context.Context, store *store, e ProcfileExtractor, img image.Image, out io.Writer) (*Slug, error) { + slug, err := slugsExtract(ctx, e, img, out) if err != nil { return slug, err } @@ -56,17 +49,17 @@ func slugsCreateByImage(ctx context.Context, store *store, e Extractor, r Resolv // SlugsExtract extracts the process types from the image, then returns a new // Slug instance. -func slugsExtract(e Extractor, img image.Image) (*Slug, error) { +func slugsExtract(ctx context.Context, extract ProcfileExtractor, img image.Image, out io.Writer) (*Slug, error) { slug := &Slug{ Image: img, } - pt, err := e.Extract(img) + p, err := extract(ctx, img, out) if err != nil { return slug, err } - slug.ProcessTypes = pt + slug.ProcessTypes = commandMapFromProcfile(p) return slug, nil } diff --git a/ssl.go b/ssl.go index eb767e99d..78ed626b5 100644 --- a/ssl.go +++ b/ssl.go @@ -4,7 +4,6 @@ import ( "time" "github.com/jinzhu/gorm" - "github.com/remind101/empire/pkg/sslcert" "github.com/remind101/pkg/timex" "golang.org/x/net/context" ) @@ -37,13 +36,11 @@ func (c *Certificate) BeforeUpdate() error { } type certificatesService struct { - store *store - manager sslcert.Manager - releaser *releaser + *Empire } func (s *certificatesService) CertificatesCreate(ctx context.Context, cert *Certificate) (*Certificate, error) { - id, err := s.manager.Add(certName(cert), cert.CertificateChain, cert.PrivateKey) + id, err := s.CertManager.Add(certName(cert), cert.CertificateChain, cert.PrivateKey) if err != nil { return cert, err } @@ -53,10 +50,10 @@ func (s *certificatesService) CertificatesCreate(ctx context.Context, cert *Cert } func (s *certificatesService) CertificatesUpdate(ctx context.Context, cert *Certificate) (*Certificate, error) { - if err := s.manager.Remove(certName(cert)); err != nil { + if err := s.CertManager.Remove(certName(cert)); err != nil { return cert, err } - id, err := s.manager.Add(certName(cert), cert.CertificateChain, cert.PrivateKey) + id, err := s.CertManager.Add(certName(cert), cert.CertificateChain, cert.PrivateKey) if err != nil { return cert, err } @@ -66,7 +63,7 @@ func (s *certificatesService) CertificatesUpdate(ctx context.Context, cert *Cert } func (s *certificatesService) CertificatesDestroy(ctx context.Context, cert *Certificate) error { - if err := s.manager.Remove(certName(cert)); err != nil { + if err := s.CertManager.Remove(certName(cert)); err != nil { return err } return s.store.CertificatesDestroy(cert) From 83c0c51e652fbd873fb1c6d5ed5008f1f18f1cef Mon Sep 17 00:00:00 2001 From: "Eric J. Holmes" Date: Sun, 13 Dec 2015 17:27:15 +0700 Subject: [PATCH 2/3] Support unofficial docker registries. --- empire.go | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/empire.go b/empire.go index be578b4a3..ef71d4039 100644 --- a/empire.go +++ b/empire.go @@ -1,6 +1,7 @@ package empire // import "github.com/remind101/empire" import ( + "fmt" "io" "io/ioutil" @@ -298,9 +299,14 @@ func PullAndExtract(c *dockerutil.Client) ProcfileExtractor { ) return ProcfileExtractor(func(ctx context.Context, img image.Image, w io.Writer) (procfile.Procfile, error) { + repo := img.Repository + if img.Registry != "" { + repo = fmt.Sprintf("%s/%s", img.Registry, img.Repository) + } + if err := c.PullImage(ctx, docker.PullImageOptions{ Registry: img.Registry, - Repository: img.Repository, + Repository: repo, Tag: img.Tag, OutputStream: w, RawJSONStream: true, From b7370df8d5f93242229192639fc058dd28560b54 Mon Sep 17 00:00:00 2001 From: "Eric J. Holmes" Date: Sun, 13 Dec 2015 17:30:06 +0700 Subject: [PATCH 3/3] Update CHANGELOG. --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index c6719489c..c3f455da5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ **Features** * `emp ps` now shows the correct uptime of the process thanks to ECS support [#683](https://github.com/remind101/empire/pull/683). +* You can now deploy images from unofficial Docker registries, such as Quay.io [#692](https://github.com/remind101/empire/pull/692). ## 0.9.2 (2015-10-27)