From f5a22be0a0da69cee0c5cf97df24627675a97b7d Mon Sep 17 00:00:00 2001 From: hnnsgstfssn Date: Wed, 22 Apr 2020 00:43:53 +0100 Subject: [PATCH] Add pkger source driver support (#377) * Add pkger source driver support As go-bindata has been abandoned [1] there are open requests, #116, for alternative sources with similar functionality. The Buffalo project [2] created packr and recently pkger [3] was announced [4] with the intention to supersede packr. This change adds support for using pkger as a source. The implementation relies on httpfs.PartialDriver for pretty much all functionality. [1] https://github.com/jteeuwen/go-bindata/issues/5 [2] https://gobuffalo.io/ [3] https://github.com/markbates/pkger [4] https://blog.gobuffalo.io/introducing-pkger-static-file-embedding-in-go-1ce76dc79c65 * pkger: rename Instance to Pkger * pkger: make WithInstance accept *Pkger * pkger: refactor and add access to global pkging.Pkger instance * pkger: fix typo and cleanup debug logging --- go.mod | 2 + go.sum | 6 ++ source/pkger/pkger.go | 83 ++++++++++++++++ source/pkger/pkger_test.go | 196 +++++++++++++++++++++++++++++++++++++ 4 files changed, 287 insertions(+) create mode 100644 source/pkger/pkger.go create mode 100644 source/pkger/pkger_test.go diff --git a/go.mod b/go.mod index d6bfa2a9a..8e58c3e5e 100644 --- a/go.mod +++ b/go.mod @@ -19,6 +19,7 @@ require ( github.com/edsrzf/mmap-go v0.0.0-20170320065105-0bce6a688712 // indirect github.com/fsouza/fake-gcs-server v1.17.0 github.com/go-sql-driver/mysql v1.5.0 + github.com/gobuffalo/here v0.6.0 github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4 github.com/gogo/protobuf v1.3.1 // indirect github.com/golang/snappy v0.0.1 // indirect @@ -28,6 +29,7 @@ require ( github.com/jackc/pgconn v1.3.2 // indirect github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0 // indirect github.com/lib/pq v1.3.0 + github.com/markbates/pkger v0.15.1 github.com/mattn/go-sqlite3 v1.10.0 github.com/nakagami/firebirdsql v0.0.0-20190310045651-3c02a58cfed8 github.com/neo4j-drivers/gobolt v1.7.4 // indirect diff --git a/go.sum b/go.sum index e467c7b16..c6e812f4d 100644 --- a/go.sum +++ b/go.sum @@ -103,6 +103,8 @@ github.com/go-sql-driver/mysql v1.5.0 h1:ozyZYNQW3x3HtqT1jira07DN2PArx2v7/mN66gG github.com/go-sql-driver/mysql v1.5.0/go.mod h1:DCzpHaOWr8IXmIStZouvnhqoel9Qv2LBy8hT2VhHyBg= github.com/go-stack/stack v1.8.0 h1:5SgMzNM5HxrEjV0ww2lTmX6E2Izsfxas4+YHWRs3Lsk= github.com/go-stack/stack v1.8.0/go.mod h1:v0f6uXyyMGvRgIKkXu+yp6POWl0qKG85gN/melR3HDY= +github.com/gobuffalo/here v0.6.0 h1:hYrd0a6gDmWxBM4TnrGw8mQg24iSVoIkHEk7FodQcBI= +github.com/gobuffalo/here v0.6.0/go.mod h1:wAG085dHOYqUpf+Ap+WOdrPTp5IYcDAs/x7PLa8Y5fM= github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4 h1:vF83LI8tAakwEwvWZtrIEx7pOySacl2TOxx6eXk4ePo= github.com/gocql/gocql v0.0.0-20190301043612-f6df8288f9b4/go.mod h1:4Fw1eo5iaEhDUs8XyuhSVCVy52Jq3L+/3GJgYkwc+/0= github.com/gogo/protobuf v1.1.1/go.mod h1:r8qH/GZQm5c6nD/R0oafs1akxWv10x8SbQlK7atdtwQ= @@ -226,6 +228,8 @@ github.com/lib/pq v1.1.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.2.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= github.com/lib/pq v1.3.0 h1:/qkRGz8zljWiDcFvgpwUpwIAPu3r07TDvs3Rws+o/pU= github.com/lib/pq v1.3.0/go.mod h1:5WUZQaWbwv1U+lTReE5YruASi9Al49XbQIvNi/34Woo= +github.com/markbates/pkger v0.15.1 h1:3MPelV53RnGSW07izx5xGxl4e/sdRD6zqseIk0rMASY= +github.com/markbates/pkger v0.15.1/go.mod h1:0JoVlrol20BSywW79rN3kdFFsE5xYM+rSCQDXbLhiuI= github.com/mattn/go-colorable v0.1.1/go.mod h1:FuOcm+DKB9mbwrcAfNl7/TZVBZ6rcnceauSikq3lYCQ= github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.7/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= @@ -522,6 +526,8 @@ gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7/go.mod h1:dt/ZhP58zS4L8KSrWD gopkg.in/yaml.v2 v2.2.1/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gopkg.in/yaml.v2 v2.2.2 h1:ZCJp+EgiOT7lHqUV2J862kp8Qj64Jo6az82+3Td9dZw= gopkg.in/yaml.v2 v2.2.2/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= +gopkg.in/yaml.v2 v2.2.7 h1:VUgggvou5XRW9mHwD/yXxIYSMtY0zoKQf/v226p2nyo= +gopkg.in/yaml.v2 v2.2.7/go.mod h1:hI93XBmqTisBFMUTm0b8Fm+jr3Dg1NNxqwp+5A1VGuI= gotest.tools/v3 v3.0.2 h1:kG1BFyqVHuQoVQiR1bWGnfz/fmHvvuiSPIV7rvl360E= gotest.tools/v3 v3.0.2/go.mod h1:3SzNCllyD9/Y+b5r9JIKQ474KzkZyqLqEfYqMsX94Bk= honnef.co/go/tools v0.0.0-20180728063816-88497007e858/go.mod h1:rf3lG4BRIbNafJWhAfAdb/ePZxsR/4RtNHQocxwk9r4= diff --git a/source/pkger/pkger.go b/source/pkger/pkger.go new file mode 100644 index 000000000..f5f2132d6 --- /dev/null +++ b/source/pkger/pkger.go @@ -0,0 +1,83 @@ +package pkger + +import ( + "fmt" + "net/http" + stdurl "net/url" + + "github.com/golang-migrate/migrate/v4/source" + "github.com/golang-migrate/migrate/v4/source/httpfs" + "github.com/markbates/pkger" + "github.com/markbates/pkger/pkging" +) + +func init() { + source.Register("pkger", &Pkger{}) +} + +// Pkger is a source.Driver that reads migrations from instances of +// pkging.Pkger. +type Pkger struct { + httpfs.PartialDriver +} + +// Open implements source.Driver. The path component of url will be used as the +// relative location of migrations. The returned driver will use the package +// scoped pkger.Open to access migrations. The relative root and any +// migrations must be added to the global pkger.Pkger instance by calling +// pkger.Apply. Refer to Pkger documentation for more information. +func (p *Pkger) Open(url string) (source.Driver, error) { + u, err := stdurl.Parse(url) + if err != nil { + return nil, err + } + + // wrap pkger to implement http.FileSystem. + fs := fsFunc(func(name string) (http.File, error) { + f, err := pkger.Open(name) + if err != nil { + return nil, err + } + return f.(http.File), nil + }) + + if err := p.Init(fs, u.Path); err != nil { + return nil, fmt.Errorf("failed to init driver with relative path %q: %w", u.Path, err) + } + + return p, nil +} + +// WithInstance returns a source.Driver that is backed by an instance of +// pkging.Pkger. The relative location of migrations is indicated by path. The +// path must exist on the pkging.Pkger instance for the driver to initialize +// successfully. +func WithInstance(instance pkging.Pkger, path string) (source.Driver, error) { + if instance == nil { + return nil, fmt.Errorf("expected instance of pkging.Pkger") + } + + // wrap pkger to implement http.FileSystem. + fs := fsFunc(func(name string) (http.File, error) { + f, err := instance.Open(name) + if err != nil { + return nil, err + } + return f.(http.File), nil + }) + + var p Pkger + + if err := p.Init(fs, path); err != nil { + return nil, fmt.Errorf("failed to init driver with relative path %q: %w", path, err) + } + + return &p, nil +} + +type fsFunc func(name string) (http.File, error) + +// Open implements http.FileSystem. +func (f fsFunc) Open(name string) (http.File, error) { + return f(name) +} diff --git a/source/pkger/pkger_test.go b/source/pkger/pkger_test.go new file mode 100644 index 000000000..bb3d561b6 --- /dev/null +++ b/source/pkger/pkger_test.go @@ -0,0 +1,196 @@ +package pkger + +import ( + "errors" + "os" + "testing" + + "github.com/gobuffalo/here" + st "github.com/golang-migrate/migrate/v4/source/testing" + "github.com/markbates/pkger" + "github.com/markbates/pkger/pkging" + "github.com/markbates/pkger/pkging/mem" +) + +func Test(t *testing.T) { + t.Run("WithInstance", func(t *testing.T) { + i := testInstance(t) + + createPkgerFile(t, i, "/1_foobar.up.sql") + createPkgerFile(t, i, "/1_foobar.down.sql") + createPkgerFile(t, i, "/3_foobar.up.sql") + createPkgerFile(t, i, "/4_foobar.up.sql") + createPkgerFile(t, i, "/4_foobar.down.sql") + createPkgerFile(t, i, "/5_foobar.down.sql") + createPkgerFile(t, i, "/7_foobar.up.sql") + createPkgerFile(t, i, "/7_foobar.down.sql") + + d, err := WithInstance(i, "/") + if err != nil { + t.Fatal(err) + } + + st.Test(t, d) + }) + + t.Run("Open", func(t *testing.T) { + i := testInstance(t) + + createPkgerFile(t, i, "/1_foobar.up.sql") + createPkgerFile(t, i, "/1_foobar.down.sql") + createPkgerFile(t, i, "/3_foobar.up.sql") + createPkgerFile(t, i, "/4_foobar.up.sql") + createPkgerFile(t, i, "/4_foobar.down.sql") + createPkgerFile(t, i, "/5_foobar.down.sql") + createPkgerFile(t, i, "/7_foobar.up.sql") + createPkgerFile(t, i, "/7_foobar.down.sql") + + registerPackageLevelInstance(t, i) + + d, err := (&Pkger{}).Open("pkger:///") + if err != nil { + t.Fatal(err) + } + + st.Test(t, d) + }) + +} + +func TestWithInstance(t *testing.T) { + t.Run("Subdir", func(t *testing.T) { + i := testInstance(t) + + // Make sure the relative root exists so that httpfs.PartialDriver can + // initialize. + createPkgerSubdir(t, i, "/subdir") + + _, err := WithInstance(i, "/subdir") + if err != nil { + t.Fatal("") + } + }) + + t.Run("NilInstance", func(t *testing.T) { + _, err := WithInstance(nil, "") + if err == nil { + t.Fatal(err) + } + }) + + t.Run("FailInit", func(t *testing.T) { + i := testInstance(t) + + _, err := WithInstance(i, "/fail") + if err == nil { + t.Fatal(err) + } + }) + + t.Run("FailWithoutMigrations", func(t *testing.T) { + i := testInstance(t) + + createPkgerSubdir(t, i, "/") + + d, err := WithInstance(i, "/") + if err != nil { + t.Fatal(err) + } + + if _, err := d.First(); !errors.Is(err, os.ErrNotExist) { + t.Fatal(err) + } + + }) +} + +func TestOpen(t *testing.T) { + + t.Run("InvalidURL", func(t *testing.T) { + _, err := (&Pkger{}).Open(":///") + if err == nil { + t.Fatal(err) + } + }) + + t.Run("Root", func(t *testing.T) { + _, err := (&Pkger{}).Open("pkger:///") + if err != nil { + t.Fatal(err) + } + }) + + t.Run("FailInit", func(t *testing.T) { + _, err := (&Pkger{}).Open("pkger:///subdir") + if err == nil { + t.Fatal(err) + } + }) + + i := testInstance(t) + createPkgerSubdir(t, i, "/subdir") + + // Note that this registers the instance globally so anything run after + // this will have access to everything container in the registered + // instance. + registerPackageLevelInstance(t, i) + + t.Run("Subdir", func(t *testing.T) { + _, err := (&Pkger{}).Open("pkger:///subdir") + if err != nil { + t.Fatal(err) + } + }) +} + +func TestClose(t *testing.T) { + d, err := (&Pkger{}).Open("pkger:///") + if err != nil { + t.Fatal(err) + } + if err := d.Close(); err != nil { + t.Fatal(err) + } +} + +func registerPackageLevelInstance(t *testing.T, pkg pkging.Pkger) { + if err := pkger.Apply(pkg, nil); err != nil { + t.Fatalf("failed to register pkger instance: %v\n", err) + } +} + +func testInstance(t *testing.T) pkging.Pkger { + pkg, err := inMemoryPkger() + if err != nil { + t.Fatalf("failed to create an pkging.Pkger instance: %v\n", err) + } + + return pkg +} + +func createPkgerSubdir(t *testing.T, pkg pkging.Pkger, subdir string) { + if err := pkg.MkdirAll(subdir, os.ModePerm); err != nil { + t.Fatalf("failed to create pkger subdir %q: %v\n", subdir, err) + } +} + +func createPkgerFile(t *testing.T, pkg pkging.Pkger, name string) { + _, err := pkg.Create(name) + if err != nil { + t.Fatalf("failed to create pkger file %q: %v\n", name, err) + } +} + +func inMemoryPkger() (*mem.Pkger, error) { + info, err := here.New().Current() + if err != nil { + return nil, err + } + + pkg, err := mem.New(info) + if err != nil { + return nil, err + } + + return pkg, nil +}