diff --git a/cmd/dep/gb_importer.go b/cmd/dep/gb_importer.go new file mode 100644 index 0000000000..60c626d8c3 --- /dev/null +++ b/cmd/dep/gb_importer.go @@ -0,0 +1,205 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "encoding/json" + "io/ioutil" + "log" + "os" + "path/filepath" + + "github.com/golang/dep" + fb "github.com/golang/dep/internal/feedback" + "github.com/golang/dep/internal/gps" + "github.com/pkg/errors" +) + +// gbImporter imports gb configuration into the dep configuration format. +type gbImporter struct { + manifest gbManifest + logger *log.Logger + verbose bool + sm gps.SourceManager +} + +func newGbImporter(logger *log.Logger, verbose bool, sm gps.SourceManager) *gbImporter { + return &gbImporter{ + logger: logger, + verbose: verbose, + sm: sm, + } +} + +// gbManifest represents the manifest file for GB projects +type gbManifest struct { + Dependencies []gbDependency `json:"dependencies"` +} + +type gbDependency struct { + Importpath string `json:"importpath"` + Repository string `json:"repository"` + + // All gb vendored dependencies have a specific revision + Revision string `json:"revision"` + + // Branch may be HEAD or an actual branch. In the case of HEAD, that means + // the user vendored a dependency by specifying a tag or a specific revision + // which results in a detached HEAD + Branch string `json:"branch"` +} + +func (i *gbImporter) Name() string { + return "gb" +} + +func (i *gbImporter) HasDepMetadata(dir string) bool { + // gb stores the manifest in the vendor tree + var m = filepath.Join(dir, "vendor", "manifest") + if _, err := os.Stat(m); err != nil { + return false + } + + return true +} + +func (i *gbImporter) Import(dir string, pr gps.ProjectRoot) (*dep.Manifest, *dep.Lock, error) { + err := i.load(dir) + if err != nil { + return nil, nil, err + } + + return i.convert(pr) +} + +// load the gb manifest +func (i *gbImporter) load(projectDir string) error { + i.logger.Println("Detected gb manifest file...") + var mf = filepath.Join(projectDir, "vendor", "manifest") + if i.verbose { + i.logger.Printf(" Loading %s", mf) + } + + var buf []byte + var err error + if buf, err = ioutil.ReadFile(mf); err != nil { + return errors.Wrapf(err, "Unable to read %s", mf) + } + if err := json.Unmarshal(buf, &i.manifest); err != nil { + return errors.Wrapf(err, "Unable to parse %s", mf) + } + + return nil +} + +// convert the gb manifest into dep configuration files. +func (i *gbImporter) convert(pr gps.ProjectRoot) (*dep.Manifest, *dep.Lock, error) { + i.logger.Println("Converting from gb manifest...") + + manifest := &dep.Manifest{ + Constraints: make(gps.ProjectConstraints), + } + + lock := &dep.Lock{} + + for _, pkg := range i.manifest.Dependencies { + if pkg.Importpath == "" { + return nil, nil, errors.New("Invalid gb configuration, package import path is required") + } + + if pkg.Revision == "" { + return nil, nil, errors.New("Invalid gb configuration, package revision is required") + } + + // Deduce the project root. This is necessary because gb manifests can have + // multiple entries for the same project root, one for each imported subpackage + var root gps.ProjectRoot + var err error + if root, err = i.sm.DeduceProjectRoot(pkg.Importpath); err != nil { + return nil, nil, err + } + + // Set the proper import path back on the dependency + pkg.Importpath = string(root) + + // If we've already locked this project root then we can skip + if projectExistsInLock(lock, pkg.Importpath) { + continue + } + + // Otherwise, attempt to convert this specific package, which returns a constraint and a lock + pc, lp, err := i.convertOne(pkg) + if err != nil { + return nil, nil, err + } + + manifest.Constraints[pc.Ident.ProjectRoot] = gps.ProjectProperties{Source: pc.Ident.Source, Constraint: pc.Constraint} + lock.P = append(lock.P, lp) + + } + + return manifest, lock, nil +} + +func (i *gbImporter) convertOne(pkg gbDependency) (pc gps.ProjectConstraint, lp gps.LockedProject, err error) { + /* + gb's vendor plugin (gb vendor), which manages the vendor tree and manifest + file, supports fetching by a specific tag or revision, but if you specify + either of those it's a detached checkout and the "branch" field is HEAD. + The only time the "branch" field is not "HEAD" is if you do not specify a + tag or revision, otherwise it's either "master" or the value of the -branch + flag + + This means that, generally, the only possible "constraint" we can really specify is + the branch name if the branch name is not HEAD. Otherwise, it's a specific revision. + + However, if we can infer a tag that points to the revision or the branch, we may be able + to use that as the constraint + + So, the order of operations to convert a single dependency in a gb manifest is: + - Find a specific version for the revision (and branch, if set) + - If there's a branch available, use that as the constraint + - If there's no branch, but we found a version from step 1, use the version as the constraint + */ + pc.Ident = gps.ProjectIdentifier{ProjectRoot: gps.ProjectRoot(pkg.Importpath), Source: pkg.Repository} + + // Generally, gb tracks revisions + var revision = gps.Revision(pkg.Revision) + + // But if the branch field is not "HEAD", we can use that as the initial constraint + var constraint gps.Constraint + if pkg.Branch != "" && pkg.Branch != "HEAD" { + constraint = gps.NewBranch(pkg.Branch) + } + + // See if we can get a version from that constraint + version, err := lookupVersionForLockedProject(pc.Ident, constraint, revision, i.sm) + if err != nil { + // Log the error, but don't fail it. It's okay if we can't find a version + i.logger.Println(err.Error()) + } + + // If the constraint is nil (no branch), but there's a version, infer a constraint from there + if constraint == nil && version != nil { + constraint, err = i.sm.InferConstraint(version.String(), pc.Ident) + if err != nil { + return + } + } + + // If there's *still* no constraint, set the constraint to the revision + if constraint == nil { + constraint = revision + } + + pc.Constraint = constraint + + lp = gps.NewLockedProject(pc.Ident, version, nil) + + fb.NewConstraintFeedback(pc, fb.DepTypeImported).LogFeedback(i.logger) + fb.NewLockedProjectFeedback(lp, fb.DepTypeImported).LogFeedback(i.logger) + + return +} diff --git a/cmd/dep/gb_importer_test.go b/cmd/dep/gb_importer_test.go new file mode 100644 index 0000000000..5caa7d2067 --- /dev/null +++ b/cmd/dep/gb_importer_test.go @@ -0,0 +1,201 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + "bytes" + "log" + "path/filepath" + "testing" + + "github.com/golang/dep/internal/gps" + "github.com/golang/dep/internal/test" + "github.com/pkg/errors" +) + +const testGbProjectRoot = "github.com/golang/notexist" + +func TestGbConfig_ImportNoVendor(t *testing.T) { + h := test.NewHelper(t) + defer h.Cleanup() + + ctx := newTestContext(h) + sm, err := ctx.SourceManager() + h.Must(err) + defer sm.Release() + + h.TempDir(filepath.Join("src", testGbProjectRoot, "vendor")) + h.TempCopy(filepath.Join(testGbProjectRoot, "vendor", "_not-a-manifest"), "gb/manifest") + projectRoot := h.Path(testGbProjectRoot) + + // Capture stderr so we can verify output + verboseOutput := &bytes.Buffer{} + ctx.Err = log.New(verboseOutput, "", 0) + + g := newGbImporter(ctx.Err, false, sm) // Disable verbose so that we don't print values that change each test run + if g.HasDepMetadata(projectRoot) { + t.Fatal("Expected the importer to return false if there's no vendor manifest") + } +} + +func TestGbConfig_Import(t *testing.T) { + h := test.NewHelper(t) + defer h.Cleanup() + + ctx := newTestContext(h) + sm, err := ctx.SourceManager() + h.Must(err) + defer sm.Release() + + h.TempDir(filepath.Join("src", testGbProjectRoot, "vendor")) + h.TempCopy(filepath.Join(testGbProjectRoot, "vendor", "manifest"), "gb/manifest") + projectRoot := h.Path(testGbProjectRoot) + + // Capture stderr so we can verify output + verboseOutput := &bytes.Buffer{} + ctx.Err = log.New(verboseOutput, "", 0) + + g := newGbImporter(ctx.Err, false, sm) // Disable verbose so that we don't print values that change each test run + if g.Name() != "gb" { + t.Fatal("Expected the importer to return the name 'gb'") + } + if !g.HasDepMetadata(projectRoot) { + t.Fatal("Expected the importer to detect the gb manifest file") + } + + m, l, err := g.Import(projectRoot, testGbProjectRoot) + h.Must(err) + + if m == nil { + t.Fatal("Expected the manifest to be generated") + } + + if l == nil { + t.Fatal("Expected the lock to be generated") + } + + goldenFile := "gb/golden.txt" + got := verboseOutput.String() + want := h.GetTestFileString(goldenFile) + if want != got { + if *test.UpdateGolden { + if err := h.WriteTestFile(goldenFile, got); err != nil { + t.Fatalf("%+v", errors.Wrapf(err, "Unable to write updated golden file %s", goldenFile)) + } + } else { + t.Fatalf("expected %s, got %s", want, got) + } + } +} + +func TestGbConfig_Convert_Project(t *testing.T) { + h := test.NewHelper(t) + defer h.Cleanup() + + ctx := newTestContext(h) + sm, err := ctx.SourceManager() + h.Must(err) + defer sm.Release() + + pkg := "github.com/sdboyer/deptest" + repo := "https://github.com/sdboyer/deptest.git" + + g := newGbImporter(ctx.Err, true, sm) + g.manifest = gbManifest{ + Dependencies: []gbDependency{ + { + Importpath: pkg, + Repository: repo, + Revision: "ff2948a2ac8f538c4ecd55962e919d1e13e74baf", + }, + }, + } + + manifest, lock, err := g.convert(testGbProjectRoot) + if err != nil { + t.Fatal(err) + } + + d, ok := manifest.Constraints[gps.ProjectRoot(pkg)] + if !ok { + t.Fatal("Expected the manifest to have a dependency for 'github.com/sdboyer/deptest' but got none") + } + + wantC := "^1.0.0" + gotC := d.Constraint.String() + if gotC != wantC { + t.Fatalf("Expected manifest constraint to be %s, got %s", wantC, gotC) + } + + gotS := d.Source + if gotS != repo { + t.Fatalf("Expected manifest source to be %s, got %s", repo, gotS) + } + + wantP := 1 + gotP := len(lock.P) + if gotP != 1 { + t.Fatalf("Expected the lock to contain %d project but got %d", wantP, gotP) + } + + p := lock.P[0] + gotPr := string(p.Ident().ProjectRoot) + if gotPr != pkg { + t.Fatalf("Expected the lock to have a project for %s but got '%s'", pkg, gotPr) + } + + gotS = p.Ident().Source + if gotS != repo { + t.Fatalf("Expected locked source to be %s, got '%s'", repo, gotS) + } + + lv := p.Version() + lpv, ok := lv.(gps.PairedVersion) + if !ok { + t.Fatalf("Expected locked version to be a PairedVersion but got %T", lv) + } + + wantRev := "ff2948a2ac8f538c4ecd55962e919d1e13e74baf" + gotRev := lpv.Revision().String() + if gotRev != wantRev { + t.Fatalf("Expected locked revision to be %s, got %s", wantRev, gotRev) + } + + wantV := "v1.0.0" + gotV := lpv.String() + if gotV != wantV { + t.Fatalf("Expected locked version to be %s, got %s", wantV, gotV) + } +} + +func TestGbConfig_Convert_BadInput(t *testing.T) { + h := test.NewHelper(t) + defer h.Cleanup() + + ctx := newTestContext(h) + sm, err := ctx.SourceManager() + h.Must(err) + defer sm.Release() + + g := newGbImporter(ctx.Err, true, sm) + g.manifest = gbManifest{ + Dependencies: []gbDependency{{Importpath: ""}}, + } + + _, _, err = g.convert(testGbProjectRoot) + if err == nil { + t.Fatal("Expected conversion to fail because the package name is empty") + } + + g = newGbImporter(ctx.Err, true, sm) + g.manifest = gbManifest{ + Dependencies: []gbDependency{{Importpath: "github.com/sdboyer/deptest"}}, + } + + _, _, err = g.convert(testGbProjectRoot) + if err == nil { + t.Fatal("Expected conversion to fail because the package has no revision") + } +} diff --git a/cmd/dep/godep_importer.go b/cmd/dep/godep_importer.go index 3dc052787b..14217606fa 100644 --- a/cmd/dep/godep_importer.go +++ b/cmd/dep/godep_importer.go @@ -187,15 +187,3 @@ func (g *godepImporter) buildLockedProject(pkg godepPackage, manifest *dep.Manif return lp } - -// projectExistsInLock checks if the given import path already existing in -// locked projects. -func projectExistsInLock(l *dep.Lock, ip string) bool { - for _, lp := range l.P { - if ip == string(lp.Ident().ProjectRoot) { - return true - } - } - - return false -} diff --git a/cmd/dep/root_analyzer.go b/cmd/dep/root_analyzer.go index 9f93b7eb22..fb7653a2f2 100644 --- a/cmd/dep/root_analyzer.go +++ b/cmd/dep/root_analyzer.go @@ -73,6 +73,7 @@ func (a *rootAnalyzer) importManifestAndLock(dir string, pr gps.ProjectRoot, sup importers := []importer{ newGlideImporter(logger, a.ctx.Verbose, a.sm), newGodepImporter(logger, a.ctx.Verbose, a.sm), + newGbImporter(logger, a.ctx.Verbose, a.sm), } for _, i := range importers { @@ -211,3 +212,15 @@ func lookupVersionForLockedProject(pi gps.ProjectIdentifier, c gps.Constraint, r // Give up and lock only to a revision return rev, nil } + +// projectExistsInLock checks if the given import path already existing in +// locked projects. +func projectExistsInLock(l *dep.Lock, ip string) bool { + for _, lp := range l.P { + if ip == string(lp.Ident().ProjectRoot) { + return true + } + } + + return false +} diff --git a/cmd/dep/testdata/gb/golden.txt b/cmd/dep/testdata/gb/golden.txt new file mode 100644 index 0000000000..30ccd26f5c --- /dev/null +++ b/cmd/dep/testdata/gb/golden.txt @@ -0,0 +1,10 @@ +Detected gb manifest file... +Converting from gb manifest... + Using master as initial constraint for imported dep github.com/sdboyer/deptestdos + Trying v2.0.0 (5c60720) as initial lock for imported dep github.com/sdboyer/deptestdos + Using ^1.0.0 as initial constraint for imported dep github.com/sdboyer/deptest + Trying v1.0.0 (ff2948a) as initial lock for imported dep github.com/sdboyer/deptest + Using master as initial constraint for imported dep github.com/carolynvs/deptest + Trying v2 (4982dd1) as initial lock for imported dep github.com/carolynvs/deptest + Using master as initial constraint for imported dep github.com/carolynvs/deptest-subpkg + Trying master (b90e5f3) as initial lock for imported dep github.com/carolynvs/deptest-subpkg diff --git a/cmd/dep/testdata/gb/manifest b/cmd/dep/testdata/gb/manifest new file mode 100644 index 0000000000..2bcb7d9e6c --- /dev/null +++ b/cmd/dep/testdata/gb/manifest @@ -0,0 +1,36 @@ +{ + "version": 0, + "dependencies": [ + { + "importpath": "github.com/sdboyer/deptestdos", + "repository": "https://github.com/sdboyer/deptestdos", + "revision": "5c607206be5decd28e6263ffffdcee067266015e", + "branch": "master" + }, + { + "importpath": "github.com/sdboyer/deptest", + "repository": "https://github.com/sdboyer/deptest", + "revision": "ff2948a2ac8f538c4ecd55962e919d1e13e74baf", + "branch": "HEAD" + }, + { + "importpath": "github.com/carolynvs/deptest", + "repository": "https://github.com/carolynvs/deptest", + "revision": "4982dd1811ef8dea67c6ba1b049ca0ba217879b2", + "branch": "master" + }, + { + "importpath": "github.com/carolynvs/deptest-subpkg/subby", + "repository": "https://github.com/carolynvs/deptest-subpkg", + "revision": "b90e5f3a888585ea5df81d3fe0c81fc6e3e3d70b", + "branch": "master", + "path": "/subby" + }, + { + "importpath": "github.com/carolynvs/deptest-subpkg", + "repository": "https://github.com/carolynvs/deptest-subpkg", + "revision": "b90e5f3a888585ea5df81d3fe0c81fc6e3e3d70b", + "branch": "master" + } + ] +} diff --git a/cmd/dep/testdata/harness_tests/init/gb/case1/final/Gopkg.lock b/cmd/dep/testdata/harness_tests/init/gb/case1/final/Gopkg.lock new file mode 100644 index 0000000000..15336f3f1d --- /dev/null +++ b/cmd/dep/testdata/harness_tests/init/gb/case1/final/Gopkg.lock @@ -0,0 +1,37 @@ +# This file is autogenerated, do not edit; changes may be undone by the next 'dep ensure'. + + +[[projects]] + name = "github.com/carolynvs/deptest" + packages = ["."] + revision = "3f4c3bea144e112a69bbe5d8d01c1b09a544253f" + source = "https://github.com/carolynvs/deptest" + version = "v0.8.1" + +[[projects]] + branch = "master" + name = "github.com/carolynvs/deptest-subpkg" + packages = ["subby"] + revision = "b90e5f3a888585ea5df81d3fe0c81fc6e3e3d70b" + source = "https://github.com/carolynvs/deptest-subpkg" + +[[projects]] + name = "github.com/sdboyer/deptest" + packages = ["."] + revision = "ff2948a2ac8f538c4ecd55962e919d1e13e74baf" + source = "https://github.com/sdboyer/deptest" + version = "v1.0.0" + +[[projects]] + branch = "master" + name = "github.com/sdboyer/deptestdos" + packages = ["."] + revision = "a0196baa11ea047dd65037287451d36b861b00ea" + source = "https://github.com/sdboyer/deptestdos" + +[solve-meta] + analyzer-name = "dep" + analyzer-version = 1 + inputs-digest = "ae7f33ba976551de3e6004f4cfa44f2981e825f42d61dad5bbe80a70a644631e" + solver-name = "gps-cdcl" + solver-version = 1 diff --git a/cmd/dep/testdata/harness_tests/init/gb/case1/final/Gopkg.toml b/cmd/dep/testdata/harness_tests/init/gb/case1/final/Gopkg.toml new file mode 100644 index 0000000000..6a2b85a03d --- /dev/null +++ b/cmd/dep/testdata/harness_tests/init/gb/case1/final/Gopkg.toml @@ -0,0 +1,20 @@ + +[[constraint]] + branch = "master" + name = "github.com/carolynvs/deptest" + source = "https://github.com/carolynvs/deptest" + +[[constraint]] + branch = "master" + name = "github.com/carolynvs/deptest-subpkg" + source = "https://github.com/carolynvs/deptest-subpkg" + +[[constraint]] + name = "github.com/sdboyer/deptest" + source = "https://github.com/sdboyer/deptest" + version = "1.0.0" + +[[constraint]] + branch = "master" + name = "github.com/sdboyer/deptestdos" + source = "https://github.com/sdboyer/deptestdos" diff --git a/cmd/dep/testdata/harness_tests/init/gb/case1/initial/src/gbtest/main.go b/cmd/dep/testdata/harness_tests/init/gb/case1/initial/src/gbtest/main.go new file mode 100644 index 0000000000..f517b9ac1c --- /dev/null +++ b/cmd/dep/testdata/harness_tests/init/gb/case1/initial/src/gbtest/main.go @@ -0,0 +1,19 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package main + +import ( + cdt "github.com/carolynvs/deptest" + "github.com/carolynvs/deptest-subpkg/subby" + "github.com/sdboyer/deptest" + "github.com/sdboyer/deptestdos" +) + +func main() { + _ = deptestdos.Bar{} + _ = deptest.Foo{} + _ = cdt.Foo{} + _ = subby.SayHi() +} diff --git a/cmd/dep/testdata/harness_tests/init/gb/case1/initial/vendor/manifest b/cmd/dep/testdata/harness_tests/init/gb/case1/initial/vendor/manifest new file mode 100644 index 0000000000..2bcb7d9e6c --- /dev/null +++ b/cmd/dep/testdata/harness_tests/init/gb/case1/initial/vendor/manifest @@ -0,0 +1,36 @@ +{ + "version": 0, + "dependencies": [ + { + "importpath": "github.com/sdboyer/deptestdos", + "repository": "https://github.com/sdboyer/deptestdos", + "revision": "5c607206be5decd28e6263ffffdcee067266015e", + "branch": "master" + }, + { + "importpath": "github.com/sdboyer/deptest", + "repository": "https://github.com/sdboyer/deptest", + "revision": "ff2948a2ac8f538c4ecd55962e919d1e13e74baf", + "branch": "HEAD" + }, + { + "importpath": "github.com/carolynvs/deptest", + "repository": "https://github.com/carolynvs/deptest", + "revision": "4982dd1811ef8dea67c6ba1b049ca0ba217879b2", + "branch": "master" + }, + { + "importpath": "github.com/carolynvs/deptest-subpkg/subby", + "repository": "https://github.com/carolynvs/deptest-subpkg", + "revision": "b90e5f3a888585ea5df81d3fe0c81fc6e3e3d70b", + "branch": "master", + "path": "/subby" + }, + { + "importpath": "github.com/carolynvs/deptest-subpkg", + "repository": "https://github.com/carolynvs/deptest-subpkg", + "revision": "b90e5f3a888585ea5df81d3fe0c81fc6e3e3d70b", + "branch": "master" + } + ] +} diff --git a/cmd/dep/testdata/harness_tests/init/gb/case1/testcase.json b/cmd/dep/testdata/harness_tests/init/gb/case1/testcase.json new file mode 100644 index 0000000000..7644a6c820 --- /dev/null +++ b/cmd/dep/testdata/harness_tests/init/gb/case1/testcase.json @@ -0,0 +1,15 @@ +{ + "commands": [ + ["init", "-no-examples"] + ], + "error-expected": "", + "gopath-initial": { + "github.com/sdboyer/deptest": "3f4c3bea144e112a69bbe5d8d01c1b09a544253f" + }, + "vendor-final": [ + "github.com/carolynvs/deptest", + "github.com/carolynvs/deptest-subpkg", + "github.com/sdboyer/deptest", + "github.com/sdboyer/deptestdos" + ] +} diff --git a/docs/FAQ.md b/docs/FAQ.md index 36e12a7a1d..e7debb842d 100644 --- a/docs/FAQ.md +++ b/docs/FAQ.md @@ -193,7 +193,7 @@ about what's going on. During `dep init` configuration from other dependency managers is detected and imported, unless `-skip-tools` is specified. -The following tools are supported: `glide` and `godep`. +The following tools are supported: `glide`, `godep`, and `gb`. See [#186](https://github.com/golang/dep/issues/186#issuecomment-306363441) for how to add support for another tool.