diff --git a/Gopkg.lock b/Gopkg.lock index a1aa86afc9..02bde4018b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -128,7 +128,7 @@ "github.com/pkg/errors", "github.com/sdboyer/constext", "golang.org/x/sync/errgroup", - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v2", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/cmd/dep/ensure.go b/cmd/dep/ensure.go index 0024482b5a..93d1a2b469 100644 --- a/cmd/dep/ensure.go +++ b/cmd/dep/ensure.go @@ -211,11 +211,6 @@ func (cmd *ensureCommand) Run(ctx *dep.Ctx, args []string) error { statchan <- status }(filepath.Join(p.AbsRoot, "vendor"), lps) - params.RootPackageTree, err = p.ParseRootPackageTree() - if err != nil { - return err - } - if fatal, err := checkErrors(params.RootPackageTree.Packages, p.Manifest.IgnoredPackages()); err != nil { if fatal { return err @@ -283,20 +278,32 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project return err } - lock := p.Lock + lock := p.ChangedLock if lock != nil { - lsat := verify.LockSatisfiesInputs(p.Lock, p.Lock.SolveMeta.InputImports, p.Manifest, params.RootPackageTree) + lsat := verify.LockSatisfiesInputs(p.Lock, p.Manifest, params.RootPackageTree) if !lsat.Passed() { - // TODO(sdboyer) print out what bits are unsatisfied here + if ctx.Verbose { + ctx.Out.Println("Gopkg.lock is out of sync with Gopkg.toml and project code:") + for _, missing := range lsat.MissingImports() { + ctx.Out.Printf("\t%s is missing from input-imports\n", missing) + } + for _, excess := range lsat.ExcessImports() { + ctx.Out.Printf("\t%s is in input-imports, but isn't imported\n", excess) + } + for pr, unmatched := range lsat.UnmatchedOverrides() { + ctx.Out.Printf("\t%s is at %s, which is not allowed by override %s\n", pr, unmatched.V, unmatched.C) + } + for pr, unmatched := range lsat.UnmatchedConstraints() { + ctx.Out.Printf("\t%s is at %s, which is not allowed by constraint %s\n", pr, unmatched.V, unmatched.C) + } + ctx.Out.Println() + } + solver, err := gps.Prepare(params, sm) if err != nil { return errors.Wrap(err, "prepare solver") } - if cmd.noVendor && cmd.dryRun { - return errors.New("Gopkg.lock was not up to date") - } - solution, err := solver.Solve(context.TODO()) if err != nil { return handleAllTheFailuresOfTheWorld(err) @@ -306,23 +313,22 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project // The user said not to touch vendor/, so definitely nothing to do. return nil } - } - sw, err := dep.NewDeltaWriter(p.Lock, lock, <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor")) + dw, err := dep.NewDeltaWriter(p.Lock, lock, <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor")) if err != nil { return err } if cmd.dryRun { - return sw.PrintPreparedActions(ctx.Out, ctx.Verbose) + return dw.PrintPreparedActions(ctx.Out, ctx.Verbose) } var logger *log.Logger if ctx.Verbose { logger = ctx.Err } - return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor") + return errors.WithMessage(dw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor") } func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error { @@ -333,9 +339,10 @@ func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Proj if p.Lock == nil { return errors.Errorf("no %s exists from which to populate vendor/", dep.LockName) } + // Pass the same lock as old and new so that the writer will observe no // difference and choose not to write it out. - sw, err := dep.NewSafeWriter(nil, p.Lock, p.Lock, dep.VendorAlways, p.Manifest.PruneOptions) + sw, err := dep.NewSafeWriter(nil, p.Lock, p.ChangedLock, dep.VendorAlways, p.Manifest.PruneOptions) if err != nil { return err } @@ -383,19 +390,19 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, return handleAllTheFailuresOfTheWorld(err) } - sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), cmd.vendorBehavior(), p.Manifest.PruneOptions) + dw, err := dep.NewDeltaWriter(p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor")) if err != nil { return err } if cmd.dryRun { - return sw.PrintPreparedActions(ctx.Out, ctx.Verbose) + return dw.PrintPreparedActions(ctx.Out, ctx.Verbose) } var logger *log.Logger if ctx.Verbose { logger = ctx.Err } - return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor") + return errors.Wrap(dw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor") } func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters, statchan chan map[string]verify.VendorStatus) error { @@ -417,16 +424,6 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm rm, _ := params.RootPackageTree.ToReachMap(true, true, false, p.Manifest.IgnoredPackages()) - // TODO(sdboyer) re-enable this once we ToReachMap() intelligently filters out normally-excluded (_*, .*), dirs from errmap - //rm, errmap := params.RootPackageTree.ToReachMap(true, true, false, p.Manifest.IgnoredPackages()) - // Having some problematic internal packages isn't cause for termination, - // but the user needs to be warned. - //for fail, err := range errmap { - //if _, is := err.Err.(*build.NoGoError); !is { - //ctx.Err.Printf("Warning: %s, %s", fail, err) - //} - //} - // Compile unique sets of 1) all external packages imported or required, and // 2) the project roots under which they fall. exmap := make(map[string]bool) @@ -673,20 +670,20 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm } sort.Strings(reqlist) - sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), dep.VendorOnChanged, p.Manifest.PruneOptions) + dw, err := dep.NewDeltaWriter(p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor")) if err != nil { return err } if cmd.dryRun { - return sw.PrintPreparedActions(ctx.Out, ctx.Verbose) + return dw.PrintPreparedActions(ctx.Out, ctx.Verbose) } var logger *log.Logger if ctx.Verbose { logger = ctx.Err } - if err := errors.Wrap(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor"); err != nil { + if err := errors.Wrap(dw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor"); err != nil { return err } diff --git a/cmd/dep/init.go b/cmd/dep/init.go index 1570e2acb6..5bdff0b345 100644 --- a/cmd/dep/init.go +++ b/cmd/dep/init.go @@ -102,12 +102,12 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error { ctx.Out.Println("Getting direct dependencies...") } - ptree, directDeps, err := p.GetDirectDependencyNames(sm) + directDeps, err := p.GetDirectDependencyNames(sm) if err != nil { return errors.Wrap(err, "init failed: unable to determine direct dependencies") } if ctx.Verbose { - ctx.Out.Printf("Checked %d directories for packages.\nFound %d direct dependencies.\n", len(ptree.Packages), len(directDeps)) + ctx.Out.Printf("Checked %d directories for packages.\nFound %d direct dependencies.\n", len(p.RootPackageTree.Packages), len(directDeps)) } // Initialize with imported data, then fill in the gaps using the GOPATH @@ -133,7 +133,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error { params := gps.SolveParameters{ RootDir: root, - RootPackageTree: ptree, + RootPackageTree: p.RootPackageTree, Manifest: p.Manifest, Lock: p.Lock, ProjectAnalyzer: rootAnalyzer, diff --git a/cmd/dep/status.go b/cmd/dep/status.go index fb998f990c..ec023a4a7c 100644 --- a/cmd/dep/status.go +++ b/cmd/dep/status.go @@ -250,13 +250,13 @@ type dotOutput struct { func (out *dotOutput) BasicHeader() error { out.g = new(graphviz).New() - ptree, err := out.p.ParseRootPackageTree() + ptree := out.p.RootPackageTree // TODO(sdboyer) should be true, true, false, out.p.Manifest.IgnoredPackages() prm, _ := ptree.ToReachMap(true, false, false, nil) out.g.createNode(string(out.p.ImportRoot), "", prm.FlattenFn(paths.IsStandardImportPath)) - return err + return nil } func (out *dotOutput) BasicFooter() error { @@ -491,10 +491,7 @@ func (os OldStatus) marshalJSON() *rawOldStatus { func (cmd *statusCommand) runOld(ctx *dep.Ctx, out oldOutputter, p *dep.Project, sm gps.SourceManager) error { // While the network churns on ListVersions() requests, statically analyze // code from the current project. - ptree, err := p.ParseRootPackageTree() - if err != nil { - return err - } + ptree := p.RootPackageTree // Set up a solver in order to check the InputHash. params := gps.SolveParameters{ @@ -662,10 +659,7 @@ type MissingStatus struct { func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceManager) (hasMissingPkgs bool, errCount int, err error) { // While the network churns on ListVersions() requests, statically analyze // code from the current project. - ptree, err := p.ParseRootPackageTree() - if err != nil { - return false, 0, err - } + ptree := p.RootPackageTree // Set up a solver in order to check the InputHash. params := gps.SolveParameters{ @@ -702,7 +696,7 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana return slp[i].Ident().Less(slp[j].Ident()) }) - lsat := verify.LockSatisfiesInputs(p.Lock, p.Lock.SolveMeta.InputImports, p.Manifest, params.RootPackageTree) + lsat := verify.LockSatisfiesInputs(p.Lock, p.Manifest, params.RootPackageTree) if lsat.Passed() { // If these are equal, we're guaranteed that the lock is a transitively // complete picture of all deps. That eliminates the need for at least @@ -997,7 +991,7 @@ func collectConstraints(ctx *dep.Ctx, p *dep.Project, sm gps.SourceManager) (con // Collect the complete set of direct project dependencies, incorporating // requireds and ignores appropriately. - _, directDeps, err := p.GetDirectDependencyNames(sm) + directDeps, err := p.GetDirectDependencyNames(sm) if err != nil { // Return empty collection, not nil, if we fail here. return constraintCollection, []error{errors.Wrap(err, "failed to get direct dependencies")} diff --git a/context.go b/context.go index 9dc33dc30f..c238125d77 100644 --- a/context.go +++ b/context.go @@ -9,9 +9,13 @@ import ( "os" "path/filepath" "runtime" + "sort" "time" "github.com/golang/dep/gps" + "github.com/golang/dep/gps/paths" + "github.com/golang/dep/gps/pkgtree" + "github.com/golang/dep/gps/verify" "github.com/golang/dep/internal/fs" "github.com/pkg/errors" ) @@ -181,9 +185,58 @@ func (c *Ctx) LoadProject() (*Project, error) { return nil, errors.Wrapf(err, "error while parsing %s", lp) } + // Parse in the root package tree. + ptree, err := p.parseRootPackageTree() + if err != nil { + return nil, err + } + + // If there's a current Lock, apply the input and pruneopt changes that we + // can know without solving. + if p.Lock != nil { + p.ChangedLock = p.Lock.dup() + p.ChangedLock.SolveMeta.InputImports = externalImportList(ptree, p.Manifest) + + for k, lp := range p.ChangedLock.Projects() { + vp := lp.(verify.VerifiableProject) + vp.PruneOpts = p.Manifest.PruneOptions.PruneOptionsFor(lp.Ident().ProjectRoot) + p.ChangedLock.P[k] = vp + } + } + return p, nil } +func externalImportList(rpt pkgtree.PackageTree, m gps.RootManifest) []string { + if m == nil { + m = &Manifest{} + } + rm, _ := rpt.ToReachMap(true, true, false, m.IgnoredPackages()) + reach := rm.FlattenFn(paths.IsStandardImportPath) + req := m.RequiredPackages() + + // If there are any requires, slide them into the reach list, as well. + if len(req) > 0 { + // Make a map of imports that are both in the import path list and the + // required list to avoid duplication. + skip := make(map[string]bool, len(req)) + for _, r := range reach { + if req[r] { + skip[r] = true + } + } + + for r := range req { + if !skip[r] { + reach = append(reach, r) + } + } + } + + sort.Strings(reach) + return reach +} + // DetectProjectGOPATH attempt to find the GOPATH containing the project. // // If p.AbsRoot is not a symlink and is within a GOPATH, the GOPATH containing p.AbsRoot is returned. diff --git a/gps/verify/digest.go b/gps/verify/digest.go index 9f0042289a..0191c95057 100644 --- a/gps/verify/digest.go +++ b/gps/verify/digest.go @@ -461,7 +461,9 @@ func VerifyDepTree(osDirname string, wantDigests map[string]VersionedDigest) (ma if expectedSum, ok := wantDigests[slashPathname]; ok { ls := EmptyDigestInLock if expectedSum.HashVersion != HashVersion { - ls = HashVersionMismatch + if !expectedSum.IsEmpty() { + ls = HashVersionMismatch + } } else if len(expectedSum.Digest) > 0 { projectSum, err := DigestFromDirectory(osPathname) if err != nil { diff --git a/gps/verify/lock.go b/gps/verify/lock.go index 3997bf9d86..1e9bf3d3e2 100644 --- a/gps/verify/lock.go +++ b/gps/verify/lock.go @@ -20,28 +20,24 @@ type VerifiableProject struct { Digest VersionedDigest } -type lockUnsatisfy uint8 - -const ( - missingFromLock lockUnsatisfy = iota - inAdditionToLock -) - -type constraintMismatch struct { - c gps.Constraint - v gps.Version +// ConstraintMismatch is a two-tuple of a gps.Version, and a gps.Constraint that +// does not allow that version. +type ConstraintMismatch struct { + C gps.Constraint + V gps.Version } -type constraintMismatches map[gps.ProjectRoot]constraintMismatch - +// LockSatisfaction holds the compound result of LockSatisfiesInputs, allowing +// the caller to inspect each of several orthogonal possible types of failure. type LockSatisfaction struct { nolock bool missingPkgs, excessPkgs []string - badovr, badconstraint constraintMismatches + badovr, badconstraint map[gps.ProjectRoot]ConstraintMismatch } -// Passed is a shortcut method to check if any problems with the evaluted lock -// were identified. +// Passed is a shortcut method that indicates whether there were any ways in +// which the Lock did not satisfy the inputs. It will return true only if no +// problems were found. func (ls LockSatisfaction) Passed() bool { if ls.nolock { return false @@ -66,19 +62,27 @@ func (ls LockSatisfaction) Passed() bool { return true } -func (ls LockSatisfaction) MissingPackages() []string { +// MissingImports reports the set of import paths that were present in the +// inputs but missing in the Lock. +func (ls LockSatisfaction) MissingImports() []string { return ls.missingPkgs } -func (ls LockSatisfaction) ExcessPackages() []string { +// ExcessImports reports the set of import paths that were present in the Lock +// but absent from the inputs. +func (ls LockSatisfaction) ExcessImports() []string { return ls.excessPkgs } -func (ls LockSatisfaction) UnmatchedOverrides() map[gps.ProjectRoot]constraintMismatch { +// UnmatchedOverrides reports any override rules that were not satisfied by the +// corresponding LockedProject in the Lock. +func (ls LockSatisfaction) UnmatchedOverrides() map[gps.ProjectRoot]ConstraintMismatch { return ls.badovr } -func (ls LockSatisfaction) UnmatchedConstraints() map[gps.ProjectRoot]constraintMismatch { +// UnmatchedOverrides reports any normal, non-override constraint rules that +// were not satisfied by the corresponding LockedProject in the Lock. +func (ls LockSatisfaction) UnmatchedConstraints() map[gps.ProjectRoot]ConstraintMismatch { return ls.badconstraint } @@ -87,6 +91,8 @@ func findEffectualConstraints(m gps.Manifest, imports map[string]bool) map[strin xt := radix.New() for pr, _ := range m.DependencyConstraints() { + // FIXME(sdboyer) this has the trailing slash ambiguity problem; adapt + // code from the solver xt.Insert(string(pr), nil) } @@ -107,7 +113,7 @@ func findEffectualConstraints(m gps.Manifest, imports map[string]bool) map[strin // compute package imports that may have been removed. Figuring out that // negative space would require exploring the entire graph to ensure there are // no in-edges for particular imports. -func LockSatisfiesInputs(l gps.Lock, oldimports []string, m gps.RootManifest, rpt pkgtree.PackageTree) LockSatisfaction { +func LockSatisfiesInputs(l gps.LockWithImports, m gps.RootManifest, rpt pkgtree.PackageTree) LockSatisfaction { if l == nil { return LockSatisfaction{nolock: true} } @@ -122,8 +128,15 @@ func LockSatisfiesInputs(l gps.Lock, oldimports []string, m gps.RootManifest, rp rm, _ := rpt.ToReachMap(true, true, false, ig) reach := rm.FlattenFn(paths.IsStandardImportPath) - inlock := make(map[string]bool, len(oldimports)) + inlock := make(map[string]bool, len(l.InputImports())) ininputs := make(map[string]bool, len(reach)+len(req)) + + type lockUnsatisfy uint8 + const ( + missingFromLock lockUnsatisfy = iota + inAdditionToLock + ) + pkgDiff := make(map[string]lockUnsatisfy) for _, imp := range reach { @@ -134,13 +147,13 @@ func LockSatisfiesInputs(l gps.Lock, oldimports []string, m gps.RootManifest, rp ininputs[imp] = true } - for _, imp := range oldimports { + for _, imp := range l.InputImports() { inlock[imp] = true } lsat := LockSatisfaction{ - badovr: make(constraintMismatches), - badconstraint: make(constraintMismatches), + badovr: make(map[gps.ProjectRoot]ConstraintMismatch), + badconstraint: make(map[gps.ProjectRoot]ConstraintMismatch), } for ip := range ininputs { @@ -152,6 +165,12 @@ func LockSatisfiesInputs(l gps.Lock, oldimports []string, m gps.RootManifest, rp } } + // Something in the missing list might already be in the packages list, + // because another package in the depgraph imports it. We could make a + // special case for that, but it would break the simplicity of the model and + // complicate the notion of LockSatisfaction.Passed(), so let's see if we + // can get away without it. + for ip := range inlock { if !ininputs[ip] { pkgDiff[ip] = inAdditionToLock @@ -167,23 +186,27 @@ func LockSatisfiesInputs(l gps.Lock, oldimports []string, m gps.RootManifest, rp } eff := findEffectualConstraints(m, ininputs) - ovr := m.Overrides() - constraints := m.DependencyConstraints() + ovr, constraints := m.Overrides(), m.DependencyConstraints() for _, lp := range l.Projects() { pr := lp.Ident().ProjectRoot - if pp, has := ovr[pr]; has && !pp.Constraint.Matches(lp.Version()) { - lsat.badovr[pr] = constraintMismatch{ - c: pp.Constraint, - v: lp.Version(), + if pp, has := ovr[pr]; has { + if !pp.Constraint.Matches(lp.Version()) { + lsat.badovr[pr] = ConstraintMismatch{ + C: pp.Constraint, + V: lp.Version(), + } } + // The constraint isn't considered if we have an override, + // independent of whether the override is satisfied. + continue } if pp, has := constraints[pr]; has && eff[string(pr)] && !pp.Constraint.Matches(lp.Version()) { - lsat.badconstraint[pr] = constraintMismatch{ - c: pp.Constraint, - v: lp.Version(), + lsat.badconstraint[pr] = ConstraintMismatch{ + C: pp.Constraint, + V: lp.Version(), } } } diff --git a/gps/verify/lockdiff.go b/gps/verify/lockdiff.go index 3a742dc90d..d8cb087af0 100644 --- a/gps/verify/lockdiff.go +++ b/gps/verify/lockdiff.go @@ -211,7 +211,7 @@ func DiffProjects2(lp1, lp2 gps.LockedProject) LockedProjectPartsDelta { SourceAfter: lp2.Ident().Source, } - ld.PackagesRemoved, ld.PackagesAdded = findAddedAndRemoved(lp1.Packages(), lp2.Packages()) + ld.PackagesAdded, ld.PackagesRemoved = findAddedAndRemoved(lp1.Packages(), lp2.Packages()) switch v := lp1.Version().(type) { case gps.PairedVersion: diff --git a/lock.go b/lock.go index fce0643b6d..a30a13d531 100644 --- a/lock.go +++ b/lock.go @@ -154,6 +154,19 @@ func (l *Lock) HasProjectWithRoot(root gps.ProjectRoot) bool { return false } +func (l *Lock) dup() *Lock { + l2 := &Lock{ + SolveMeta: l.SolveMeta, + P: make([]gps.LockedProject, len(l.P)), + } + + l2.SolveMeta.InputImports = make([]string, len(l.SolveMeta.InputImports)) + copy(l2.SolveMeta.InputImports, l.SolveMeta.InputImports) + copy(l2.P, l.P) + + return l2 +} + // toRaw converts the manifest into a representation suitable to write to the lock file func (l *Lock) toRaw() rawLock { raw := rawLock{ diff --git a/project.go b/project.go index 0247ae9b40..5de3ce53cf 100644 --- a/project.go +++ b/project.go @@ -11,7 +11,6 @@ import ( "sort" "github.com/golang/dep/gps" - "github.com/golang/dep/gps/paths" "github.com/golang/dep/gps/pkgtree" "github.com/golang/dep/internal/fs" "github.com/pkg/errors" @@ -101,13 +100,19 @@ type Project struct { // If AbsRoot is not a symlink, then ResolvedAbsRoot should equal AbsRoot. ResolvedAbsRoot string // ImportRoot is the import path of the project's root directory. - ImportRoot gps.ProjectRoot - Manifest *Manifest - Lock *Lock // Optional + ImportRoot gps.ProjectRoot + // The Manifest, as read from Gopkg.toml on disk. + Manifest *Manifest + // The Lock, as read from Gopkg.lock on disk. + Lock *Lock // Optional + // The above Lock, with changes applied to it. There are two possible classes of + // changes: + // 1. Changes to InputImports + // 2. Changes to per-project prune options + ChangedLock *Lock + // The PackageTree representing the project, with hidden and ignored + // packages already trimmed. RootPackageTree pkgtree.PackageTree - // If populated, contains the results of comparing the Lock against the - // current vendor tree, per verify.VerifyDepTree(). - //VendorStatus map[string]verify.VendorStatus } // SetRoot sets the project AbsRoot and ResolvedAbsRoot. If root is not a symlink, ResolvedAbsRoot will be set to root. @@ -127,25 +132,28 @@ func (p *Project) MakeParams() gps.SolveParameters { params := gps.SolveParameters{ RootDir: p.AbsRoot, ProjectAnalyzer: Analyzer{}, + RootPackageTree: p.RootPackageTree, } if p.Manifest != nil { params.Manifest = p.Manifest } - if p.Lock != nil { - params.Lock = p.Lock + // It should be impossible for p.ChangedLock to be nil if p.Lock is non-nil; + // we always want to use the former for solving. + if p.ChangedLock != nil { + params.Lock = p.ChangedLock } return params } -// ParseRootPackageTree analyzes the root project's disk contents to create a +// parseRootPackageTree analyzes the root project's disk contents to create a // PackageTree, trimming out packages that are not relevant for root projects // along the way. // // The resulting tree is cached internally at p.RootPackageTree. -func (p *Project) ParseRootPackageTree() (pkgtree.PackageTree, error) { +func (p *Project) parseRootPackageTree() (pkgtree.PackageTree, error) { if p.RootPackageTree.Packages == nil { ptree, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot)) if err != nil { @@ -177,49 +185,28 @@ func (p *Project) ParseRootPackageTree() (pkgtree.PackageTree, error) { // This function will correctly utilize ignores and requireds from an existing // manifest, if one is present, but will also do the right thing without a // manifest. -func (p *Project) GetDirectDependencyNames(sm gps.SourceManager) (pkgtree.PackageTree, map[gps.ProjectRoot]bool, error) { - ptree, err := p.ParseRootPackageTree() - if err != nil { - return pkgtree.PackageTree{}, nil, err - } - - var ig *pkgtree.IgnoredRuleset - var req map[string]bool - if p.Manifest != nil { - ig = p.Manifest.IgnoredPackages() - req = p.Manifest.RequiredPackages() - } - - rm, _ := ptree.ToReachMap(true, true, false, ig) - reach := rm.FlattenFn(paths.IsStandardImportPath) - - if len(req) > 0 { - // Make a map of imports that are both in the import path list and the - // required list to avoid duplication. - skip := make(map[string]bool, len(req)) - for _, r := range reach { - if req[r] { - skip[r] = true - } - } - - for r := range req { - if !skip[r] { - reach = append(reach, r) - } +func (p *Project) GetDirectDependencyNames(sm gps.SourceManager) (map[gps.ProjectRoot]bool, error) { + var reach []string + if p.ChangedLock != nil { + reach = p.ChangedLock.InputImports() + } else { + ptree, err := p.parseRootPackageTree() + if err != nil { + return nil, err } + reach = externalImportList(ptree, p.Manifest) } directDeps := map[gps.ProjectRoot]bool{} for _, ip := range reach { pr, err := sm.DeduceProjectRoot(ip) if err != nil { - return pkgtree.PackageTree{}, nil, err + return nil, err } directDeps[pr] = true } - return ptree, directDeps, nil + return directDeps, nil } // FindIneffectualConstraints looks for constraint rules expressed in the @@ -233,7 +220,7 @@ func (p *Project) FindIneffectualConstraints(sm gps.SourceManager) []gps.Project return nil } - _, dd, err := p.GetDirectDependencyNames(sm) + dd, err := p.GetDirectDependencyNames(sm) if err != nil { return nil } diff --git a/txn_writer.go b/txn_writer.go index d15d670150..d16b145e2e 100644 --- a/txn_writer.go +++ b/txn_writer.go @@ -5,6 +5,7 @@ package dep import ( + "bytes" "context" "fmt" "io/ioutil" @@ -466,6 +467,7 @@ const ( solveChanged hashMismatch hashVersionMismatch + hashAbsent missingFromTree projectAdded projectRemoved @@ -518,10 +520,12 @@ func NewDeltaWriter(oldLock, newLock *Lock, status map[string]verify.VendorStatu switch stat { case verify.NotInTree: sw.changed[pr] = missingFromTree - case verify.EmptyDigestInLock, verify.DigestMismatchInLock: + case verify.DigestMismatchInLock: sw.changed[pr] = hashMismatch case verify.HashVersionMismatch: sw.changed[pr] = hashVersionMismatch + case verify.EmptyDigestInLock: + sw.changed[pr] = hashAbsent } } } @@ -564,29 +568,51 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l dropped := []gps.ProjectRoot{} // TODO(sdboyer) add a txn/rollback layer, like the safewriter? + i := 0 + tot := len(dw.changed) for pr, reason := range dw.changed { + if reason == projectRemoved { + dropped = append(dropped, pr) + continue + } + to := filepath.FromSlash(filepath.Join(vnewpath, string(pr))) po := dw.pruneOptions.PruneOptionsFor(pr) + if err := sm.ExportPrunedProject(context.TODO(), projs[pr], po, to); err != nil { + return errors.Wrapf(err, "failed to export %s", pr) + } + + i++ lpd := dw.lockDiff.ProjectDeltas[pr] + v, id := projs[pr].Version(), projs[pr].Ident() + var buf bytes.Buffer + fmt.Fprintf(&buf, "(%d/%d) Wrote %s@%s: ", i, tot, id, v) switch reason { case noChange: panic(fmt.Sprintf("wtf, no change for %s", pr)) case solveChanged: if lpd.SourceChanged() { - logger.Printf("Writing %s: source changed (%s -> %s)", pr, lpd.SourceBefore, lpd.SourceAfter) + fmt.Fprintf(&buf, "source changed (%s -> %s)", lpd.SourceBefore, lpd.SourceAfter) } else if lpd.VersionChanged() { - logger.Printf("Writing %s: version changed (%s -> %s)", pr, lpd.VersionBefore, lpd.VersionAfter) + bv, av := "(none)", "(none)" + if lpd.VersionBefore != nil { + bv = lpd.VersionBefore.String() + } + if lpd.VersionAfter != nil { + av = lpd.VersionAfter.String() + } + fmt.Fprintf(&buf, "version changed (%s -> %s)", bv, av) } else if lpd.RevisionChanged() { - logger.Printf("Writing %s: revision changed (%s -> %s)", pr, lpd.RevisionBefore, lpd.RevisionAfter) + fmt.Fprintf(&buf, "revision changed (%s -> %s)", lpd.RevisionBefore, lpd.RevisionAfter) } else if lpd.PackagesChanged() { la, lr := len(lpd.PackagesAdded), len(lpd.PackagesRemoved) if la > 0 && lr > 0 { - logger.Printf("Writing %s: packages changed (%v added, %v removed)", pr, la, lr) + fmt.Fprintf(&buf, "packages changed (%v added, %v removed)", la, lr) } else if la > 0 { - logger.Printf("Writing %s: packages changed (%v added)", pr, la) + fmt.Fprintf(&buf, "packages changed (%v added)", la) } else { - logger.Printf("Writing %s: packages changed (%v removed)", pr, lr) + fmt.Fprintf(&buf, "packages changed (%v removed)", lr) } } else if lpd.PruneOptsChanged() { // Override what's on the lockdiff with the extra info we have; @@ -594,23 +620,20 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l // value from the input param in place. old := lpd.PruneOptsBefore & ^gps.PruneNestedVendorDirs new := lpd.PruneOptsAfter & ^gps.PruneNestedVendorDirs - logger.Printf("Writing %s: prune options changed (%s -> %s)", pr, old, new) + fmt.Fprintf(&buf, "prune options changed (%s -> %s)", old, new) } case hashMismatch: - logger.Printf("Writing %s: hash mismatch between Gopkg.lock and vendor contents", pr) + fmt.Fprintf(&buf, "hash mismatch between Gopkg.lock and vendor contents") case hashVersionMismatch: - logger.Printf("Writing %s: hashing algorithm mismatch", pr) + fmt.Fprintf(&buf, "hashing algorithm mismatch") + case hashAbsent: + fmt.Fprintf(&buf, "hash digest absent from lock") case projectAdded: - logger.Printf("Writing new project %s", pr) - case projectRemoved: - dropped = append(dropped, pr) - continue + fmt.Fprintf(&buf, "new project") case missingFromTree: - logger.Printf("Writing %s: missing from vendor", pr) - } - if err := sm.ExportPrunedProject(context.TODO(), projs[pr], po, to); err != nil { - return errors.Wrapf(err, "failed to export %s", pr) + fmt.Fprintf(&buf, "missing from vendor", pr) } + logger.Print(buf.String()) digest, err := verify.DigestFromDirectory(to) if err != nil { @@ -649,9 +672,17 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l } } - for _, pr := range dropped { - // Kind of a lie to print this here. ¯\_(ツ)_/¯ - logger.Printf("Discarding unused project %s", pr) + for i, pr := range dropped { + // Kind of a lie to print this. ¯\_(ツ)_/¯ + logger.Printf("(%d/%d) Removed unused project %s", tot-(len(dropped)-i-1), tot, pr) + } + + // Ensure vendor/.git is preserved if present + if hasDotGit(vpath) { + err = fs.RenameWithFallback(filepath.Join(vpath, ".git"), filepath.Join(vnewpath, "vendor/.git")) + if _, ok := err.(*os.LinkError); ok { + return errors.Wrap(err, "failed to preserve vendor/.git") + } } err = os.RemoveAll(vpath)