From ac40bf2436340b179b7e8913c4384a98401b18ec Mon Sep 17 00:00:00 2001 From: sam boyer Date: Sun, 1 Jul 2018 15:13:22 -0400 Subject: [PATCH] dep: Tell the user why we're solving Add output to all of the information we assemble when checking if the Lock satisfies the current input set. Also some refactoring of the ctx.LoadProject() process to have fewer partial states. --- Gopkg.lock | 2 +- cmd/dep/ensure.go | 63 ++++++++++++++---------------- cmd/dep/init.go | 6 +-- cmd/dep/status.go | 18 +++------ context.go | 53 +++++++++++++++++++++++++ gps/verify/digest.go | 4 +- gps/verify/lock.go | 89 ++++++++++++++++++++++++++---------------- gps/verify/lockdiff.go | 2 +- lock.go | 13 ++++++ project.go | 75 +++++++++++++++-------------------- txn_writer.go | 73 ++++++++++++++++++++++++---------- 11 files changed, 249 insertions(+), 149 deletions(-) diff --git a/Gopkg.lock b/Gopkg.lock index a1aa86afc9..02bde4018b 100644 --- a/Gopkg.lock +++ b/Gopkg.lock @@ -128,7 +128,7 @@ "github.com/pkg/errors", "github.com/sdboyer/constext", "golang.org/x/sync/errgroup", - "gopkg.in/yaml.v2" + "gopkg.in/yaml.v2", ] solver-name = "gps-cdcl" solver-version = 1 diff --git a/cmd/dep/ensure.go b/cmd/dep/ensure.go index 0024482b5a..93d1a2b469 100644 --- a/cmd/dep/ensure.go +++ b/cmd/dep/ensure.go @@ -211,11 +211,6 @@ func (cmd *ensureCommand) Run(ctx *dep.Ctx, args []string) error { statchan <- status }(filepath.Join(p.AbsRoot, "vendor"), lps) - params.RootPackageTree, err = p.ParseRootPackageTree() - if err != nil { - return err - } - if fatal, err := checkErrors(params.RootPackageTree.Packages, p.Manifest.IgnoredPackages()); err != nil { if fatal { return err @@ -283,20 +278,32 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project return err } - lock := p.Lock + lock := p.ChangedLock if lock != nil { - lsat := verify.LockSatisfiesInputs(p.Lock, p.Lock.SolveMeta.InputImports, p.Manifest, params.RootPackageTree) + lsat := verify.LockSatisfiesInputs(p.Lock, p.Manifest, params.RootPackageTree) if !lsat.Passed() { - // TODO(sdboyer) print out what bits are unsatisfied here + if ctx.Verbose { + ctx.Out.Println("Gopkg.lock is out of sync with Gopkg.toml and project code:") + for _, missing := range lsat.MissingImports() { + ctx.Out.Printf("\t%s is missing from input-imports\n", missing) + } + for _, excess := range lsat.ExcessImports() { + ctx.Out.Printf("\t%s is in input-imports, but isn't imported\n", excess) + } + for pr, unmatched := range lsat.UnmatchedOverrides() { + ctx.Out.Printf("\t%s is at %s, which is not allowed by override %s\n", pr, unmatched.V, unmatched.C) + } + for pr, unmatched := range lsat.UnmatchedConstraints() { + ctx.Out.Printf("\t%s is at %s, which is not allowed by constraint %s\n", pr, unmatched.V, unmatched.C) + } + ctx.Out.Println() + } + solver, err := gps.Prepare(params, sm) if err != nil { return errors.Wrap(err, "prepare solver") } - if cmd.noVendor && cmd.dryRun { - return errors.New("Gopkg.lock was not up to date") - } - solution, err := solver.Solve(context.TODO()) if err != nil { return handleAllTheFailuresOfTheWorld(err) @@ -306,23 +313,22 @@ func (cmd *ensureCommand) runDefault(ctx *dep.Ctx, args []string, p *dep.Project // The user said not to touch vendor/, so definitely nothing to do. return nil } - } - sw, err := dep.NewDeltaWriter(p.Lock, lock, <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor")) + dw, err := dep.NewDeltaWriter(p.Lock, lock, <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor")) if err != nil { return err } if cmd.dryRun { - return sw.PrintPreparedActions(ctx.Out, ctx.Verbose) + return dw.PrintPreparedActions(ctx.Out, ctx.Verbose) } var logger *log.Logger if ctx.Verbose { logger = ctx.Err } - return errors.WithMessage(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor") + return errors.WithMessage(dw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor") } func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters) error { @@ -333,9 +339,10 @@ func (cmd *ensureCommand) runVendorOnly(ctx *dep.Ctx, args []string, p *dep.Proj if p.Lock == nil { return errors.Errorf("no %s exists from which to populate vendor/", dep.LockName) } + // Pass the same lock as old and new so that the writer will observe no // difference and choose not to write it out. - sw, err := dep.NewSafeWriter(nil, p.Lock, p.Lock, dep.VendorAlways, p.Manifest.PruneOptions) + sw, err := dep.NewSafeWriter(nil, p.Lock, p.ChangedLock, dep.VendorAlways, p.Manifest.PruneOptions) if err != nil { return err } @@ -383,19 +390,19 @@ func (cmd *ensureCommand) runUpdate(ctx *dep.Ctx, args []string, p *dep.Project, return handleAllTheFailuresOfTheWorld(err) } - sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), cmd.vendorBehavior(), p.Manifest.PruneOptions) + dw, err := dep.NewDeltaWriter(p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor")) if err != nil { return err } if cmd.dryRun { - return sw.PrintPreparedActions(ctx.Out, ctx.Verbose) + return dw.PrintPreparedActions(ctx.Out, ctx.Verbose) } var logger *log.Logger if ctx.Verbose { logger = ctx.Err } - return errors.Wrap(sw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor") + return errors.Wrap(dw.Write(p.AbsRoot, sm, false, logger), "grouped write of manifest, lock and vendor") } func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm gps.SourceManager, params gps.SolveParameters, statchan chan map[string]verify.VendorStatus) error { @@ -417,16 +424,6 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm rm, _ := params.RootPackageTree.ToReachMap(true, true, false, p.Manifest.IgnoredPackages()) - // TODO(sdboyer) re-enable this once we ToReachMap() intelligently filters out normally-excluded (_*, .*), dirs from errmap - //rm, errmap := params.RootPackageTree.ToReachMap(true, true, false, p.Manifest.IgnoredPackages()) - // Having some problematic internal packages isn't cause for termination, - // but the user needs to be warned. - //for fail, err := range errmap { - //if _, is := err.Err.(*build.NoGoError); !is { - //ctx.Err.Printf("Warning: %s, %s", fail, err) - //} - //} - // Compile unique sets of 1) all external packages imported or required, and // 2) the project roots under which they fall. exmap := make(map[string]bool) @@ -673,20 +670,20 @@ func (cmd *ensureCommand) runAdd(ctx *dep.Ctx, args []string, p *dep.Project, sm } sort.Strings(reqlist) - sw, err := dep.NewSafeWriter(nil, p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), dep.VendorOnChanged, p.Manifest.PruneOptions) + dw, err := dep.NewDeltaWriter(p.Lock, dep.LockFromSolution(solution, p.Manifest.PruneOptions), <-statchan, p.Manifest.PruneOptions, filepath.Join(p.AbsRoot, "vendor")) if err != nil { return err } if cmd.dryRun { - return sw.PrintPreparedActions(ctx.Out, ctx.Verbose) + return dw.PrintPreparedActions(ctx.Out, ctx.Verbose) } var logger *log.Logger if ctx.Verbose { logger = ctx.Err } - if err := errors.Wrap(sw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor"); err != nil { + if err := errors.Wrap(dw.Write(p.AbsRoot, sm, true, logger), "grouped write of manifest, lock and vendor"); err != nil { return err } diff --git a/cmd/dep/init.go b/cmd/dep/init.go index 1570e2acb6..5bdff0b345 100644 --- a/cmd/dep/init.go +++ b/cmd/dep/init.go @@ -102,12 +102,12 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error { ctx.Out.Println("Getting direct dependencies...") } - ptree, directDeps, err := p.GetDirectDependencyNames(sm) + directDeps, err := p.GetDirectDependencyNames(sm) if err != nil { return errors.Wrap(err, "init failed: unable to determine direct dependencies") } if ctx.Verbose { - ctx.Out.Printf("Checked %d directories for packages.\nFound %d direct dependencies.\n", len(ptree.Packages), len(directDeps)) + ctx.Out.Printf("Checked %d directories for packages.\nFound %d direct dependencies.\n", len(p.RootPackageTree.Packages), len(directDeps)) } // Initialize with imported data, then fill in the gaps using the GOPATH @@ -133,7 +133,7 @@ func (cmd *initCommand) Run(ctx *dep.Ctx, args []string) error { params := gps.SolveParameters{ RootDir: root, - RootPackageTree: ptree, + RootPackageTree: p.RootPackageTree, Manifest: p.Manifest, Lock: p.Lock, ProjectAnalyzer: rootAnalyzer, diff --git a/cmd/dep/status.go b/cmd/dep/status.go index fb998f990c..ec023a4a7c 100644 --- a/cmd/dep/status.go +++ b/cmd/dep/status.go @@ -250,13 +250,13 @@ type dotOutput struct { func (out *dotOutput) BasicHeader() error { out.g = new(graphviz).New() - ptree, err := out.p.ParseRootPackageTree() + ptree := out.p.RootPackageTree // TODO(sdboyer) should be true, true, false, out.p.Manifest.IgnoredPackages() prm, _ := ptree.ToReachMap(true, false, false, nil) out.g.createNode(string(out.p.ImportRoot), "", prm.FlattenFn(paths.IsStandardImportPath)) - return err + return nil } func (out *dotOutput) BasicFooter() error { @@ -491,10 +491,7 @@ func (os OldStatus) marshalJSON() *rawOldStatus { func (cmd *statusCommand) runOld(ctx *dep.Ctx, out oldOutputter, p *dep.Project, sm gps.SourceManager) error { // While the network churns on ListVersions() requests, statically analyze // code from the current project. - ptree, err := p.ParseRootPackageTree() - if err != nil { - return err - } + ptree := p.RootPackageTree // Set up a solver in order to check the InputHash. params := gps.SolveParameters{ @@ -662,10 +659,7 @@ type MissingStatus struct { func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceManager) (hasMissingPkgs bool, errCount int, err error) { // While the network churns on ListVersions() requests, statically analyze // code from the current project. - ptree, err := p.ParseRootPackageTree() - if err != nil { - return false, 0, err - } + ptree := p.RootPackageTree // Set up a solver in order to check the InputHash. params := gps.SolveParameters{ @@ -702,7 +696,7 @@ func runStatusAll(ctx *dep.Ctx, out outputter, p *dep.Project, sm gps.SourceMana return slp[i].Ident().Less(slp[j].Ident()) }) - lsat := verify.LockSatisfiesInputs(p.Lock, p.Lock.SolveMeta.InputImports, p.Manifest, params.RootPackageTree) + lsat := verify.LockSatisfiesInputs(p.Lock, p.Manifest, params.RootPackageTree) if lsat.Passed() { // If these are equal, we're guaranteed that the lock is a transitively // complete picture of all deps. That eliminates the need for at least @@ -997,7 +991,7 @@ func collectConstraints(ctx *dep.Ctx, p *dep.Project, sm gps.SourceManager) (con // Collect the complete set of direct project dependencies, incorporating // requireds and ignores appropriately. - _, directDeps, err := p.GetDirectDependencyNames(sm) + directDeps, err := p.GetDirectDependencyNames(sm) if err != nil { // Return empty collection, not nil, if we fail here. return constraintCollection, []error{errors.Wrap(err, "failed to get direct dependencies")} diff --git a/context.go b/context.go index 9dc33dc30f..c238125d77 100644 --- a/context.go +++ b/context.go @@ -9,9 +9,13 @@ import ( "os" "path/filepath" "runtime" + "sort" "time" "github.com/golang/dep/gps" + "github.com/golang/dep/gps/paths" + "github.com/golang/dep/gps/pkgtree" + "github.com/golang/dep/gps/verify" "github.com/golang/dep/internal/fs" "github.com/pkg/errors" ) @@ -181,9 +185,58 @@ func (c *Ctx) LoadProject() (*Project, error) { return nil, errors.Wrapf(err, "error while parsing %s", lp) } + // Parse in the root package tree. + ptree, err := p.parseRootPackageTree() + if err != nil { + return nil, err + } + + // If there's a current Lock, apply the input and pruneopt changes that we + // can know without solving. + if p.Lock != nil { + p.ChangedLock = p.Lock.dup() + p.ChangedLock.SolveMeta.InputImports = externalImportList(ptree, p.Manifest) + + for k, lp := range p.ChangedLock.Projects() { + vp := lp.(verify.VerifiableProject) + vp.PruneOpts = p.Manifest.PruneOptions.PruneOptionsFor(lp.Ident().ProjectRoot) + p.ChangedLock.P[k] = vp + } + } + return p, nil } +func externalImportList(rpt pkgtree.PackageTree, m gps.RootManifest) []string { + if m == nil { + m = &Manifest{} + } + rm, _ := rpt.ToReachMap(true, true, false, m.IgnoredPackages()) + reach := rm.FlattenFn(paths.IsStandardImportPath) + req := m.RequiredPackages() + + // If there are any requires, slide them into the reach list, as well. + if len(req) > 0 { + // Make a map of imports that are both in the import path list and the + // required list to avoid duplication. + skip := make(map[string]bool, len(req)) + for _, r := range reach { + if req[r] { + skip[r] = true + } + } + + for r := range req { + if !skip[r] { + reach = append(reach, r) + } + } + } + + sort.Strings(reach) + return reach +} + // DetectProjectGOPATH attempt to find the GOPATH containing the project. // // If p.AbsRoot is not a symlink and is within a GOPATH, the GOPATH containing p.AbsRoot is returned. diff --git a/gps/verify/digest.go b/gps/verify/digest.go index 9f0042289a..0191c95057 100644 --- a/gps/verify/digest.go +++ b/gps/verify/digest.go @@ -461,7 +461,9 @@ func VerifyDepTree(osDirname string, wantDigests map[string]VersionedDigest) (ma if expectedSum, ok := wantDigests[slashPathname]; ok { ls := EmptyDigestInLock if expectedSum.HashVersion != HashVersion { - ls = HashVersionMismatch + if !expectedSum.IsEmpty() { + ls = HashVersionMismatch + } } else if len(expectedSum.Digest) > 0 { projectSum, err := DigestFromDirectory(osPathname) if err != nil { diff --git a/gps/verify/lock.go b/gps/verify/lock.go index 3997bf9d86..1e9bf3d3e2 100644 --- a/gps/verify/lock.go +++ b/gps/verify/lock.go @@ -20,28 +20,24 @@ type VerifiableProject struct { Digest VersionedDigest } -type lockUnsatisfy uint8 - -const ( - missingFromLock lockUnsatisfy = iota - inAdditionToLock -) - -type constraintMismatch struct { - c gps.Constraint - v gps.Version +// ConstraintMismatch is a two-tuple of a gps.Version, and a gps.Constraint that +// does not allow that version. +type ConstraintMismatch struct { + C gps.Constraint + V gps.Version } -type constraintMismatches map[gps.ProjectRoot]constraintMismatch - +// LockSatisfaction holds the compound result of LockSatisfiesInputs, allowing +// the caller to inspect each of several orthogonal possible types of failure. type LockSatisfaction struct { nolock bool missingPkgs, excessPkgs []string - badovr, badconstraint constraintMismatches + badovr, badconstraint map[gps.ProjectRoot]ConstraintMismatch } -// Passed is a shortcut method to check if any problems with the evaluted lock -// were identified. +// Passed is a shortcut method that indicates whether there were any ways in +// which the Lock did not satisfy the inputs. It will return true only if no +// problems were found. func (ls LockSatisfaction) Passed() bool { if ls.nolock { return false @@ -66,19 +62,27 @@ func (ls LockSatisfaction) Passed() bool { return true } -func (ls LockSatisfaction) MissingPackages() []string { +// MissingImports reports the set of import paths that were present in the +// inputs but missing in the Lock. +func (ls LockSatisfaction) MissingImports() []string { return ls.missingPkgs } -func (ls LockSatisfaction) ExcessPackages() []string { +// ExcessImports reports the set of import paths that were present in the Lock +// but absent from the inputs. +func (ls LockSatisfaction) ExcessImports() []string { return ls.excessPkgs } -func (ls LockSatisfaction) UnmatchedOverrides() map[gps.ProjectRoot]constraintMismatch { +// UnmatchedOverrides reports any override rules that were not satisfied by the +// corresponding LockedProject in the Lock. +func (ls LockSatisfaction) UnmatchedOverrides() map[gps.ProjectRoot]ConstraintMismatch { return ls.badovr } -func (ls LockSatisfaction) UnmatchedConstraints() map[gps.ProjectRoot]constraintMismatch { +// UnmatchedOverrides reports any normal, non-override constraint rules that +// were not satisfied by the corresponding LockedProject in the Lock. +func (ls LockSatisfaction) UnmatchedConstraints() map[gps.ProjectRoot]ConstraintMismatch { return ls.badconstraint } @@ -87,6 +91,8 @@ func findEffectualConstraints(m gps.Manifest, imports map[string]bool) map[strin xt := radix.New() for pr, _ := range m.DependencyConstraints() { + // FIXME(sdboyer) this has the trailing slash ambiguity problem; adapt + // code from the solver xt.Insert(string(pr), nil) } @@ -107,7 +113,7 @@ func findEffectualConstraints(m gps.Manifest, imports map[string]bool) map[strin // compute package imports that may have been removed. Figuring out that // negative space would require exploring the entire graph to ensure there are // no in-edges for particular imports. -func LockSatisfiesInputs(l gps.Lock, oldimports []string, m gps.RootManifest, rpt pkgtree.PackageTree) LockSatisfaction { +func LockSatisfiesInputs(l gps.LockWithImports, m gps.RootManifest, rpt pkgtree.PackageTree) LockSatisfaction { if l == nil { return LockSatisfaction{nolock: true} } @@ -122,8 +128,15 @@ func LockSatisfiesInputs(l gps.Lock, oldimports []string, m gps.RootManifest, rp rm, _ := rpt.ToReachMap(true, true, false, ig) reach := rm.FlattenFn(paths.IsStandardImportPath) - inlock := make(map[string]bool, len(oldimports)) + inlock := make(map[string]bool, len(l.InputImports())) ininputs := make(map[string]bool, len(reach)+len(req)) + + type lockUnsatisfy uint8 + const ( + missingFromLock lockUnsatisfy = iota + inAdditionToLock + ) + pkgDiff := make(map[string]lockUnsatisfy) for _, imp := range reach { @@ -134,13 +147,13 @@ func LockSatisfiesInputs(l gps.Lock, oldimports []string, m gps.RootManifest, rp ininputs[imp] = true } - for _, imp := range oldimports { + for _, imp := range l.InputImports() { inlock[imp] = true } lsat := LockSatisfaction{ - badovr: make(constraintMismatches), - badconstraint: make(constraintMismatches), + badovr: make(map[gps.ProjectRoot]ConstraintMismatch), + badconstraint: make(map[gps.ProjectRoot]ConstraintMismatch), } for ip := range ininputs { @@ -152,6 +165,12 @@ func LockSatisfiesInputs(l gps.Lock, oldimports []string, m gps.RootManifest, rp } } + // Something in the missing list might already be in the packages list, + // because another package in the depgraph imports it. We could make a + // special case for that, but it would break the simplicity of the model and + // complicate the notion of LockSatisfaction.Passed(), so let's see if we + // can get away without it. + for ip := range inlock { if !ininputs[ip] { pkgDiff[ip] = inAdditionToLock @@ -167,23 +186,27 @@ func LockSatisfiesInputs(l gps.Lock, oldimports []string, m gps.RootManifest, rp } eff := findEffectualConstraints(m, ininputs) - ovr := m.Overrides() - constraints := m.DependencyConstraints() + ovr, constraints := m.Overrides(), m.DependencyConstraints() for _, lp := range l.Projects() { pr := lp.Ident().ProjectRoot - if pp, has := ovr[pr]; has && !pp.Constraint.Matches(lp.Version()) { - lsat.badovr[pr] = constraintMismatch{ - c: pp.Constraint, - v: lp.Version(), + if pp, has := ovr[pr]; has { + if !pp.Constraint.Matches(lp.Version()) { + lsat.badovr[pr] = ConstraintMismatch{ + C: pp.Constraint, + V: lp.Version(), + } } + // The constraint isn't considered if we have an override, + // independent of whether the override is satisfied. + continue } if pp, has := constraints[pr]; has && eff[string(pr)] && !pp.Constraint.Matches(lp.Version()) { - lsat.badconstraint[pr] = constraintMismatch{ - c: pp.Constraint, - v: lp.Version(), + lsat.badconstraint[pr] = ConstraintMismatch{ + C: pp.Constraint, + V: lp.Version(), } } } diff --git a/gps/verify/lockdiff.go b/gps/verify/lockdiff.go index 3a742dc90d..d8cb087af0 100644 --- a/gps/verify/lockdiff.go +++ b/gps/verify/lockdiff.go @@ -211,7 +211,7 @@ func DiffProjects2(lp1, lp2 gps.LockedProject) LockedProjectPartsDelta { SourceAfter: lp2.Ident().Source, } - ld.PackagesRemoved, ld.PackagesAdded = findAddedAndRemoved(lp1.Packages(), lp2.Packages()) + ld.PackagesAdded, ld.PackagesRemoved = findAddedAndRemoved(lp1.Packages(), lp2.Packages()) switch v := lp1.Version().(type) { case gps.PairedVersion: diff --git a/lock.go b/lock.go index fce0643b6d..a30a13d531 100644 --- a/lock.go +++ b/lock.go @@ -154,6 +154,19 @@ func (l *Lock) HasProjectWithRoot(root gps.ProjectRoot) bool { return false } +func (l *Lock) dup() *Lock { + l2 := &Lock{ + SolveMeta: l.SolveMeta, + P: make([]gps.LockedProject, len(l.P)), + } + + l2.SolveMeta.InputImports = make([]string, len(l.SolveMeta.InputImports)) + copy(l2.SolveMeta.InputImports, l.SolveMeta.InputImports) + copy(l2.P, l.P) + + return l2 +} + // toRaw converts the manifest into a representation suitable to write to the lock file func (l *Lock) toRaw() rawLock { raw := rawLock{ diff --git a/project.go b/project.go index 0247ae9b40..5de3ce53cf 100644 --- a/project.go +++ b/project.go @@ -11,7 +11,6 @@ import ( "sort" "github.com/golang/dep/gps" - "github.com/golang/dep/gps/paths" "github.com/golang/dep/gps/pkgtree" "github.com/golang/dep/internal/fs" "github.com/pkg/errors" @@ -101,13 +100,19 @@ type Project struct { // If AbsRoot is not a symlink, then ResolvedAbsRoot should equal AbsRoot. ResolvedAbsRoot string // ImportRoot is the import path of the project's root directory. - ImportRoot gps.ProjectRoot - Manifest *Manifest - Lock *Lock // Optional + ImportRoot gps.ProjectRoot + // The Manifest, as read from Gopkg.toml on disk. + Manifest *Manifest + // The Lock, as read from Gopkg.lock on disk. + Lock *Lock // Optional + // The above Lock, with changes applied to it. There are two possible classes of + // changes: + // 1. Changes to InputImports + // 2. Changes to per-project prune options + ChangedLock *Lock + // The PackageTree representing the project, with hidden and ignored + // packages already trimmed. RootPackageTree pkgtree.PackageTree - // If populated, contains the results of comparing the Lock against the - // current vendor tree, per verify.VerifyDepTree(). - //VendorStatus map[string]verify.VendorStatus } // SetRoot sets the project AbsRoot and ResolvedAbsRoot. If root is not a symlink, ResolvedAbsRoot will be set to root. @@ -127,25 +132,28 @@ func (p *Project) MakeParams() gps.SolveParameters { params := gps.SolveParameters{ RootDir: p.AbsRoot, ProjectAnalyzer: Analyzer{}, + RootPackageTree: p.RootPackageTree, } if p.Manifest != nil { params.Manifest = p.Manifest } - if p.Lock != nil { - params.Lock = p.Lock + // It should be impossible for p.ChangedLock to be nil if p.Lock is non-nil; + // we always want to use the former for solving. + if p.ChangedLock != nil { + params.Lock = p.ChangedLock } return params } -// ParseRootPackageTree analyzes the root project's disk contents to create a +// parseRootPackageTree analyzes the root project's disk contents to create a // PackageTree, trimming out packages that are not relevant for root projects // along the way. // // The resulting tree is cached internally at p.RootPackageTree. -func (p *Project) ParseRootPackageTree() (pkgtree.PackageTree, error) { +func (p *Project) parseRootPackageTree() (pkgtree.PackageTree, error) { if p.RootPackageTree.Packages == nil { ptree, err := pkgtree.ListPackages(p.ResolvedAbsRoot, string(p.ImportRoot)) if err != nil { @@ -177,49 +185,28 @@ func (p *Project) ParseRootPackageTree() (pkgtree.PackageTree, error) { // This function will correctly utilize ignores and requireds from an existing // manifest, if one is present, but will also do the right thing without a // manifest. -func (p *Project) GetDirectDependencyNames(sm gps.SourceManager) (pkgtree.PackageTree, map[gps.ProjectRoot]bool, error) { - ptree, err := p.ParseRootPackageTree() - if err != nil { - return pkgtree.PackageTree{}, nil, err - } - - var ig *pkgtree.IgnoredRuleset - var req map[string]bool - if p.Manifest != nil { - ig = p.Manifest.IgnoredPackages() - req = p.Manifest.RequiredPackages() - } - - rm, _ := ptree.ToReachMap(true, true, false, ig) - reach := rm.FlattenFn(paths.IsStandardImportPath) - - if len(req) > 0 { - // Make a map of imports that are both in the import path list and the - // required list to avoid duplication. - skip := make(map[string]bool, len(req)) - for _, r := range reach { - if req[r] { - skip[r] = true - } - } - - for r := range req { - if !skip[r] { - reach = append(reach, r) - } +func (p *Project) GetDirectDependencyNames(sm gps.SourceManager) (map[gps.ProjectRoot]bool, error) { + var reach []string + if p.ChangedLock != nil { + reach = p.ChangedLock.InputImports() + } else { + ptree, err := p.parseRootPackageTree() + if err != nil { + return nil, err } + reach = externalImportList(ptree, p.Manifest) } directDeps := map[gps.ProjectRoot]bool{} for _, ip := range reach { pr, err := sm.DeduceProjectRoot(ip) if err != nil { - return pkgtree.PackageTree{}, nil, err + return nil, err } directDeps[pr] = true } - return ptree, directDeps, nil + return directDeps, nil } // FindIneffectualConstraints looks for constraint rules expressed in the @@ -233,7 +220,7 @@ func (p *Project) FindIneffectualConstraints(sm gps.SourceManager) []gps.Project return nil } - _, dd, err := p.GetDirectDependencyNames(sm) + dd, err := p.GetDirectDependencyNames(sm) if err != nil { return nil } diff --git a/txn_writer.go b/txn_writer.go index d15d670150..d16b145e2e 100644 --- a/txn_writer.go +++ b/txn_writer.go @@ -5,6 +5,7 @@ package dep import ( + "bytes" "context" "fmt" "io/ioutil" @@ -466,6 +467,7 @@ const ( solveChanged hashMismatch hashVersionMismatch + hashAbsent missingFromTree projectAdded projectRemoved @@ -518,10 +520,12 @@ func NewDeltaWriter(oldLock, newLock *Lock, status map[string]verify.VendorStatu switch stat { case verify.NotInTree: sw.changed[pr] = missingFromTree - case verify.EmptyDigestInLock, verify.DigestMismatchInLock: + case verify.DigestMismatchInLock: sw.changed[pr] = hashMismatch case verify.HashVersionMismatch: sw.changed[pr] = hashVersionMismatch + case verify.EmptyDigestInLock: + sw.changed[pr] = hashAbsent } } } @@ -564,29 +568,51 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l dropped := []gps.ProjectRoot{} // TODO(sdboyer) add a txn/rollback layer, like the safewriter? + i := 0 + tot := len(dw.changed) for pr, reason := range dw.changed { + if reason == projectRemoved { + dropped = append(dropped, pr) + continue + } + to := filepath.FromSlash(filepath.Join(vnewpath, string(pr))) po := dw.pruneOptions.PruneOptionsFor(pr) + if err := sm.ExportPrunedProject(context.TODO(), projs[pr], po, to); err != nil { + return errors.Wrapf(err, "failed to export %s", pr) + } + + i++ lpd := dw.lockDiff.ProjectDeltas[pr] + v, id := projs[pr].Version(), projs[pr].Ident() + var buf bytes.Buffer + fmt.Fprintf(&buf, "(%d/%d) Wrote %s@%s: ", i, tot, id, v) switch reason { case noChange: panic(fmt.Sprintf("wtf, no change for %s", pr)) case solveChanged: if lpd.SourceChanged() { - logger.Printf("Writing %s: source changed (%s -> %s)", pr, lpd.SourceBefore, lpd.SourceAfter) + fmt.Fprintf(&buf, "source changed (%s -> %s)", lpd.SourceBefore, lpd.SourceAfter) } else if lpd.VersionChanged() { - logger.Printf("Writing %s: version changed (%s -> %s)", pr, lpd.VersionBefore, lpd.VersionAfter) + bv, av := "(none)", "(none)" + if lpd.VersionBefore != nil { + bv = lpd.VersionBefore.String() + } + if lpd.VersionAfter != nil { + av = lpd.VersionAfter.String() + } + fmt.Fprintf(&buf, "version changed (%s -> %s)", bv, av) } else if lpd.RevisionChanged() { - logger.Printf("Writing %s: revision changed (%s -> %s)", pr, lpd.RevisionBefore, lpd.RevisionAfter) + fmt.Fprintf(&buf, "revision changed (%s -> %s)", lpd.RevisionBefore, lpd.RevisionAfter) } else if lpd.PackagesChanged() { la, lr := len(lpd.PackagesAdded), len(lpd.PackagesRemoved) if la > 0 && lr > 0 { - logger.Printf("Writing %s: packages changed (%v added, %v removed)", pr, la, lr) + fmt.Fprintf(&buf, "packages changed (%v added, %v removed)", la, lr) } else if la > 0 { - logger.Printf("Writing %s: packages changed (%v added)", pr, la) + fmt.Fprintf(&buf, "packages changed (%v added)", la) } else { - logger.Printf("Writing %s: packages changed (%v removed)", pr, lr) + fmt.Fprintf(&buf, "packages changed (%v removed)", lr) } } else if lpd.PruneOptsChanged() { // Override what's on the lockdiff with the extra info we have; @@ -594,23 +620,20 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l // value from the input param in place. old := lpd.PruneOptsBefore & ^gps.PruneNestedVendorDirs new := lpd.PruneOptsAfter & ^gps.PruneNestedVendorDirs - logger.Printf("Writing %s: prune options changed (%s -> %s)", pr, old, new) + fmt.Fprintf(&buf, "prune options changed (%s -> %s)", old, new) } case hashMismatch: - logger.Printf("Writing %s: hash mismatch between Gopkg.lock and vendor contents", pr) + fmt.Fprintf(&buf, "hash mismatch between Gopkg.lock and vendor contents") case hashVersionMismatch: - logger.Printf("Writing %s: hashing algorithm mismatch", pr) + fmt.Fprintf(&buf, "hashing algorithm mismatch") + case hashAbsent: + fmt.Fprintf(&buf, "hash digest absent from lock") case projectAdded: - logger.Printf("Writing new project %s", pr) - case projectRemoved: - dropped = append(dropped, pr) - continue + fmt.Fprintf(&buf, "new project") case missingFromTree: - logger.Printf("Writing %s: missing from vendor", pr) - } - if err := sm.ExportPrunedProject(context.TODO(), projs[pr], po, to); err != nil { - return errors.Wrapf(err, "failed to export %s", pr) + fmt.Fprintf(&buf, "missing from vendor", pr) } + logger.Print(buf.String()) digest, err := verify.DigestFromDirectory(to) if err != nil { @@ -649,9 +672,17 @@ func (dw *DeltaWriter) Write(path string, sm gps.SourceManager, examples bool, l } } - for _, pr := range dropped { - // Kind of a lie to print this here. ¯\_(ツ)_/¯ - logger.Printf("Discarding unused project %s", pr) + for i, pr := range dropped { + // Kind of a lie to print this. ¯\_(ツ)_/¯ + logger.Printf("(%d/%d) Removed unused project %s", tot-(len(dropped)-i-1), tot, pr) + } + + // Ensure vendor/.git is preserved if present + if hasDotGit(vpath) { + err = fs.RenameWithFallback(filepath.Join(vpath, ".git"), filepath.Join(vnewpath, "vendor/.git")) + if _, ok := err.(*os.LinkError); ok { + return errors.Wrap(err, "failed to preserve vendor/.git") + } } err = os.RemoveAll(vpath)