diff --git a/internal/gps/prune.go b/internal/gps/prune.go new file mode 100644 index 0000000000..b67e01b1ff --- /dev/null +++ b/internal/gps/prune.go @@ -0,0 +1,325 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gps + +import ( + "log" + "os" + "path/filepath" + "strings" + + "github.com/pkg/errors" +) + +// PruneOptions represents the pruning options used to write the dependecy tree. +type PruneOptions uint8 + +const ( + // PruneNestedVendorDirs indicates if nested vendor directories should be pruned. + PruneNestedVendorDirs = 1 << iota + // PruneUnusedPackages indicates if unused Go packages should be pruned. + PruneUnusedPackages + // PruneNonGoFiles indicates if non-Go files should be pruned. + // Files matching licenseFilePrefixes and legalFileSubstrings are kept in + // an attempt to comply with legal requirements. + PruneNonGoFiles + // PruneGoTestFiles indicates if Go test files should be pruned. + PruneGoTestFiles +) + +var ( + // licenseFilePrefixes is a list of name prefixes for license files. + licenseFilePrefixes = []string{ + "license", + "licence", + "copying", + "unlicense", + "copyright", + "copyleft", + } + // legalFileSubstrings contains substrings that are likey part of a legal + // declaration file. + legalFileSubstrings = []string{ + "authors", + "contributors", + "legal", + "notice", + "disclaimer", + "patent", + "third-party", + "thirdparty", + } +) + +// Prune removes excess files from the dep tree whose root is baseDir based +// on the PruneOptions passed. +// +// A Lock must be passed if PruneUnusedPackages is toggled on. +func Prune(baseDir string, options PruneOptions, l Lock, logger *log.Logger) error { + // TODO(ibrasho) allow passing specific options per project + for _, lp := range l.Projects() { + projectDir := filepath.Join(baseDir, string(lp.Ident().ProjectRoot)) + err := PruneProject(projectDir, lp, options, logger) + if err != nil { + return err + } + } + + return nil +} + +// PruneProject remove excess files according to the options passed, from +// the lp directory in baseDir. +func PruneProject(baseDir string, lp LockedProject, options PruneOptions, logger *log.Logger) error { + projectDir := filepath.Join(baseDir, string(lp.Ident().ProjectRoot)) + + if (options & PruneNestedVendorDirs) != 0 { + if err := pruneNestedVendorDirs(projectDir); err != nil { + return err + } + } + + if (options & PruneUnusedPackages) != 0 { + if err := pruneUnusedPackages(lp, projectDir, logger); err != nil { + return errors.Wrap(err, "failed to prune unused packages") + } + } + + if (options & PruneNonGoFiles) != 0 { + if err := pruneNonGoFiles(projectDir, logger); err != nil { + return errors.Wrap(err, "failed to prune non-Go files") + } + } + + if (options & PruneGoTestFiles) != 0 { + if err := pruneGoTestFiles(projectDir, logger); err != nil { + return errors.Wrap(err, "failed to prune Go test files") + } + } + + return nil +} + +// pruneNestedVendorDirs deletes all nested vendor directories within baseDir. +func pruneNestedVendorDirs(baseDir string) error { + return filepath.Walk(baseDir, stripVendor) +} + +// pruneUnusedPackages deletes unimported packages found within baseDir. +// Determining whether packages are imported or not is based on the passed LockedProject. +func pruneUnusedPackages(lp LockedProject, projectDir string, logger *log.Logger) error { + pr := string(lp.Ident().ProjectRoot) + logger.Printf("Calculating unused packages in %s to prune.\n", pr) + + unusedPackages, err := calculateUnusedPackages(lp, projectDir) + if err != nil { + return errors.Wrapf(err, "could not calculate unused packages in %s", pr) + } + + logger.Printf("Found the following unused packages in %s:\n", pr) + for pkg := range unusedPackages { + logger.Printf(" * %s\n", filepath.Join(pr, pkg)) + } + + unusedPackagesFiles, err := collectUnusedPackagesFiles(projectDir, unusedPackages) + if err != nil { + return errors.Wrapf(err, "could not collect unused packages' files in %s", pr) + } + + if err := deleteFiles(unusedPackagesFiles); err != nil { + return errors.Wrapf(err, "") + } + + return nil +} + +// calculateUnusedPackages generates a list of unused packages in lp. +func calculateUnusedPackages(lp LockedProject, projectDir string) (map[string]struct{}, error) { + // TODO(ibrasho): optimize this... + unused := make(map[string]struct{}) + imported := make(map[string]struct{}) + for _, pkg := range lp.Packages() { + imported[pkg] = struct{}{} + } + + err := filepath.Walk(projectDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Ignore anything that's not a directory. + if !info.IsDir() { + return nil + } + + pkg, err := filepath.Rel(projectDir, path) + if err != nil { + return errors.Wrap(err, "unexpected error while calculating unused packages") + } + + pkg = filepath.ToSlash(pkg) + if _, ok := imported[pkg]; !ok { + unused[pkg] = struct{}{} + } + + return nil + }) + + return unused, err +} + +// collectUnusedPackagesFiles returns a slice of all files in the unused packages in projectDir. +func collectUnusedPackagesFiles(projectDir string, unusedPackages map[string]struct{}) ([]string, error) { + // TODO(ibrasho): is this useful? + files := make([]string, 0, len(unusedPackages)) + + err := filepath.Walk(projectDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Ignore directories. + if info.IsDir() { + return nil + } + + // Ignore perserved files. + if isPreservedFile(info.Name()) { + return nil + } + + pkg, err := filepath.Rel(projectDir, filepath.Dir(path)) + if err != nil { + return errors.Wrap(err, "unexpected error while calculating unused packages") + } + + pkg = filepath.ToSlash(pkg) + if _, ok := unusedPackages[pkg]; ok { + files = append(files, path) + } + + return nil + }) + + return files, err +} + +// pruneNonGoFiles delete all non-Go files existing within baseDir. +// Files with names that are prefixed by any entry in preservedNonGoFiles +// are not deleted. +func pruneNonGoFiles(baseDir string, logger *log.Logger) error { + files, err := collectNonGoFiles(baseDir, logger) + if err != nil { + return errors.Wrap(err, "could not collect non-Go files") + } + + if err := deleteFiles(files); err != nil { + return errors.Wrap(err, "could not prune Go test files") + } + + return nil +} + +// collectNonGoFiles returns a slice containing all non-Go files in baseDir. +// Files meeting the checks in isPreservedFile are not returned. +func collectNonGoFiles(baseDir string, logger *log.Logger) ([]string, error) { + files := make([]string, 0) + + err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Ignore directories. + if info.IsDir() { + return nil + } + + // Ignore all Go files. + if strings.HasSuffix(info.Name(), ".go") { + return nil + } + + // Ignore perserved files. + if isPreservedFile(info.Name()) { + return nil + } + + files = append(files, path) + + return nil + }) + + return files, err +} + +// isPreservedFile checks if the file name indicates that the file should be +// preserved based on licenseFilePrefixes or legalFileSubstrings. +func isPreservedFile(name string) bool { + name = strings.ToLower(name) + + for _, prefix := range licenseFilePrefixes { + if strings.HasPrefix(name, prefix) { + return true + } + } + + for _, substring := range legalFileSubstrings { + if strings.Contains(name, substring) { + return true + } + } + + return false +} + +// pruneGoTestFiles deletes all Go test files (*_test.go) within baseDir. +func pruneGoTestFiles(baseDir string, logger *log.Logger) error { + files, err := collectGoTestFiles(baseDir) + if err != nil { + return errors.Wrap(err, "could not collect Go test files") + } + + if err := deleteFiles(files); err != nil { + return errors.Wrap(err, "could not prune Go test files") + } + + return nil +} + +// collectGoTestFiles returns a slice contains all Go test files (any files +// prefixed with _test.go) in baseDir. +func collectGoTestFiles(baseDir string) ([]string, error) { + files := make([]string, 0) + + err := filepath.Walk(baseDir, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + // Ignore directories. + if info.IsDir() { + return nil + } + + // Ignore any files that is not a Go test file. + if strings.HasSuffix(info.Name(), "_test.go") { + files = append(files, path) + } + + return nil + }) + + return files, err +} + +func deleteFiles(paths []string) error { + for _, path := range paths { + if err := os.Remove(path); err != nil { + return err + } + } + return nil +} diff --git a/internal/gps/prune_test.go b/internal/gps/prune_test.go new file mode 100644 index 0000000000..de7791cd77 --- /dev/null +++ b/internal/gps/prune_test.go @@ -0,0 +1,324 @@ +// Copyright 2017 The Go Authors. All rights reserved. +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +package gps + +import ( + "io/ioutil" + "log" + "testing" + + "github.com/golang/dep/internal/test" +) + +func TestPruneUnusedPackages(t *testing.T) { + h := test.NewHelper(t) + defer h.Cleanup() + + h.TempDir(".") + + pr := "github.com/test/project" + pi := ProjectIdentifier{ProjectRoot: ProjectRoot(pr)} + + testcases := []struct { + name string + lp LockedProject + fs fsTestCase + err bool + }{ + { + "one-package", + LockedProject{ + pi: pi, + pkgs: []string{"."}, + }, + fsTestCase{ + before: filesystemState{ + files: []fsPath{ + {"main.go"}, + }, + }, + after: filesystemState{ + files: []fsPath{ + {"main.go"}, + }, + }, + }, + false, + }, + { + "nested-package", + LockedProject{ + pi: pi, + pkgs: []string{"pkg"}, + }, + fsTestCase{ + before: filesystemState{ + dirs: []fsPath{ + {"pkg"}, + }, + files: []fsPath{ + {"main.go"}, + {"pkg", "main.go"}, + }, + }, + after: filesystemState{ + dirs: []fsPath{ + {"pkg"}, + }, + files: []fsPath{ + {"pkg", "main.go"}, + }, + }, + }, + false, + }, + { + "complex-project", + LockedProject{ + pi: pi, + pkgs: []string{"pkg", "pkg/nestedpkg/otherpkg"}, + }, + fsTestCase{ + before: filesystemState{ + dirs: []fsPath{ + {"pkg"}, + {"pkg", "nestedpkg"}, + {"pkg", "nestedpkg", "otherpkg"}, + }, + files: []fsPath{ + {"main.go"}, + {"COPYING"}, + {"pkg", "main.go"}, + {"pkg", "nestedpkg", "main.go"}, + {"pkg", "nestedpkg", "PATENT.md"}, + {"pkg", "nestedpkg", "otherpkg", "main.go"}, + }, + }, + after: filesystemState{ + dirs: []fsPath{ + {"pkg"}, + {"pkg", "nestedpkg"}, + {"pkg", "nestedpkg", "otherpkg"}, + }, + files: []fsPath{ + {"COPYING"}, + {"pkg", "main.go"}, + {"pkg", "nestedpkg", "PATENT.md"}, + {"pkg", "nestedpkg", "otherpkg", "main.go"}, + }, + }, + }, + false, + }, + } + + logger := log.New(ioutil.Discard, "", 0) + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + h.TempDir(pr) + projectDir := h.Path(pr) + tc.fs.before.root = projectDir + tc.fs.after.root = projectDir + + tc.fs.before.setup(t) + + err := pruneUnusedPackages(tc.lp, projectDir, logger) + if tc.err && err == nil { + t.Errorf("expected an error, got nil") + } else if !tc.err && err != nil { + t.Errorf("unexpected error: %s", err) + } + + tc.fs.after.assert(t) + }) + } +} + +func TestPruneNonGoFiles(t *testing.T) { + h := test.NewHelper(t) + defer h.Cleanup() + + h.TempDir(".") + + testcases := []struct { + name string + fs fsTestCase + err bool + }{ + { + "one-file", + fsTestCase{ + before: filesystemState{ + files: []fsPath{ + {"README.md"}, + }, + }, + after: filesystemState{}, + }, + false, + }, + { + "multiple-files", + fsTestCase{ + before: filesystemState{ + files: []fsPath{ + {"main.go"}, + {"main_test.go"}, + {"README"}, + }, + }, + after: filesystemState{ + files: []fsPath{ + {"main.go"}, + {"main_test.go"}, + }, + }, + }, + false, + }, + { + "mixed-files", + fsTestCase{ + before: filesystemState{ + dirs: []fsPath{ + {"dir"}, + }, + files: []fsPath{ + {"dir", "main.go"}, + {"dir", "main_test.go"}, + {"dir", "db.sqlite"}, + }, + }, + after: filesystemState{ + dirs: []fsPath{ + {"dir"}, + }, + files: []fsPath{ + {"dir", "main.go"}, + {"dir", "main_test.go"}, + }, + }, + }, + false, + }, + } + + logger := log.New(ioutil.Discard, "", 0) + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + h.TempDir(tc.name) + baseDir := h.Path(tc.name) + tc.fs.before.root = baseDir + tc.fs.after.root = baseDir + + tc.fs.before.setup(t) + + err := pruneNonGoFiles(baseDir, logger) + if tc.err && err == nil { + t.Errorf("expected an error, got nil") + } else if !tc.err && err != nil { + t.Errorf("unexpected error: %s", err) + } + + tc.fs.after.assert(t) + }) + } +} + +func TestPruneGoTestFiles(t *testing.T) { + h := test.NewHelper(t) + defer h.Cleanup() + + h.TempDir(".") + + testcases := []struct { + name string + fs fsTestCase + err bool + }{ + { + "one-test-file", + fsTestCase{ + before: filesystemState{ + files: []fsPath{ + {"main_test.go"}, + }, + }, + after: filesystemState{}, + }, + false, + }, + { + "multiple-files", + fsTestCase{ + before: filesystemState{ + dirs: []fsPath{ + {"dir"}, + }, + files: []fsPath{ + {"dir", "main_test.go"}, + {"dir", "main2_test.go"}, + }, + }, + after: filesystemState{ + dirs: []fsPath{ + {"dir"}, + }, + }, + }, + false, + }, + { + "mixed-files", + fsTestCase{ + before: filesystemState{ + dirs: []fsPath{ + {"dir"}, + }, + files: []fsPath{ + {"dir", "main.go"}, + {"dir", "main2.go"}, + {"dir", "main_test.go"}, + {"dir", "main2_test.go"}, + }, + }, + after: filesystemState{ + dirs: []fsPath{ + {"dir"}, + }, + files: []fsPath{ + {"dir", "main.go"}, + {"dir", "main2.go"}, + }, + }, + }, + false, + }, + } + + logger := log.New(ioutil.Discard, "", 0) + + for _, tc := range testcases { + t.Run(tc.name, func(t *testing.T) { + h.TempDir(tc.name) + baseDir := h.Path(tc.name) + tc.fs.before.root = baseDir + tc.fs.after.root = baseDir + + tc.fs.before.setup(t) + + err := pruneGoTestFiles(baseDir, logger) + if tc.err && err == nil { + t.Errorf("expected an error, got nil") + } else if !tc.err && err != nil { + t.Errorf("unexpected error: %s", err) + } + + tc.fs.after.assert(t) + }) + } +} diff --git a/internal/gps/strip_vendor.go b/internal/gps/strip_vendor.go index 184a0d2d2e..1b224d0e6a 100644 --- a/internal/gps/strip_vendor.go +++ b/internal/gps/strip_vendor.go @@ -16,29 +16,27 @@ func stripVendor(path string, info os.FileInfo, err error) error { return err } - if info.Name() == "vendor" { - if _, err := os.Lstat(path); err != nil { + // Skip anything not named vendor + if info.Name() != "vendor" { + return nil + } + + // If the file is a symlink to a directory, delete the symlink. + if (info.Mode() & os.ModeSymlink) != 0 { + realInfo, err := os.Stat(path) + if err != nil { return err } - - if (info.Mode() & os.ModeSymlink) != 0 { - realInfo, err := os.Stat(path) - if err != nil { - return err - } - if realInfo.IsDir() { - return os.Remove(path) - } + if realInfo.IsDir() { + return os.Remove(path) } + } - if info.IsDir() { - if err := os.RemoveAll(path); err != nil { - return err - } - return filepath.SkipDir + if info.IsDir() { + if err := os.RemoveAll(path); err != nil { + return err } - - return nil + return filepath.SkipDir } return nil diff --git a/internal/gps/strip_vendor_windows.go b/internal/gps/strip_vendor_windows.go index 7d6acc4a3c..5e51513590 100644 --- a/internal/gps/strip_vendor_windows.go +++ b/internal/gps/strip_vendor_windows.go @@ -14,38 +14,42 @@ func stripVendor(path string, info os.FileInfo, err error) error { return err } - if info.Name() == "vendor" { - if _, err := os.Lstat(path); err == nil { - symlink := (info.Mode() & os.ModeSymlink) != 0 - dir := info.IsDir() - - switch { - case symlink && dir: - // This could be a windows junction directory. Support for these in the - // standard library is spotty, and we could easily delete an important - // folder if we called os.Remove or os.RemoveAll. Just skip these. - // - // TODO: If we could distinguish between junctions and Windows symlinks, - // we might be able to safely delete symlinks, even though junctions are - // dangerous. - return filepath.SkipDir - - case symlink: - realInfo, err := os.Stat(path) - if err != nil { - return err - } - if realInfo.IsDir() { - return os.Remove(path) - } - - case dir: - if err := os.RemoveAll(path); err != nil { - return err - } - return filepath.SkipDir - } + if info.Name() != "vendor" { + return nil + } + + if _, err := os.Lstat(path); err != nil { + return nil + } + + symlink := (info.Mode() & os.ModeSymlink) != 0 + dir := info.IsDir() + + switch { + case symlink && dir: + // This could be a windows junction directory. Support for these in the + // standard library is spotty, and we could easily delete an important + // folder if we called os.Remove or os.RemoveAll. Just skip these. + // + // TODO: If we could distinguish between junctions and Windows symlinks, + // we might be able to safely delete symlinks, even though junctions are + // dangerous. + return filepath.SkipDir + + case symlink: + realInfo, err := os.Stat(path) + if err != nil { + return err + } + if realInfo.IsDir() { + return os.Remove(path) + } + + case dir: + if err := os.RemoveAll(path); err != nil { + return err } + return filepath.SkipDir } return nil