go-fuzz-build: use go/packages

golang.org/x/tools/go/packages is a new package designed to aid in
locating, parsing, and typechecking Go packages.
This change updates go-fuzz-build to use it.
This effectively resulted in a rewrite of go-fuzz-build/main.go.


Some benefits are:

* Improved performance. go-fuzz-build appears to be 2x faster on
  first build, and 4x faster on subsequent builds.
  This is due to several factors. One big one is the use of cached
  type information.
  And there are more performance improvements available;
  see the TODOs in the code.

* More robust. Previously, the list of ignored packages had to be
  updated every time the standard library changed.
  Now the ignored packages are computed on the fly.

* More user-friendly. You can now invoke go-fuzz-build without
  a package path; it assumes ".", and relative paths work seamlessly.

* Closer to functionality in a module-enabled world.
  See the TODOs in the code.

* Simpler and better documented.


Outstanding issue:

* cgo code is not instrumented.
  See the discussion and linked Go toolchain issue in the code.


Implementation notes:

* go-fuzz-defs has been split into two.
  The constants, which are shared across go-fuzz, go-fuzz-build,
  and instrumented code, have been left in place.
  The types, which were only needed by go-fuzz and go-fuzz-build,
  have been moved to a new internal package.
  This allowed us to break the go-fuzz-dep/go-fuzz-defs dependency
  and std-depends-on-non-std problems in a simpler way:
  We simply make another copy of go-fuzz-defs.
  See the comments in the code for details.

* There is now a shared AST across the coverage and sonar passes.
  This works out OK, since the sonar pass used to request coverage first.
  Now we just have to be careful about the order in which we use and
  mutate the AST. This is well documented in the code.

go-fuzz-build/cover.go

@@ -17,36 +17,30 @@ import (
 	"strings"
 
 	. "github.com/dvyukov/go-fuzz/go-fuzz-defs"
+	. "github.com/dvyukov/go-fuzz/internal/go-fuzz-types"
 )
 
 const fuzzdepPkg = "_go_fuzz_dep_"
 
-func instrument(pkg, shortName, fullName string, fset *token.FileSet, parsedFile *ast.File, info *types.Info, out io.Writer, lits map[Literal]struct{}, blocks *[]CoverBlock, sonar *[]CoverBlock) {
+func instrument(pkg, fullName string, fset *token.FileSet, parsedFile *ast.File, info *types.Info, out io.Writer, blocks *[]CoverBlock, sonar *[]CoverBlock) {
 	file := &File{
-		fset:      fset,
-		pkg:       pkg,
-		shortName: shortName,
-		fullName:  fullName,
-		astFile:   parsedFile,
-		blocks:    blocks,
-		info:      info,
-	}
-	file.addImport("go-fuzz-dep", fuzzdepPkg, "Main")
-
-	if lits != nil {
-		ast.Walk(&LiteralCollector{lits}, file.astFile)
-	}
-
-	ast.Walk(file, file.astFile)
-
-	if sonar != nil {
+		fset:     fset,
+		pkg:      pkg,
+		fullName: fullName,
+		astFile:  parsedFile,
+		blocks:   blocks,
+		info:     info,
+	}
+	if sonar == nil {
+		file.addImport("go-fuzz-dep", fuzzdepPkg, "Main")
+		ast.Walk(file, file.astFile)
+	} else {
 		s := &Sonar{
-			fset:      fset,
-			shortName: shortName,
-			fullName:  fullName,
-			pkg:       pkg,
-			blocks:    sonar,
-			info:      info,
+			fset:     fset,
+			fullName: fullName,
+			pkg:      pkg,
+			blocks:   sonar,
+			info:     info,
 		}
 		ast.Walk(s, file.astFile)
 	}
@@ -55,12 +49,11 @@ func instrument(pkg, shortName, fullName string, fset *token.FileSet, parsedFile
 }
 
 type Sonar struct {
-	fset      *token.FileSet
-	shortName string
-	fullName  string
-	pkg       string
-	blocks    *[]CoverBlock
-	info      *types.Info
+	fset     *token.FileSet
+	fullName string
+	pkg      string
+	blocks   *[]CoverBlock
+	info     *types.Info
 }
 
 var sonarSeq = 0
@@ -394,6 +387,7 @@ func isLen(n ast.Expr) bool {
 }
 
 type LiteralCollector struct {
+	ctxt *Context
 	lits map[Literal]struct{}
 }
 
@@ -421,18 +415,18 @@ func (lc *LiteralCollector) Visit(n ast.Node) (w ast.Visitor) {
 		lit := nn.Value
 		switch nn.Kind {
 		case token.STRING:
-			lc.lits[Literal{unquote(lit), true}] = struct{}{}
+			lc.lits[Literal{lc.unquote(lit), true}] = struct{}{}
 		case token.CHAR:
-			lc.lits[Literal{unquote(lit), false}] = struct{}{}
+			lc.lits[Literal{lc.unquote(lit), false}] = struct{}{}
 		case token.INT:
 			if lit[0] < '0' || lit[0] > '9' {
-				failf("unsupported literal '%v'", lit)
+				lc.ctxt.failf("unsupported literal '%v'", lit)
 			}
 			v, err := strconv.ParseInt(lit, 0, 64)
 			if err != nil {
 				u, err := strconv.ParseUint(lit, 0, 64)
 				if err != nil {
-					failf("failed to parse int literal '%v': %v", lit, err)
+					lc.ctxt.failf("failed to parse int literal '%v': %v", lit, err)
 				}
 				v = int64(u)
 			}
@@ -492,13 +486,12 @@ func initialComments(content []byte) []byte {
 }
 
 type File struct {
-	fset      *token.FileSet
-	pkg       string
-	shortName string
-	fullName  string
-	astFile   *ast.File
-	blocks    *[]CoverBlock
-	info      *types.Info
+	fset     *token.FileSet
+	pkg      string
+	fullName string
+	astFile  *ast.File
+	blocks   *[]CoverBlock
+	info     *types.Info
 }
 
 var slashslash = []byte("//")
@@ -898,10 +891,10 @@ func hasFuncLiteral(n ast.Node) (bool, token.Pos) {
 	return literal.found(), token.Pos(literal)
 }
 
-func unquote(s string) string {
+func (lc *LiteralCollector) unquote(s string) string {
 	t, err := strconv.Unquote(s)
 	if err != nil {
-		failf("cover: improperly quoted string %q\n", s)
+		lc.ctxt.failf("cover: improperly quoted string %q\n", s)
 	}
 	return t
 }