go/packages: Ignore more errors in go list

Adds some hacks to work around go list bugs causing golang/go#31462 and
golang/go#32755.

In addition, appends error messages generated in this manner to stdout instead
of discarding stdout entirely, and ensures that this does not cause a
conflicting package error by adding an internal field to packageJson.

go/packages/golist.go

@@ -269,6 +269,17 @@ func runContainsQueries(cfg *Config, driver driver, response *responseDeduper, q
 // modCacheRegexp splits a path in a module cache into module, module version, and package.
 var modCacheRegexp = regexp.MustCompile(`(.*)@([^/\\]*)(.*)`)
 
+// noFileRe looks for a package name in a "no Go files in" errors line that is output by go list.
+var noFileRe = regexp.MustCompile(`(?m)^go build ([\p{L}\p{Nd}\/.]+): (no Go files in.*)$`)
+
+// cgoErrRe attempts to detect cgo errors from go list stderr output
+var cgoErrRe = regexp.MustCompile(
+	`(collect2: error: ld returned)|(could not determine kind of name for C\.)|` +
+		`(fatal error: \S+.(c|h): No such file or directory)`)
+
+// goListErrRe attempts to extract errors from go list stderr output
+var goListErrRe = regexp.MustCompile(`(?m)^# ([\p{L}\p{Nd}\/.]+)\n((?:[^#].*\n)+)`)
+
 func runNamedQueries(cfg *Config, driver driver, response *responseDeduper, queries []string) error {
 	// calling `go env` isn't free; bail out if there's nothing to do.
 	if len(queries) == 0 {
@@ -553,6 +564,8 @@ type jsonPackage struct {
 	ForTest         string // q in a "p [q.test]" package, else ""
 	DepOnly         bool
 
+	Gen bool // generated by this code
+
 	Error *jsonPackageError
 }
 
@@ -611,11 +624,15 @@ func golistDriver(cfg *Config, words ...string) (*driverResponse, error) {
 		}
 
 		if old, found := seen[p.ImportPath]; found {
-			if !reflect.DeepEqual(p, old) {
+			if p.Gen {
+				// new package is generated, skip it
+				continue
+			} else if !reflect.DeepEqual(p, old) {
 				return nil, fmt.Errorf("internal error: go list gives conflicting information for package %v", p.ImportPath)
+			} else if !old.Gen {
+				// skip the duplicate if the old package wasn't generated
+				continue
 			}
-			// skip the duplicate
-			continue
 		}
 		seen[p.ImportPath] = p
 
@@ -804,15 +821,23 @@ func invokeGo(cfg *Config, args ...string) (*bytes.Buffer, error) {
 		// a zero exit status and set an error on that package.
 		if len(stderr.String()) > 0 && strings.Contains(stderr.String(), "no Go files in") {
 			// try to extract package name from string
-			stderrStr := stderr.String()
-			var importPath string
-			colon := strings.Index(stderrStr, ":")
-			if colon > 0 && strings.HasPrefix(stderrStr, "go build ") {
-				importPath = stderrStr[len("go build "):colon]
+			for _, matches := range noFileRe.FindAllStringSubmatch(stderr.String(), -1) {
+				stdout.WriteString(fmt.Sprintf("\n"+`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q},"Gen": true}`,
+					matches[1], matches[2]))
 			}
-			output := fmt.Sprintf(`{"ImportPath": %q,"Incomplete": true,"Error": {"Pos": "","Err": %q}}`,
-				importPath, strings.Trim(stderrStr, "\n"))
-			return bytes.NewBufferString(output), nil
+			return stdout, nil
+		}
+
+		// Workaround for an instance of golang.org/issue/26755: go list -e will
+		// return a non-zero exit status if cgo preprocessing fails. But it should
+		// return a zero exit status and set an error on that package.
+		if len(stderr.String()) > 0 && cgoErrRe.MatchString(stderr.String()) {
+			for _, matches := range goListErrRe.FindAllStringSubmatch(stderr.String(), -1) {
+				stdout.WriteString(fmt.Sprintf("\n"+`{"ImportPath": %q,"Incomplete":true,"Error": {"Pos": "","Err": %q},"Gen": true}`,
+					matches[1], strings.Trim(matches[2], "\n")))
+			}
+
+			return stdout, nil
 		}
 
 		// Export mode entails a build.