go/tools/gopackagesdriver: add automatic target detection

.vscode/extensions.json

@@ -0,0 +1,8 @@
+{
+  // See http://go.microsoft.com/fwlink/?LinkId=827846
+  // for the documentation about the extensions.json format
+  "recommendations": [
+    "bazelbuild.vscode-bazel",
+    "golang.go",
+  ]
+}

.vscode/settings.json

@@ -0,0 +1,32 @@
+{
+  "editor.formatOnSave": true,
+  "files.trimTrailingWhitespace": true,
+  "files.insertFinalNewline": true,
+  "go.goroot": "${workspaceFolder}/bazel-${workspaceFolderBasename}/external/go_sdk",
+  "go.toolsEnvVars": {
+    "GOPACKAGESDRIVER": "${workspaceFolder}/tools/gopackagesdriver.sh"
+  },
+  "go.enableCodeLens": {
+    "references": false,
+    "runtest": false
+  },
+  "gopls": {
+    "formatting.gofumpt": true,
+    "formatting.local": "github.com/bazelbuild/rules_go",
+    "ui.completion.usePlaceholders": true,
+    "ui.semanticTokens": true,
+    "ui.codelenses": {
+      "gc_details": false,
+      "regenerate_cgo": false,
+      "generate": false,
+      "test": false,
+      "tidy": false,
+      "upgrade_dependency": false,
+      "vendor": false
+    },
+  },
+  "go.useLanguageServer": true,
+  "go.buildOnSave": "off",
+  "go.lintOnSave": "off",
+  "go.vetOnSave": "off",
+}

go/tools/builders/stdliblist.go

@@ -111,7 +111,7 @@ func stdlibPackageID(importPath string) string {
 
 func execRootPath(execRoot, p string) string {
 	dir, _ := filepath.Rel(execRoot, p)
-	return filepath.Join("__BAZEL_EXECROOT__", dir)
+	return filepath.Join("__BAZEL_OUTPUT_BASE__", dir)
 }
 
 func absoluteSourcesPaths(execRoot, pkgDir string, srcs []string) []string {

go/tools/gopackagesdriver/BUILD.bazel

@@ -5,11 +5,13 @@ go_library(
     srcs = [
         "bazel.go",
         "bazel_json_builder.go",
+        "build_context.go",
         "driver_request.go",
         "flatpackage.go",
         "json_packages_driver.go",
         "main.go",
         "packageregistry.go",
+        "utils.go",
     ],
     importpath = "github.com/bazelbuild/rules_go/go/tools/gopackagesdriver",
     visibility = ["//visibility:private"],

go/tools/gopackagesdriver/aspect.bzl

@@ -17,90 +17,123 @@ load(
     "GoArchive",
     "GoStdLib",
 )
-load(
-    "//go/private:context.bzl",
-    "go_context",
-)
 load(
     "@bazel_skylib//lib:paths.bzl",
     "paths",
 )
-load(
-    "@bazel_skylib//lib:collections.bzl",
-    "collections",
-)
 
 GoPkgInfo = provider()
 
 def _is_file_external(f):
     return f.owner.workspace_root != ""
 
 def _file_path(f):
-    if f.is_source and not _is_file_external(f):
-        return paths.join("__BAZEL_WORKSPACE__", f.path)
-    return paths.join("__BAZEL_EXECROOT__", f.path)
+    prefix = "__BAZEL_WORKSPACE__"
+    if not f.is_source:
+        prefix = "__BAZEL_EXECROOT__"
+    elif _is_file_external(f):
+        prefix = "__BAZEL_OUTPUT_BASE__"
+    return paths.join(prefix, f.path)
+
+def _go_archive_to_pkg(archive):
+    return struct(
+        ID = str(archive.data.label),
+        PkgPath = archive.data.importpath,
+        ExportFile = _file_path(archive.data.export_file),
+        GoFiles = [
+            _file_path(src)
+            for src in archive.data.orig_srcs
+        ],
+        CompiledGoFiles = [
+            _file_path(src)
+            for src in archive.data.srcs
+        ],
+    )
+
+def _make_pkg_json(ctx, archive, pkg_info):
+    pkg_json_file = ctx.actions.declare_file(archive.data.name + ".pkg.json")
+    ctx.actions.write(pkg_json_file, content = pkg_info.to_json())
+    return pkg_json_file
 
 def _go_pkg_info_aspect_impl(target, ctx):
     # Fetch the stdlib JSON file from the inner most target
     stdlib_json_file = None
 
     deps_transitive_json_file = []
     deps_transitive_export_file = []
-    for dep in getattr(ctx.rule.attr, "deps", []):
-        if GoPkgInfo in dep:
-            pkg_info = dep[GoPkgInfo]
-            deps_transitive_json_file.append(pkg_info.transitive_json_file)
-            deps_transitive_export_file.append(pkg_info.transitive_export_file)
-            # Fetch the stdlib json from the first dependency
-            if not stdlib_json_file:
-                stdlib_json_file = pkg_info.stdlib_json_file
-
-    # If deps are embedded, do not gather their json or export_file since they
-    # are included in the current target, but do gather their deps'.
-    for dep in getattr(ctx.rule.attr, "embed", []):
-        if GoPkgInfo in dep:
-            pkg_info = dep[GoPkgInfo]
-            deps_transitive_json_file.append(pkg_info.deps_transitive_json_file)
-            deps_transitive_export_file.append(pkg_info.deps_transitive_export_file)
-
-    pkg_json_file = None
-    export_file = None
+    deps_transitive_compiled_go_files = []
+
+    for attr in ["deps", "embed"]:
+        for dep in getattr(ctx.rule.attr, attr, []):
+            if GoPkgInfo in dep:
+                pkg_info = dep[GoPkgInfo]
+                if attr == "deps":
+                    deps_transitive_json_file.append(pkg_info.transitive_json_file)
+                    deps_transitive_export_file.append(pkg_info.transitive_export_file)
+                    deps_transitive_compiled_go_files.append(pkg_info.transitive_compiled_go_files)
+                elif attr == "embed":
+                    # If deps are embedded, do not gather their json or export_file since they
+                    # are included in the current target, but do gather their deps'.
+                    deps_transitive_json_file.append(pkg_info.deps_transitive_json_file)
+                    deps_transitive_export_file.append(pkg_info.deps_transitive_export_file)
+                    deps_transitive_compiled_go_files.append(pkg_info.deps_transitive_compiled_go_files)
+
+                # Fetch the stdlib json from the first dependency
+                if not stdlib_json_file:
+                    stdlib_json_file = pkg_info.stdlib_json_file
+
+    pkg_json_files = []
+    compiled_go_files = []
+    export_files = []
+
     if GoArchive in target:
         archive = target[GoArchive]
-        export_file = archive.data.export_file
-        pkg = struct(
-            ID = str(archive.data.label),
-            PkgPath = archive.data.importpath,
-            ExportFile = _file_path(archive.data.export_file),
-            GoFiles = [
-                _file_path(src)
-                for src in archive.data.orig_srcs
-            ],
-            CompiledGoFiles = [
-                _file_path(src)
-                for src in archive.data.srcs
-            ],
-        )
-        pkg_json_file = ctx.actions.declare_file(archive.data.name + ".pkg.json")
-        ctx.actions.write(pkg_json_file, content = pkg.to_json())
-        # If there was no stdlib json in any dependencies, fetch it from the
-        # current go_ node.
-        if not stdlib_json_file:
-            stdlib_json_file = ctx.attr._go_stdlib[GoStdLib]._list_json
+        compiled_go_files.extend(archive.source.srcs)
+        export_files.append(archive.data.export_file)
+        pkg = _go_archive_to_pkg(archive)
+        pkg_json_files.append(_make_pkg_json(ctx, archive, pkg))
+
+        # if the rule is a test, we need to get the embedded go_library with the current
+        # test's sources. For that, consume the dependency via GoArchive.direct so that
+        # the test source files are there. Then, create the pkg json file directly. Only
+        # do that for direct dependencies that are not defined as deps, and use the
+        # importpath to find which.
+        if ctx.rule.kind == "go_test":
+            deps_targets = [
+                dep[GoArchive].data.importpath
+                for dep in ctx.rule.attr.deps
+                if GoArchive in dep
+            ]
+            for archive in target[GoArchive].direct:
+                if archive.data.importpath not in deps_targets:
+                    pkg = _go_archive_to_pkg(archive)
+                    pkg_json_files.append(_make_pkg_json(ctx, archive, pkg))
+                    compiled_go_files.extend(archive.source.srcs)
+                    export_files.append(archive.data.export_file)
+
+    # If there was no stdlib json in any dependencies, fetch it from the
+    # current go_ node.
+    if not stdlib_json_file:
+        stdlib_json_file = ctx.attr._go_stdlib[GoStdLib]._list_json
 
     pkg_info = GoPkgInfo(
-        json = pkg_json_file,
         stdlib_json_file = stdlib_json_file,
         transitive_json_file = depset(
-            direct = [pkg_json_file] if pkg_json_file else [],
+            direct = pkg_json_files,
             transitive = deps_transitive_json_file,
         ),
         deps_transitive_json_file = depset(
             transitive = deps_transitive_json_file,
         ),
-        export_file = export_file,
+        transitive_compiled_go_files = depset(
+            direct = compiled_go_files,
+            transitive = deps_transitive_compiled_go_files,
+        ),
+        deps_transitive_compiled_go_files = depset(
+            transitive = deps_transitive_compiled_go_files,
+        ),
         transitive_export_file = depset(
-            direct = [export_file] if export_file else [],
+            direct = export_files,
             transitive = deps_transitive_export_file,
         ),
         deps_transitive_export_file = depset(
@@ -112,8 +145,9 @@ def _go_pkg_info_aspect_impl(target, ctx):
         pkg_info,
         OutputGroupInfo(
             go_pkg_driver_json_file = pkg_info.transitive_json_file,
+            go_pkg_driver_srcs = pkg_info.transitive_compiled_go_files,
             go_pkg_driver_export_file = pkg_info.transitive_export_file,
-            go_pkg_driver_stdlib_json_file = depset([pkg_info.stdlib_json_file] if pkg_info.stdlib_json_file else [])
+            go_pkg_driver_stdlib_json_file = depset([pkg_info.stdlib_json_file] if pkg_info.stdlib_json_file else []),
         ),
     ]
 

go/tools/gopackagesdriver/bazel.go

@@ -15,8 +15,11 @@
 package main
 
 import (
+	"bufio"
+	"bytes"
 	"context"
 	"encoding/json"
+	"errors"
 	"fmt"
 	"io/ioutil"
 	"net/url"
@@ -32,8 +35,8 @@ const (
 
 type Bazel struct {
 	bazelBin      string
-	execRoot      string
 	workspaceRoot string
+	info          map[string]string
 }
 
 // Minimal BEP structs to access the build outputs
@@ -51,22 +54,34 @@ func NewBazel(ctx context.Context, bazelBin, workspaceRoot string) (*Bazel, erro
 		bazelBin:      bazelBin,
 		workspaceRoot: workspaceRoot,
 	}
-	if execRoot, err := b.run(ctx, "info", "execution_root"); err != nil {
-		return nil, fmt.Errorf("unable to find execution root: %w", err)
-	} else {
-		b.execRoot = strings.TrimSpace(execRoot)
+	if err := b.fillInfo(ctx); err != nil {
+		return nil, fmt.Errorf("unable to query bazel info: %w", err)
 	}
 	return b, nil
 }
 
+func (b *Bazel) fillInfo(ctx context.Context) error {
+	b.info = map[string]string{}
+	output, err := b.run(ctx, "info")
+	if err != nil {
+		return err
+	}
+	scanner := bufio.NewScanner(bytes.NewBufferString(output))
+	for scanner.Scan() {
+		parts := strings.SplitN(strings.TrimSpace(scanner.Text()), ":", 2)
+		b.info[strings.TrimSpace(parts[0])] = strings.TrimSpace(parts[1])
+	}
+	return nil
+}
+
 func (b *Bazel) run(ctx context.Context, command string, args ...string) (string, error) {
 	cmd := exec.CommandContext(ctx, b.bazelBin, append([]string{
 		command,
 		"--tool_tag=" + toolTag,
 		"--ui_actions_shown=0",
 	}, args...)...)
 	fmt.Fprintln(os.Stderr, "Running:", cmd.Args)
-	cmd.Dir = b.workspaceRoot
+	cmd.Dir = b.WorkspaceRoot()
 	cmd.Stderr = os.Stderr
 	output, err := cmd.Output()
 	return string(output), err
@@ -88,7 +103,13 @@ func (b *Bazel) Build(ctx context.Context, args ...string) ([]string, error) {
 		"--build_event_json_file_path_conversion=no",
 	}, args...)
 	if _, err := b.run(ctx, "build", args...); err != nil {
-		return nil, fmt.Errorf("bazel build failed: %w", err)
+		// Ignore a regular build failure to get partial data.
+		// See https://docs.bazel.build/versions/main/guide.html#what-exit-code-will-i-get on
+		// exit codes.
+		var exerr *exec.ExitError
+		if !errors.As(err, &exerr) || exerr.ExitCode() != 1 {
+			return nil, fmt.Errorf("bazel build failed: %w", err)
+		}
 	}
 
 	files := make([]string, 0)
@@ -120,10 +141,22 @@ func (b *Bazel) Query(ctx context.Context, args ...string) ([]string, error) {
 	return strings.Split(strings.TrimSpace(output), "\n"), nil
 }
 
+func (b *Bazel) QueryLabels(ctx context.Context, args ...string) ([]string, error) {
+	output, err := b.run(ctx, "query", args...)
+	if err != nil {
+		return nil, fmt.Errorf("bazel query failed: %w", err)
+	}
+	return strings.Split(strings.TrimSpace(output), "\n"), nil
+}
+
 func (b *Bazel) WorkspaceRoot() string {
 	return b.workspaceRoot
 }
 
 func (b *Bazel) ExecutionRoot() string {
-	return b.execRoot
+	return b.info["execution_root"]
+}
+
+func (b *Bazel) OutputBase() string {
+	return b.info["output_base"]
 }