From 0cd4433fcc2df21311b2620a11849e7d85dac0dd Mon Sep 17 00:00:00 2001 From: Steeve Morin Date: Mon, 28 Jun 2021 21:19:25 +0200 Subject: [PATCH] go/tools: add gopackagesdriver (#2858) This commit introduces the GOPACKAGESDRIVER for rules_go Signed-off-by: Steeve Morin --- go/private/actions/stdlib.bzl | 20 +- go/tools/builders/BUILD.bazel | 1 + go/tools/builders/builder.go | 2 + go/tools/builders/stdlib.go | 2 +- go/tools/builders/stdliblist.go | 210 ++++++++++++++++++ go/tools/gopackagesdriver/BUILD.bazel | 22 ++ go/tools/gopackagesdriver/aspect.bzl | 128 +++++++++++ go/tools/gopackagesdriver/bazel.go | 129 +++++++++++ .../gopackagesdriver/bazel_json_builder.go | 105 +++++++++ go/tools/gopackagesdriver/driver_request.go | 88 ++++++++ go/tools/gopackagesdriver/flatpackage.go | 148 ++++++++++++ .../gopackagesdriver/json_packages_driver.go | 59 +++++ go/tools/gopackagesdriver/main.go | 121 ++++++++++ go/tools/gopackagesdriver/packageregistry.go | 130 +++++++++++ 14 files changed, 1162 insertions(+), 3 deletions(-) create mode 100644 go/tools/builders/stdliblist.go create mode 100644 go/tools/gopackagesdriver/BUILD.bazel create mode 100644 go/tools/gopackagesdriver/aspect.bzl create mode 100644 go/tools/gopackagesdriver/bazel.go create mode 100644 go/tools/gopackagesdriver/bazel_json_builder.go create mode 100644 go/tools/gopackagesdriver/driver_request.go create mode 100644 go/tools/gopackagesdriver/flatpackage.go create mode 100644 go/tools/gopackagesdriver/json_packages_driver.go create mode 100644 go/tools/gopackagesdriver/main.go create mode 100644 go/tools/gopackagesdriver/packageregistry.go diff --git a/go/private/actions/stdlib.bzl b/go/private/actions/stdlib.bzl index 24f8e5f74f..80f4011fa5 100644 --- a/go/private/actions/stdlib.bzl +++ b/go/private/actions/stdlib.bzl @@ -51,10 +51,25 @@ def _should_use_sdk_stdlib(go): not go.mode.pure and go.mode.link == LINKMODE_NORMAL) +def _build_stdlib_list_json(go): + out = go.declare_file(go, "stdlib.pkg.json") + args = go.builder_args(go, "stdliblist") + args.add("-out", out) + go.actions.run( + inputs = go.sdk_files, + outputs = [out], + mnemonic = "GoStdlibList", + executable = go.toolchain._builder, + arguments = [args], + env = go.env, + ) + return out + def _sdk_stdlib(go): return GoStdLib( - root_file = go.sdk.root_file, + _list_json = _build_stdlib_list_json(go), libs = go.sdk.libs, + root_file = go.sdk.root_file, ) def _build_stdlib(go): @@ -99,6 +114,7 @@ def _build_stdlib(go): env = env, ) return GoStdLib( - root_file = root_file, + _list_json = _build_stdlib_list_json(go), libs = [pkg], + root_file = root_file, ) diff --git a/go/tools/builders/BUILD.bazel b/go/tools/builders/BUILD.bazel index 4635dc4087..41e54fdb1c 100644 --- a/go/tools/builders/BUILD.bazel +++ b/go/tools/builders/BUILD.bazel @@ -35,6 +35,7 @@ filegroup( "read.go", "replicate.go", "stdlib.go", + "stdliblist.go", ] + select({ "@bazel_tools//src/conditions:windows": ["path_windows.go"], "//conditions:default": ["path.go"], diff --git a/go/tools/builders/builder.go b/go/tools/builders/builder.go index 905c04e697..8ccd746efb 100644 --- a/go/tools/builders/builder.go +++ b/go/tools/builders/builder.go @@ -59,6 +59,8 @@ func main() { action = pack case "stdlib": action = stdlib + case "stdliblist": + action = stdliblist default: log.Fatalf("unknown action: %s", verb) } diff --git a/go/tools/builders/stdlib.go b/go/tools/builders/stdlib.go index 6daa6883de..bd020095e8 100644 --- a/go/tools/builders/stdlib.go +++ b/go/tools/builders/stdlib.go @@ -112,7 +112,7 @@ You may need to use the flags --cpu=x64_windows --compiler=mingw-gcc.`) // we strip the build ids, since they won't be used after this. installArgs := goenv.goCmd("install", "-toolexec", abs(os.Args[0])+" filterbuildid") if len(build.Default.BuildTags) > 0 { - installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, " ")) + installArgs = append(installArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) } gcflags := []string{} diff --git a/go/tools/builders/stdliblist.go b/go/tools/builders/stdliblist.go new file mode 100644 index 0000000000..90564a2d3e --- /dev/null +++ b/go/tools/builders/stdliblist.go @@ -0,0 +1,210 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "bytes" + "encoding/json" + "flag" + "go/build" + "os" + "path/filepath" + "strings" +) + +// Copy and pasted from golang.org/x/tools/go/packages +type flatPackagesError struct { + Pos string // "file:line:col" or "file:line" or "" or "-" + Msg string + Kind flatPackagesErrorKind +} + +type flatPackagesErrorKind int + +const ( + UnknownError flatPackagesErrorKind = iota + ListError + ParseError + TypeError +) + +func (err flatPackagesError) Error() string { + pos := err.Pos + if pos == "" { + pos = "-" // like token.Position{}.String() + } + return pos + ": " + err.Msg +} + +// flatPackage is the JSON form of Package +// It drops all the type and syntax fields, and transforms the Imports +type flatPackage struct { + ID string + Name string `json:",omitempty"` + PkgPath string `json:",omitempty"` + Standard bool `json:",omitempty"` + Errors []flatPackagesError `json:",omitempty"` + GoFiles []string `json:",omitempty"` + CompiledGoFiles []string `json:",omitempty"` + OtherFiles []string `json:",omitempty"` + ExportFile string `json:",omitempty"` + Imports map[string]string `json:",omitempty"` +} + +type goListPackage struct { + Dir string // directory containing package sources + ImportPath string // import path of package in dir + Name string // package name + Target string // install path + Goroot bool // is this package in the Go root? + Standard bool // is this package part of the standard Go library? + Root string // Go root or Go path dir containing this package + Export string // file containing export data (when using -export) + // Source files + GoFiles []string // .go source files (excluding CgoFiles, TestGoFiles, XTestGoFiles) + CgoFiles []string // .go source files that import "C" + CompiledGoFiles []string // .go files presented to compiler (when using -compiled) + IgnoredGoFiles []string // .go source files ignored due to build constraints + IgnoredOtherFiles []string // non-.go source files ignored due to build constraints + CFiles []string // .c source files + CXXFiles []string // .cc, .cxx and .cpp source files + MFiles []string // .m source files + HFiles []string // .h, .hh, .hpp and .hxx source files + FFiles []string // .f, .F, .for and .f90 Fortran source files + SFiles []string // .s source files + SwigFiles []string // .swig files + SwigCXXFiles []string // .swigcxx files + SysoFiles []string // .syso object files to add to archive + TestGoFiles []string // _test.go files in package + XTestGoFiles []string // _test.go files outside package + // Embedded files + EmbedPatterns []string // //go:embed patterns + EmbedFiles []string // files matched by EmbedPatterns + TestEmbedPatterns []string // //go:embed patterns in TestGoFiles + TestEmbedFiles []string // files matched by TestEmbedPatterns + XTestEmbedPatterns []string // //go:embed patterns in XTestGoFiles + XTestEmbedFiles []string // files matched by XTestEmbedPatterns + // Dependency information + Imports []string // import paths used by this package + ImportMap map[string]string // map from source import to ImportPath (identity entries omitted) + // Error information + Incomplete bool // this package or a dependency has an error + Error *flatPackagesError // error loading package + DepsErrors []*flatPackagesError // errors loading dependencies +} + +func stdlibPackageID(importPath string) string { + return "@io_bazel_rules_go//stdlib:" + importPath +} + +func execRootPath(execRoot, p string) string { + dir, _ := filepath.Rel(execRoot, p) + return filepath.Join("__BAZEL_EXECROOT__", dir) +} + +func absoluteSourcesPaths(execRoot, pkgDir string, srcs []string) []string { + ret := make([]string, 0, len(srcs)) + pkgDir = execRootPath(execRoot, pkgDir) + for _, src := range srcs { + ret = append(ret, filepath.Join(pkgDir, src)) + } + return ret +} + +func flatPackageForStd(execRoot string, pkg *goListPackage) *flatPackage { + // Don't use generated files from the stdlib + goFiles := absoluteSourcesPaths(execRoot, pkg.Dir, pkg.GoFiles) + + newPkg := &flatPackage{ + ID: stdlibPackageID(pkg.ImportPath), + Name: pkg.Name, + PkgPath: pkg.ImportPath, + ExportFile: execRootPath(execRoot, pkg.Target), + Imports: map[string]string{}, + Standard: pkg.Standard, + GoFiles: goFiles, + CompiledGoFiles: goFiles, + } + for _, imp := range pkg.Imports { + newPkg.Imports[imp] = stdlibPackageID(imp) + } + // We don't support CGo for now + delete(newPkg.Imports, "C") + return newPkg +} + +// stdliblist runs `go list -json` on the standard library and saves it to a file. +func stdliblist(args []string) error { + // process the args + flags := flag.NewFlagSet("stdliblist", flag.ExitOnError) + goenv := envFlags(flags) + out := flags.String("out", "", "Path to output go list json") + if err := flags.Parse(args); err != nil { + return err + } + if err := goenv.checkFlags(); err != nil { + return err + } + + // Ensure paths are absolute. + absPaths := []string{} + for _, path := range filepath.SplitList(os.Getenv("PATH")) { + absPaths = append(absPaths, abs(path)) + } + os.Setenv("PATH", strings.Join(absPaths, string(os.PathListSeparator))) + os.Setenv("GOROOT", abs(os.Getenv("GOROOT"))) + // Make sure we have an absolute path to the C compiler. + // TODO(#1357): also take absolute paths of includes and other paths in flags. + os.Setenv("CC", abs(os.Getenv("CC"))) + + execRoot := abs(".") + + cachePath := abs(*out + ".gocache") + defer os.RemoveAll(cachePath) + os.Setenv("GOCACHE", cachePath) + os.Setenv("GOMODCACHE", cachePath) + os.Setenv("GOPATH", cachePath) + + listArgs := goenv.goCmd("list") + if len(build.Default.BuildTags) > 0 { + listArgs = append(listArgs, "-tags", strings.Join(build.Default.BuildTags, ",")) + } + listArgs = append(listArgs, "-json", "builtin", "std", "runtime/cgo") + + jsonFile, err := os.Create(*out) + if err != nil { + return err + } + defer jsonFile.Close() + + jsonData := &bytes.Buffer{} + if err := goenv.runCommandToFile(jsonData, listArgs); err != nil { + return err + } + + encoder := json.NewEncoder(jsonFile) + decoder := json.NewDecoder(jsonData) + for decoder.More() { + var pkg *goListPackage + if err := decoder.Decode(&pkg); err != nil { + return err + } + if err := encoder.Encode(flatPackageForStd(execRoot, pkg)); err != nil { + return err + } + } + + return nil +} diff --git a/go/tools/gopackagesdriver/BUILD.bazel b/go/tools/gopackagesdriver/BUILD.bazel new file mode 100644 index 0000000000..5c7ee3369b --- /dev/null +++ b/go/tools/gopackagesdriver/BUILD.bazel @@ -0,0 +1,22 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_binary", "go_library") + +go_library( + name = "gopackagesdriver_lib", + srcs = [ + "bazel.go", + "bazel_json_builder.go", + "driver_request.go", + "flatpackage.go", + "json_packages_driver.go", + "main.go", + "packageregistry.go", + ], + importpath = "github.com/bazelbuild/rules_go/go/tools/gopackagesdriver", + visibility = ["//visibility:private"], +) + +go_binary( + name = "gopackagesdriver", + embed = [":gopackagesdriver_lib"], + visibility = ["//visibility:public"], +) diff --git a/go/tools/gopackagesdriver/aspect.bzl b/go/tools/gopackagesdriver/aspect.bzl new file mode 100644 index 0000000000..3a15d96b9b --- /dev/null +++ b/go/tools/gopackagesdriver/aspect.bzl @@ -0,0 +1,128 @@ +# Copyright 2021 The Bazel Go Rules Authors. All rights reserved. +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +load( + "//go/private:providers.bzl", + "GoArchive", + "GoStdLib", +) +load( + "//go/private:context.bzl", + "go_context", +) +load( + "@bazel_skylib//lib:paths.bzl", + "paths", +) +load( + "@bazel_skylib//lib:collections.bzl", + "collections", +) + +GoPkgInfo = provider() + +def _is_file_external(f): + return f.owner.workspace_root != "" + +def _file_path(f): + if f.is_source and not _is_file_external(f): + return paths.join("__BAZEL_WORKSPACE__", f.path) + return paths.join("__BAZEL_EXECROOT__", f.path) + +def _go_pkg_info_aspect_impl(target, ctx): + # Fetch the stdlib JSON file from the inner most target + stdlib_json_file = None + + deps_transitive_json_file = [] + deps_transitive_export_file = [] + for dep in getattr(ctx.rule.attr, "deps", []): + if GoPkgInfo in dep: + pkg_info = dep[GoPkgInfo] + deps_transitive_json_file.append(pkg_info.transitive_json_file) + deps_transitive_export_file.append(pkg_info.transitive_export_file) + # Fetch the stdlib json from the first dependency + if not stdlib_json_file: + stdlib_json_file = pkg_info.stdlib_json_file + + # If deps are embedded, do not gather their json or export_file since they + # are included in the current target, but do gather their deps'. + for dep in getattr(ctx.rule.attr, "embed", []): + if GoPkgInfo in dep: + pkg_info = dep[GoPkgInfo] + deps_transitive_json_file.append(pkg_info.deps_transitive_json_file) + deps_transitive_export_file.append(pkg_info.deps_transitive_export_file) + + pkg_json_file = None + export_file = None + if GoArchive in target: + archive = target[GoArchive] + export_file = archive.data.export_file + pkg = struct( + ID = str(archive.data.label), + PkgPath = archive.data.importpath, + ExportFile = _file_path(archive.data.export_file), + GoFiles = [ + _file_path(src) + for src in archive.data.orig_srcs + ], + CompiledGoFiles = [ + _file_path(src) + for src in archive.data.srcs + ], + ) + pkg_json_file = ctx.actions.declare_file(archive.data.name + ".pkg.json") + ctx.actions.write(pkg_json_file, content = pkg.to_json()) + # If there was no stdlib json in any dependencies, fetch it from the + # current go_ node. + if not stdlib_json_file: + stdlib_json_file = ctx.attr._go_stdlib[GoStdLib]._list_json + + pkg_info = GoPkgInfo( + json = pkg_json_file, + stdlib_json_file = stdlib_json_file, + transitive_json_file = depset( + direct = [pkg_json_file] if pkg_json_file else [], + transitive = deps_transitive_json_file, + ), + deps_transitive_json_file = depset( + transitive = deps_transitive_json_file, + ), + export_file = export_file, + transitive_export_file = depset( + direct = [export_file] if export_file else [], + transitive = deps_transitive_export_file, + ), + deps_transitive_export_file = depset( + transitive = deps_transitive_export_file, + ), + ) + + return [ + pkg_info, + OutputGroupInfo( + go_pkg_driver_json_file = pkg_info.transitive_json_file, + go_pkg_driver_export_file = pkg_info.transitive_export_file, + go_pkg_driver_stdlib_json_file = depset([pkg_info.stdlib_json_file] if pkg_info.stdlib_json_file else []) + ), + ] + +go_pkg_info_aspect = aspect( + implementation = _go_pkg_info_aspect_impl, + attr_aspects = ["embed", "deps"], + attrs = { + "_go_stdlib": attr.label( + default = "@io_bazel_rules_go//:stdlib", + ), + }, +) diff --git a/go/tools/gopackagesdriver/bazel.go b/go/tools/gopackagesdriver/bazel.go new file mode 100644 index 0000000000..772a86c0a2 --- /dev/null +++ b/go/tools/gopackagesdriver/bazel.go @@ -0,0 +1,129 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "fmt" + "io/ioutil" + "net/url" + "os" + "os/exec" + "path/filepath" + "strings" +) + +const ( + toolTag = "gopackagesdriver" +) + +type Bazel struct { + bazelBin string + execRoot string + workspaceRoot string +} + +// Minimal BEP structs to access the build outputs +type BEPNamedSet struct { + NamedSetOfFiles *struct { + Files []struct { + Name string `json:"name"` + URI string `json:"uri"` + } `json:"files"` + } `json:"namedSetOfFiles"` +} + +func NewBazel(ctx context.Context, bazelBin, workspaceRoot string) (*Bazel, error) { + b := &Bazel{ + bazelBin: bazelBin, + workspaceRoot: workspaceRoot, + } + if execRoot, err := b.run(ctx, "info", "execution_root"); err != nil { + return nil, fmt.Errorf("unable to find execution root: %w", err) + } else { + b.execRoot = strings.TrimSpace(execRoot) + } + return b, nil +} + +func (b *Bazel) run(ctx context.Context, command string, args ...string) (string, error) { + cmd := exec.CommandContext(ctx, b.bazelBin, append([]string{ + command, + "--tool_tag=" + toolTag, + "--ui_actions_shown=0", + }, args...)...) + fmt.Fprintln(os.Stderr, "Running:", cmd.Args) + cmd.Dir = b.workspaceRoot + cmd.Stderr = os.Stderr + output, err := cmd.Output() + return string(output), err +} + +func (b *Bazel) Build(ctx context.Context, args ...string) ([]string, error) { + jsonFile, err := ioutil.TempFile("", "gopackagesdriver_bep_") + if err != nil { + return nil, fmt.Errorf("unable to create BEP JSON file: %w", err) + } + defer func() { + jsonFile.Close() + os.Remove(jsonFile.Name()) + }() + + args = append([]string{ + "--show_result=0", + "--build_event_json_file=" + jsonFile.Name(), + "--build_event_json_file_path_conversion=no", + }, args...) + if _, err := b.run(ctx, "build", args...); err != nil { + return nil, fmt.Errorf("bazel build failed: %w", err) + } + + files := make([]string, 0) + decoder := json.NewDecoder(jsonFile) + for decoder.More() { + var namedSet BEPNamedSet + if err := decoder.Decode(&namedSet); err != nil { + return nil, fmt.Errorf("unable to decode %s: %w", jsonFile.Name(), err) + } + if namedSet.NamedSetOfFiles != nil { + for _, f := range namedSet.NamedSetOfFiles.Files { + fileUrl, err := url.Parse(f.URI) + if err != nil { + return nil, fmt.Errorf("unable to parse file URI: %w", err) + } + files = append(files, filepath.FromSlash(fileUrl.Path)) + } + } + } + + return files, nil +} + +func (b *Bazel) Query(ctx context.Context, args ...string) ([]string, error) { + output, err := b.run(ctx, "query", args...) + if err != nil { + return nil, fmt.Errorf("bazel query failed: %w", err) + } + return strings.Split(strings.TrimSpace(output), "\n"), nil +} + +func (b *Bazel) WorkspaceRoot() string { + return b.workspaceRoot +} + +func (b *Bazel) ExecutionRoot() string { + return b.execRoot +} diff --git a/go/tools/gopackagesdriver/bazel_json_builder.go b/go/tools/gopackagesdriver/bazel_json_builder.go new file mode 100644 index 0000000000..0950a45f72 --- /dev/null +++ b/go/tools/gopackagesdriver/bazel_json_builder.go @@ -0,0 +1,105 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "fmt" + "strings" +) + +type BazelJSONBuilder struct { + bazel *Bazel + query string + tagFilters string + targets []string +} + +const ( + OutputGroupDriverJSONFile = "go_pkg_driver_json_file" + OutputGroupStdLibJSONFile = "go_pkg_driver_stdlib_json_file" + OutputGroupExportFile = "go_pkg_driver_export_file" +) + +func NewBazelJSONBuilder(bazel *Bazel, query, tagFilters string, targets []string) (*BazelJSONBuilder, error) { + return &BazelJSONBuilder{ + bazel: bazel, + query: query, + tagFilters: tagFilters, + targets: targets, + }, nil +} + +func (b *BazelJSONBuilder) outputGroupsForMode(mode LoadMode) string { + og := OutputGroupDriverJSONFile + "," + OutputGroupStdLibJSONFile + if mode&NeedExportsFile != 0 || true { // override for now + og += "," + OutputGroupExportFile + } + return og +} + +func (b *BazelJSONBuilder) Build(ctx context.Context, mode LoadMode) ([]string, error) { + buildsArgs := []string{ + "--aspects=@io_bazel_rules_go//go/tools/gopackagesdriver:aspect.bzl%go_pkg_info_aspect", + "--output_groups=" + b.outputGroupsForMode(mode), + "--keep_going", // Build all possible packages + } + + if b.tagFilters != "" { + buildsArgs = append(buildsArgs, "--build_tag_filters="+b.tagFilters) + } + + if b.query != "" { + queryTargets, err := b.bazel.Query( + ctx, + "--order_output=no", + "--output=label", + "--experimental_graphless_query", + "--nodep_deps", + "--noimplicit_deps", + "--notool_deps", + b.query, + ) + if err != nil { + return nil, fmt.Errorf("unable to query %v: %w", b.query, err) + } + buildsArgs = append(buildsArgs, queryTargets...) + } + + buildsArgs = append(buildsArgs, b.targets...) + + files, err := b.bazel.Build(ctx, buildsArgs...) + if err != nil { + return nil, fmt.Errorf("unable to bazel build %v: %w", buildsArgs, err) + } + + ret := []string{} + for _, f := range files { + if !strings.HasSuffix(f, ".pkg.json") { + continue + } + ret = append(ret, f) + } + + return ret, nil +} + +func (b *BazelJSONBuilder) PathResolver() PathResolverFunc { + return func(p string) string { + p = strings.Replace(p, "__BAZEL_EXECROOT__", b.bazel.ExecutionRoot(), 1) + p = strings.Replace(p, "__BAZEL_WORKSPACE__", b.bazel.WorkspaceRoot(), 1) + return p + } +} diff --git a/go/tools/gopackagesdriver/driver_request.go b/go/tools/gopackagesdriver/driver_request.go new file mode 100644 index 0000000000..8fe2f63e22 --- /dev/null +++ b/go/tools/gopackagesdriver/driver_request.go @@ -0,0 +1,88 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "io" +) + +// From https://pkg.go.dev/golang.org/x/tools/go/packages#LoadMode +type LoadMode int + +// Only NeedExportsFile is needed in our case +const ( + // NeedName adds Name and PkgPath. + NeedName LoadMode = 1 << iota + + // NeedFiles adds GoFiles and OtherFiles. + NeedFiles + + // NeedCompiledGoFiles adds CompiledGoFiles. + NeedCompiledGoFiles + + // NeedImports adds Imports. If NeedDeps is not set, the Imports field will contain + // "placeholder" Packages with only the ID set. + NeedImports + + // NeedDeps adds the fields requested by the LoadMode in the packages in Imports. + NeedDeps + + // NeedExportsFile adds ExportFile. + NeedExportsFile + + // NeedTypes adds Types, Fset, and IllTyped. + NeedTypes + + // NeedSyntax adds Syntax. + NeedSyntax + + // NeedTypesInfo adds TypesInfo. + NeedTypesInfo + + // NeedTypesSizes adds TypesSizes. + NeedTypesSizes + + // typecheckCgo enables full support for type checking cgo. Requires Go 1.15+. + // Modifies CompiledGoFiles and Types, and has no effect on its own. + typecheckCgo + + // NeedModule adds Module. + NeedModule +) + +// From https://github.com/golang/tools/blob/v0.1.0/go/packages/external.go#L32 +// Most fields are disabled since there are no needs for them +type DriverRequest struct { + Mode LoadMode `json:"mode"` + // Env specifies the environment the underlying build system should be run in. + // Env []string `json:"env"` + // BuildFlags are flags that should be passed to the underlying build system. + // BuildFlags []string `json:"build_flags"` + // Tests specifies whether the patterns should also return test packages. + // Tests bool `json:"tests"` + // Overlay maps file paths (relative to the driver's working directory) to the byte contents + // of overlay files. + // Overlay map[string][]byte `json:"overlay"` +} + +func ReadDriverRequest(r io.Reader) (*DriverRequest, error) { + req := &DriverRequest{} + if err := json.NewDecoder(r).Decode(&req); err != nil { + return nil, fmt.Errorf("unable to decode driver request: %w", err) + } + return req, nil +} diff --git a/go/tools/gopackagesdriver/flatpackage.go b/go/tools/gopackagesdriver/flatpackage.go new file mode 100644 index 0000000000..b0fbe0ff36 --- /dev/null +++ b/go/tools/gopackagesdriver/flatpackage.go @@ -0,0 +1,148 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "encoding/json" + "fmt" + "go/parser" + "go/token" + "os" + "strconv" + "strings" +) + +type ResolvePkgFunc func(importPath string) *FlatPackage + +// Copy and pasted from golang.org/x/tools/go/packages +type FlatPackagesError struct { + Pos string // "file:line:col" or "file:line" or "" or "-" + Msg string + Kind FlatPackagesErrorKind +} + +type FlatPackagesErrorKind int + +const ( + UnknownError FlatPackagesErrorKind = iota + ListError + ParseError + TypeError +) + +func (err FlatPackagesError) Error() string { + pos := err.Pos + if pos == "" { + pos = "-" // like token.Position{}.String() + } + return pos + ": " + err.Msg +} + +// FlatPackage is the JSON form of Package +// It drops all the type and syntax fields, and transforms the Imports +type FlatPackage struct { + ID string + Name string `json:",omitempty"` + PkgPath string `json:",omitempty"` + Errors []FlatPackagesError `json:",omitempty"` + GoFiles []string `json:",omitempty"` + CompiledGoFiles []string `json:",omitempty"` + OtherFiles []string `json:",omitempty"` + ExportFile string `json:",omitempty"` + Imports map[string]string `json:",omitempty"` + Standard bool `json:",omitempty"` +} + +type PackageFunc func(pkg *FlatPackage) +type PathResolverFunc func(path string) string + +func resolvePathsInPlace(prf PathResolverFunc, paths []string) { + for i, path := range paths { + paths[i] = prf(path) + } +} + +func WalkFlatPackagesFromJSON(jsonFile string, onPkg PackageFunc) error { + f, err := os.Open(jsonFile) + if err != nil { + return fmt.Errorf("unable to open package JSON file: %w", err) + } + defer f.Close() + + decoder := json.NewDecoder(f) + for decoder.More() { + pkg := &FlatPackage{} + if err := decoder.Decode(&pkg); err != nil { + return fmt.Errorf("unable to decode package in %s: %w", f.Name(), err) + } + onPkg(pkg) + } + return nil +} + +func (fp *FlatPackage) ResolvePaths(prf PathResolverFunc) error { + resolvePathsInPlace(prf, fp.CompiledGoFiles) + resolvePathsInPlace(prf, fp.GoFiles) + resolvePathsInPlace(prf, fp.OtherFiles) + fp.ExportFile = prf(fp.ExportFile) + return nil +} + +func (fp *FlatPackage) IsStdlib() bool { + return fp.Standard +} + +func (fp *FlatPackage) ResolveImports(resolve ResolvePkgFunc) { + // Stdlib packages are already complete import wise + if fp.IsStdlib() { + return + } + + fset := token.NewFileSet() + + for _, file := range fp.CompiledGoFiles { + f, err := parser.ParseFile(fset, file, nil, parser.ImportsOnly) + if err != nil { + continue + } + // If the name is not provided, fetch it from the sources + if fp.Name == "" { + fp.Name = f.Name.Name + } + for _, rawImport := range f.Imports { + imp, err := strconv.Unquote(rawImport.Path.Value) + if err != nil { + continue + } + // We don't handle CGo for now + if imp == "C" { + continue + } + if _, ok := fp.Imports[imp]; ok { + continue + } + if pkg := resolve(imp); pkg != nil { + if fp.Imports == nil { + fp.Imports = map[string]string{} + } + fp.Imports[imp] = pkg.ID + } + } + } +} + +func (fp *FlatPackage) IsRoot() bool { + return strings.HasPrefix(fp.ID, "//") +} diff --git a/go/tools/gopackagesdriver/json_packages_driver.go b/go/tools/gopackagesdriver/json_packages_driver.go new file mode 100644 index 0000000000..a2cd0dcece --- /dev/null +++ b/go/tools/gopackagesdriver/json_packages_driver.go @@ -0,0 +1,59 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "fmt" + "go/types" +) + +type JSONPackagesDriver struct { + registry *PackageRegistry +} + +func NewJSONPackagesDriver(jsonFiles []string, prf PathResolverFunc) (*JSONPackagesDriver, error) { + jpd := &JSONPackagesDriver{ + registry: NewPackageRegistry(), + } + + for _, f := range jsonFiles { + if err := WalkFlatPackagesFromJSON(f, func(pkg *FlatPackage) { + jpd.registry.Add(pkg) + }); err != nil { + return nil, fmt.Errorf("unable to walk json: %w", err) + } + } + + if err := jpd.registry.ResolvePaths(prf); err != nil { + return nil, fmt.Errorf("unable to resolve paths: %w", err) + } + + if err := jpd.registry.ResolveImports(); err != nil { + return nil, fmt.Errorf("unable to resolve paths: %w", err) + } + + return jpd, nil +} + +func (b *JSONPackagesDriver) Match(pattern ...string) *driverResponse { + rootPkgs, packages := b.registry.Match(pattern...) + + return &driverResponse{ + NotHandled: false, + Sizes: types.SizesFor("gc", "amd64").(*types.StdSizes), + Roots: rootPkgs, + Packages: packages, + } +} diff --git a/go/tools/gopackagesdriver/main.go b/go/tools/gopackagesdriver/main.go new file mode 100644 index 0000000000..954c084dab --- /dev/null +++ b/go/tools/gopackagesdriver/main.go @@ -0,0 +1,121 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "context" + "encoding/json" + "fmt" + "go/types" + "os" + "os/signal" + "strings" +) + +type driverResponse struct { + NotHandled bool + + // Sizes, if not nil, is the types.Sizes to use when type checking. + Sizes *types.StdSizes + + // Roots is the set of package IDs that make up the root packages. + // We have to encode this separately because when we encode a single package + // we cannot know if it is one of the roots as that requires knowledge of the + // graph it is part of. + Roots []string `json:",omitempty"` + + // Packages is the full set of packages in the graph. + // The packages are not connected into a graph. + // The Imports if populated will be stubs that only have their ID set. + // Imports will be connected and then type and syntax information added in a + // later pass (see refine). + Packages []*FlatPackage +} + +var ( + bazelBin = getenvDefault("GOPACKAGESDRIVER_BAZEL", "bazel") + workspaceRoot = os.Getenv("BUILD_WORKSPACE_DIRECTORY") + targets = strings.Fields(os.Getenv("GOPACKAGESDRIVER_BAZEL_TARGETS")) + targetsQueryStr = os.Getenv("GOPACKAGESDRIVER_BAZEL_QUERY") + targetsTagFilters = os.Getenv("GOPACKAGESDRIVER_BAZEL_TAG_FILTERS") +) + +func getenvDefault(key, defaultValue string) string { + if v, ok := os.LookupEnv(key); ok { + return v + } + return defaultValue +} + +func signalContext(parentCtx context.Context, signals ...os.Signal) (ctx context.Context, stop context.CancelFunc) { + ctx, cancel := context.WithCancel(parentCtx) + ch := make(chan os.Signal, 1) + go func() { + select { + case <-ch: + cancel() + case <-ctx.Done(): + } + }() + signal.Notify(ch, signals...) + + return ctx, cancel +} + +func run() error { + ctx, cancel := signalContext(context.Background(), os.Interrupt) + defer cancel() + + request, err := ReadDriverRequest(os.Stdin) + if err != nil { + return fmt.Errorf("unable to read request: %w", err) + } + + bazel, err := NewBazel(ctx, bazelBin, workspaceRoot) + if err != nil { + return fmt.Errorf("unable to create bazel instance: %w", err) + } + + bazelJsonBuilder, err := NewBazelJSONBuilder(bazel, targetsQueryStr, targetsTagFilters, targets) + if err != nil { + return fmt.Errorf("unable to build JSON files: %w", err) + } + + jsonFiles, err := bazelJsonBuilder.Build(ctx, request.Mode) + if err != nil { + return fmt.Errorf("unable to build JSON files: %w", err) + } + + driver, err := NewJSONPackagesDriver(jsonFiles, bazelJsonBuilder.PathResolver()) + if err != nil { + return fmt.Errorf("unable to load JSON files: %w", err) + } + + response := driver.Match(os.Args[1:]...) + // return nil + + if err := json.NewEncoder(os.Stdout).Encode(response); err != nil { + return fmt.Errorf("unable to encode response: %w", err) + } + + return nil +} + +func main() { + if err := run(); err != nil { + fmt.Fprintf(os.Stderr, "error: %w", err) + os.Exit(1) + } +} diff --git a/go/tools/gopackagesdriver/packageregistry.go b/go/tools/gopackagesdriver/packageregistry.go new file mode 100644 index 0000000000..cd979d84d5 --- /dev/null +++ b/go/tools/gopackagesdriver/packageregistry.go @@ -0,0 +1,130 @@ +// Copyright 2021 The Bazel Authors. All rights reserved. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package main + +import ( + "strings" +) + +type PackageRegistry struct { + packagesByID map[string]*FlatPackage + packagesByImportPath map[string]*FlatPackage + packagesByFile map[string]*FlatPackage +} + +func NewPackageRegistry(pkgs ...*FlatPackage) *PackageRegistry { + pr := &PackageRegistry{ + packagesByID: map[string]*FlatPackage{}, + packagesByImportPath: map[string]*FlatPackage{}, + packagesByFile: map[string]*FlatPackage{}, + } + pr.Add(pkgs...) + return pr +} + +func (pr *PackageRegistry) Add(pkgs ...*FlatPackage) *PackageRegistry { + for _, pkg := range pkgs { + pr.packagesByID[pkg.ID] = pkg + pr.packagesByImportPath[pkg.PkgPath] = pkg + } + return pr +} + +func (pr *PackageRegistry) FromPkgPath(pkgPath string) *FlatPackage { + return pr.packagesByImportPath[pkgPath] +} + +func (pr *PackageRegistry) Remove(pkgs ...*FlatPackage) *PackageRegistry { + for _, pkg := range pkgs { + delete(pr.packagesByImportPath, pkg.PkgPath) + } + return pr +} + +func (pr *PackageRegistry) ResolvePaths(prf PathResolverFunc) error { + for _, pkg := range pr.packagesByImportPath { + pkg.ResolvePaths(prf) + for _, f := range pkg.CompiledGoFiles { + pr.packagesByFile[f] = pkg + } + } + return nil +} + +func (pr *PackageRegistry) ResolveImports() error { + for _, pkg := range pr.packagesByImportPath { + pkg.ResolveImports(func(importPath string) *FlatPackage { + return pr.FromPkgPath(importPath) + }) + } + return nil +} + +func (pr *PackageRegistry) walk(acc map[string]*FlatPackage, root string) { + pkg := pr.packagesByID[root] + acc[pkg.ID] = pkg + for _, pkgID := range pkg.Imports { + if _, ok := acc[pkgID]; !ok { + pr.walk(acc, pkgID) + } + } +} + +func (pr *PackageRegistry) Match(patterns ...string) ([]string, []*FlatPackage) { + roots := map[string]struct{}{} + wildcard := false + + for _, pattern := range patterns { + if strings.HasPrefix(pattern, "file=") { + f := strings.TrimPrefix(pattern, "file=") + if pkg, ok := pr.packagesByFile[f]; ok { + roots[pkg.ID] = struct{}{} + } + } else if pattern == "." || pattern == "./..." { + wildcard = true + } else { + if pkg, ok := pr.packagesByImportPath[pattern]; ok { + roots[pkg.ID] = struct{}{} + } + } + } + + if wildcard { + retPkgs := make([]*FlatPackage, 0, len(pr.packagesByImportPath)) + retRoots := make([]string, 0, len(pr.packagesByImportPath)) + for _, pkg := range pr.packagesByImportPath { + if strings.HasPrefix(pkg.ID, "//") { + retRoots = append(retRoots, pkg.ID) + roots[pkg.ID] = struct{}{} + } + retPkgs = append(retPkgs, pkg) + } + return retRoots, retPkgs + } + + walkedPackages := map[string]*FlatPackage{} + retRoots := make([]string, 0, len(roots)) + for rootPkg := range roots { + retRoots = append(retRoots, rootPkg) + pr.walk(walkedPackages, rootPkg) + } + + retPkgs := make([]*FlatPackage, 0, len(walkedPackages)) + for _, pkg := range walkedPackages { + retPkgs = append(retPkgs, pkg) + } + + return retRoots, retPkgs +}