Make Go runfiles library repo mapping aware (#3347)

.bazelci/presubmit.yml

@@ -24,7 +24,7 @@ tasks:
   ubuntu2004_bcr_tests:
     name: BCR test module
     platform: ubuntu2004
-    bazel: b12f3a93a55019276879bd2d3edbd201c913675a
+    bazel: 6.0.0rc2
     working_directory: tests/bcr
     build_flags:
       - "--allow_yanked_versions=all"

go/BUILD.bazel

@@ -7,6 +7,7 @@ filegroup(
         "//go/config:all_files",
         "//go/constraints/amd64:all_files",
         "//go/platform:all_files",
+        "//go/runfiles:all_files",
         "//go/toolchain:all_files",
         "//go/tools:all_files",
         "//go/private:all_files",

go/runfiles/BUILD.bazel

@@ -32,3 +32,10 @@ alias(
     actual = ":runfiles",
     visibility = ["//visibility:public"],
 )
+
+filegroup(
+    name = "all_files",
+    testonly = True,
+    srcs = glob(["**"]),
+    visibility = ["//visibility:public"],
+)

go/runfiles/directory.go

@@ -21,8 +21,14 @@ import "path/filepath"
 // environmental variable RUNFILES_DIR.
 type Directory string
 
-func (d Directory) new() *Runfiles {
-	return &Runfiles{d, directoryVar + "=" + string(d)}
+func (d Directory) new(sourceRepo SourceRepo) (*Runfiles, error) {
+	r := &Runfiles{
+		impl:       d,
+		env:        directoryVar + "=" + string(d),
+		sourceRepo: string(sourceRepo),
+	}
+	err := r.loadRepoMapping()
+	return r, err
 }
 
 func (d Directory) path(s string) (string, error) {

go/runfiles/global.go

@@ -14,18 +14,26 @@
 
 package runfiles
 
-import "sync"
+import (
+	"regexp"
+	"runtime"
+	"sync"
+)
 
 // Rlocation returns the absolute path name of a runfile.  The runfile name must be
 // a relative path, using the slash (not backslash) as directory separator.  If
 // the runfiles manifest maps s to an empty name (indicating an empty runfile
 // not present in the filesystem), Rlocation returns an error that wraps ErrEmpty.
-func Rlocation(s string) (string, error) {
+func Rlocation(path string) (string, error) {
+	return RlocationFrom(path, CallerRepository())
+}
+
+func RlocationFrom(path string, sourceRepo string) (string, error) {
 	r, err := g.get()
 	if err != nil {
 		return "", err
 	}
-	return r.Rlocation(s)
+	return r.WithSourceRepo(sourceRepo).Rlocation(path)
 }
 
 // Env returns additional environmental variables to pass to subprocesses.
@@ -42,6 +50,35 @@ func Env() ([]string, error) {
 	return r.Env(), nil
 }
 
+var legacyExternalGeneratedFile = regexp.MustCompile(`^bazel-out[/][^/]+/bin/external/([^/]+)/`)
+var legacyExternalFile = regexp.MustCompile(`^external/([^/]+)/`)
+
+// CurrentRepository returns the canonical name of the Bazel repository that
+// contains the source file of the caller of CurrentRepository.
+func CurrentRepository() string {
+	return callerRepository(1)
+}
+
+// CallerRepository returns the canonical name of the Bazel repository that
+// contains the source file of the caller of the function that itself calls
+// CallerRepository.
+func CallerRepository() string {
+	return callerRepository(2)
+}
+
+func callerRepository(skip int) string {
+	_, file, _, _ := runtime.Caller(skip + 1)
+	if match := legacyExternalGeneratedFile.FindStringSubmatch(file); match != nil {
+		return match[1]
+	}
+	if match := legacyExternalFile.FindStringSubmatch(file); match != nil {
+		return match[1]
+	}
+	// If a file is not in an external repository, it is in the main repository,
+	// which has the empty string as its canonical name.
+	return ""
+}
+
 type global struct {
 	once     sync.Once
 	runfiles *Runfiles

go/runfiles/manifest.go

@@ -28,13 +28,18 @@ import (
 // environmental variable RUNFILES_MANIFEST_FILE.
 type ManifestFile string
 
-func (f ManifestFile) new() (*Runfiles, error) {
+func (f ManifestFile) new(sourceRepo SourceRepo) (*Runfiles, error) {
 	m, err := f.parse()
 	if err != nil {
 		return nil, err
 	}
-
-	return &Runfiles{m, manifestFileVar + "=" + string(f)}, nil
+	r := &Runfiles{
+		impl:       m,
+		env:        manifestFileVar + "=" + string(f),
+		sourceRepo: string(sourceRepo),
+	}
+	err = r.loadRepoMapping()
+	return r, err
 }
 
 type manifest map[string]string

go/runfiles/runfiles.go

@@ -36,6 +36,7 @@
 package runfiles
 
 import (
+	"bufio"
 	"errors"
 	"fmt"
 	"os"
@@ -48,17 +49,26 @@ const (
 	manifestFileVar = "RUNFILES_MANIFEST_FILE"
 )
 
+type repoMappingKey struct {
+	sourceRepo             string
+	targetRepoApparentName string
+}
+
 // Runfiles allows access to Bazel runfiles.  Use New to create Runfiles
 // objects; the zero Runfiles object always returns errors.  See
 // https://docs.bazel.build/skylark/rules.html#runfiles for some information on
 // Bazel runfiles.
 type Runfiles struct {
 	// We don’t need concurrency control since Runfiles objects are
 	// immutable once created.
-	impl runfiles
-	env  string
+	impl        runfiles
+	env         string
+	repoMapping map[repoMappingKey]string
+	sourceRepo  string
 }
 
+const noSourceRepoSentinel = "_not_a_valid_repository_name"
+
 // New creates a given Runfiles object.  By default, it uses os.Args and the
 // RUNFILES_MANIFEST_FILE and RUNFILES_DIR environmental variables to find the
 // runfiles location.  This can be overwritten by passing some options.
@@ -67,35 +77,40 @@ type Runfiles struct {
 // https://docs.google.com/document/d/e/2PACX-1vSDIrFnFvEYhKsCMdGdD40wZRBX3m3aZ5HhVj4CtHPmiXKDCxioTUbYsDydjKtFDAzER5eg7OjJWs3V/pub.
 func New(opts ...Option) (*Runfiles, error) {
 	var o options
+	o.sourceRepo = noSourceRepoSentinel
 	for _, a := range opts {
 		a.apply(&o)
 	}
 
+	if o.sourceRepo == noSourceRepoSentinel {
+		o.sourceRepo = SourceRepo(CallerRepository())
+	}
+
 	if o.manifest == "" {
 		o.manifest = ManifestFile(os.Getenv(manifestFileVar))
 	}
 	if o.manifest != "" {
-		return o.manifest.new()
+		return o.manifest.new(o.sourceRepo)
 	}
 
 	if o.directory == "" {
 		o.directory = Directory(os.Getenv(directoryVar))
 	}
 	if o.directory != "" {
-		return o.directory.new(), nil
+		return o.directory.new(o.sourceRepo)
 	}
 
 	if o.program == "" {
 		o.program = ProgramName(os.Args[0])
 	}
 	manifest := ManifestFile(o.program + ".runfiles_manifest")
 	if stat, err := os.Stat(string(manifest)); err == nil && stat.Mode().IsRegular() {
-		return manifest.new()
+		return manifest.new(o.sourceRepo)
 	}
 
 	dir := Directory(o.program + ".runfiles")
 	if stat, err := os.Stat(string(dir)); err == nil && stat.IsDir() {
-		return dir.new(), nil
+		return dir.new(o.sourceRepo)
 	}
 
 	return nil, errors.New("runfiles: no runfiles found")
@@ -132,7 +147,16 @@ func (r *Runfiles) Rlocation(path string) (string, error) {
 		return path, nil
 	}
 
-	p, err := r.impl.path(path)
+	mappedPath := path
+	split := strings.SplitN(path, "/", 2)
+	if len(split) == 2 {
+		key := repoMappingKey{r.sourceRepo, split[0]}
+		if targetRepoDirectory, exists := r.repoMapping[key]; exists {
+			mappedPath = targetRepoDirectory + "/" + split[1]
+		}
+	}
+
+	p, err := r.impl.path(mappedPath)
 	if err != nil {
 		return "", Error{path, err}
 	}
@@ -152,6 +176,20 @@ func isNormalizedPath(s string) error {
 	return nil
 }
 
+// loadRepoMapping loads the repo mapping (if it exists) using the impl.
+// This mutates the Runfiles object, but is idempotent.
+func (r *Runfiles) loadRepoMapping() error {
+	repoMappingPath, err := r.impl.path(repoMappingRlocation)
+	// If Bzlmod is disabled, the repository mapping manifest isn't created, so
+	// it is not an error if it is missing.
+	if err != nil {
+		return nil
+	}
+	r.repoMapping, err = parseRepoMapping(repoMappingPath)
+	// If the repository mapping manifest exists, it must be valid.
+	return err
+}
+
 // Env returns additional environmental variables to pass to subprocesses.
 // Each element is of the form “key=value”.  Pass these variables to
 // Bazel-built binaries so they can find their runfiles as well.  See the
@@ -166,15 +204,33 @@ func (r *Runfiles) Env() []string {
 	return []string{r.env}
 }
 
+// WithSourceRepo returns a Runfiles instance identical to the current one,
+// except that it uses the given repository's repository mapping when resolving
+// runfiles paths.
+func (r *Runfiles) WithSourceRepo(sourceRepo string) *Runfiles {
+	if r.sourceRepo == sourceRepo {
+		return r
+	}
+	clone := *r
+	clone.sourceRepo = sourceRepo
+	return &clone
+}
+
 // Option is an option for the New function to override runfiles discovery.
 type Option interface {
 	apply(*options)
 }
 
-// ProgramName is an Option that sets the program name.  If not set, New uses
+// ProgramName is an Option that sets the program name. If not set, New uses
 // os.Args[0].
 type ProgramName string
 
+// SourceRepo is an Option that sets the canonical name of the repository whose
+// repository mapping should be used to resolve runfiles paths. If not set, New
+// uses the repository containing the source file from which New is called.
+// Use CurrentRepository to get the name of the current repository.
+type SourceRepo string
+
 // Error represents a failure to look up a runfile.
 type Error struct {
 	// Runfile name that caused the failure.
@@ -197,15 +253,53 @@ func (e Error) Unwrap() error { return e.Err }
 var ErrEmpty = errors.New("empty runfile")
 
 type options struct {
-	program   ProgramName
-	manifest  ManifestFile
-	directory Directory
+	program    ProgramName
+	manifest   ManifestFile
+	directory  Directory
+	sourceRepo SourceRepo
 }
 
 func (p ProgramName) apply(o *options)  { o.program = p }
 func (m ManifestFile) apply(o *options) { o.manifest = m }
 func (d Directory) apply(o *options)    { o.directory = d }
+func (sr SourceRepo) apply(o *options)  { o.sourceRepo = sr }
 
 type runfiles interface {
 	path(string) (string, error)
 }
+
+// The runfiles root symlink under which the repository mapping can be found.
+// https://cs.opensource.google/bazel/bazel/+/1b073ac0a719a09c9b2d1a52680517ab22dc971e:src/main/java/com/google/devtools/build/lib/analysis/Runfiles.java;l=424
+const repoMappingRlocation = "_repo_mapping"
+
+// Parses a repository mapping manifest file emitted with Bzlmod enabled.
+func parseRepoMapping(path string) (map[repoMappingKey]string, error) {
+	r, err := os.Open(path)
+	if err != nil {
+		// The repo mapping manifest only exists with Bzlmod, so it's not an
+		// error if it's missing. Since any repository name not contained in the
+		// mapping is assumed to be already canonical, an empty map is
+		// equivalent to not applying any mapping.
+		return nil, nil
+	}
+	defer r.Close()
+
+	// Each line of the repository mapping manifest has the form:
+	// canonical name of source repo,apparent name of target repo,target repo runfiles directory
+	// https://cs.opensource.google/bazel/bazel/+/1b073ac0a719a09c9b2d1a52680517ab22dc971e:src/main/java/com/google/devtools/build/lib/analysis/RepoMappingManifestAction.java;l=117
+	s := bufio.NewScanner(r)
+	repoMapping := make(map[repoMappingKey]string)
+	for s.Scan() {
+		fields := strings.SplitN(s.Text(), ",", 3)
+		if len(fields) != 3 {
+			return nil, fmt.Errorf("runfiles: bad repo mapping line %q in file %s", s.Text(), path)
+		}
+		repoMapping[repoMappingKey{fields[0], fields[1]}] = fields[2]
+	}
+
+	if err = s.Err(); err != nil {
+		return nil, fmt.Errorf("runfiles: error parsing repo mapping file %s: %w", path, err)
+	}
+
+	return repoMapping, nil
+}

tests/bcr/.bazelversion

@@ -1 +1 @@
-last_green
+839ce7f5c40240d8b6f49c416c3769e226f43fee

tests/bcr/MODULE.bazel

@@ -10,6 +10,11 @@ local_path_override(
 )
 bazel_dep(name = "gazelle", version = "0.26.0")
 bazel_dep(name = "protobuf", version = "3.19.6")
+bazel_dep(name = "other_module", version = "")
+local_path_override(
+    module_name = "other_module",
+    path = "other_module",
+)
 
 # Test that this correctly downloads the SDK by requesting it from the commandline (see presubmit.yml).
 go_sdk = use_extension("@my_rules_go//go:extensions.bzl", "go_sdk")

tests/bcr/other_module/BUILD.bazel

@@ -0,0 +1 @@
+exports_files(["bar.txt"])

tests/bcr/other_module/MODULE.bazel

@@ -0,0 +1,3 @@
+module(name = "other_module")
+
+bazel_dep(name = "rules_go", version = "")

tests/bcr/other_module/bar.txt

@@ -0,0 +1 @@
+hello

tests/bcr/runfiles/BUILD.bazel

@@ -0,0 +1,9 @@
+load("@my_rules_go//go:def.bzl", "go_test")
+
+go_test(
+    name = "runfiles_test",
+    srcs = ["runfiles_test.go"],
+    args = ["$(rlocationpath @other_module//:bar.txt)"],
+    data = ["@other_module//:bar.txt"],
+    deps = ["@my_rules_go//go/runfiles"],
+)