cmd/go/internal/modfetch: fix Lookup, Import; add ImportRepoRev

- Explain Lookup and Import, module paths vs import paths.

- Add ImportRepoRev, because neither Lookup nor Import
  is appropriate for translating legacy versioning system configs.
  There's more to do in ImportRepoRev, but it's already more
  accurate than before.

- Drop the old ad-hoc logic about github/bitbucket/etc
  in favor of resolving source code hosts using the "go get" logic.
  This enables paths like git.apache.org/thrift.git/lib/go/thrift again.

- Structure $GOPATH/src/mod/cache a little more: make separate
  top-level directories mod/cache/download and mod/cache/vcs.

Fixes golang/go#23983.
Fixes golang/go#24687.
Fixes golang/go#25590.
Fixes golang/go#25654.

Change-Id: I0de2d75c5846a1c054079e1c67b9a1100ed161c8
Reviewed-on: https://go-review.googlesource.com/120042
Run-TryBot: Russ Cox <[email protected]>
TryBot-Result: Gobot Gobot <[email protected]>
Reviewed-by: Bryan C. Mills <[email protected]>

vendor/cmd/go/internal/modfetch/coderepo.go

@@ -293,7 +293,7 @@ func (r *codeRepo) legacyGoMod(rev, dir string) []byte {
 		if convert == nil {
 			continue
 		}
-		if err := ConvertLegacyConfig(mf, file, data); err != nil {
+		if err := ConvertLegacyConfig(mf, path.Join(r.codeRoot+"@"+rev, dir, file), data); err != nil {
 			continue
 		}
 		break

vendor/cmd/go/internal/modfetch/coderepo_test.go

@@ -243,8 +243,10 @@ var codeRepoTests = []struct {
 	},
 	{
 		// package in subdirectory - custom domain
-		path:    "golang.org/x/net/context",
-		lookerr: "module root is \"golang.org/x/net\"",
+		// In general we can't reject these definitively in Lookup,
+		// but gopkg.in is special.
+		path:    "gopkg.in/yaml.v2/abc",
+		lookerr: "invalid module path \"gopkg.in/yaml.v2/abc\"",
 	},
 	{
 		// package in subdirectory - github
@@ -329,13 +331,13 @@ func TestCodeRepo(t *testing.T) {
 	for _, tt := range codeRepoTests {
 		t.Run(strings.Replace(tt.path, "/", "_", -1)+"/"+tt.rev, func(t *testing.T) {
 			repo, err := Lookup(tt.path)
-			if err != nil {
-				if tt.lookerr != "" {
-					if err.Error() == tt.lookerr {
-						return
-					}
-					t.Errorf("Lookup(%q): %v, want error %q", tt.path, err, tt.lookerr)
+			if tt.lookerr != "" {
+				if err != nil && err.Error() == tt.lookerr {
+					return
 				}
+				t.Errorf("Lookup(%q): %v, want error %q", tt.path, err, tt.lookerr)
+			}
+			if err != nil {
 				t.Fatalf("Lookup(%q): %v", tt.path, err)
 			}
 			if tt.mpath == "" {
@@ -442,7 +444,8 @@ var importTests = []struct {
 	},
 	{
 		path: "golang.org/x/foo/bar",
-		err:  "unknown module golang.org/x/foo/bar: no go-import tags",
+		// TODO(rsc): This error comes from old go get and is terrible. Fix.
+		err: `unrecognized import path "golang.org/x/foo/bar" (parse https://golang.org/x/foo/bar?go-get=1: no go-import meta tags ())`,
 	},
 }
 
@@ -452,14 +455,14 @@ func TestImport(t *testing.T) {
 	for _, tt := range importTests {
 		t.Run(strings.Replace(tt.path, "/", "_", -1), func(t *testing.T) {
 			repo, info, err := Import(tt.path, nil)
-			if err != nil {
-				if tt.err != "" {
-					if err.Error() == tt.err {
-						return
-					}
-					t.Errorf("Import(%q): %v, want error %q", tt.path, err, tt.err)
+			if tt.err != "" {
+				if err != nil && err.Error() == tt.err {
+					return
 				}
-				t.Fatalf("Lookup(%q): %v", tt.path, err)
+				t.Fatalf("Import(%q): %v, want error %q", tt.path, err, tt.err)
+			}
+			if err != nil {
+				t.Fatalf("Import(%q): %v", tt.path, err)
 			}
 			if mpath := repo.ModulePath(); mpath != tt.mpath {
 				t.Errorf("repo.ModulePath() = %q (%v), want %q", mpath, info.Version, tt.mpath)

vendor/cmd/go/internal/modfetch/convert.go

@@ -33,28 +33,20 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error {
 	if convert == nil {
 		return fmt.Errorf("unknown legacy config file %s", file)
 	}
-	result, err := convert(file, data)
+	mf, err := convert(file, data)
 	if err != nil {
 		return fmt.Errorf("parsing %s: %v", file, err)
 	}
 
 	// Convert requirements block, which may use raw SHA1 hashes as versions,
 	// to valid semver requirement list, respecting major versions.
 	var work par.Work
-	for _, r := range result.Require {
+	for _, r := range mf.Require {
 		m := r.Mod
 		if m.Path == "" {
 			continue
 		}
-
-		// TODO: Something better here.
-		if strings.HasPrefix(m.Path, "github.com/") || strings.HasPrefix(m.Path, "golang.org/x/") {
-			f := strings.Split(m.Path, "/")
-			if len(f) > 3 {
-				m.Path = strings.Join(f[:3], "/")
-			}
-		}
-		work.Add(m)
+		work.Add(r.Mod)
 	}
 
 	var (
@@ -63,13 +55,14 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error {
 	)
 	work.Do(10, func(item interface{}) {
 		r := item.(module.Version)
-		info, err := Stat(r.Path, r.Version)
+		repo, info, err := ImportRepoRev(r.Path, r.Version)
 		if err != nil {
-			fmt.Fprintf(os.Stderr, "vgo: stat %s@%s: %v\n", r.Path, r.Version, err)
+			fmt.Fprintf(os.Stderr, "vgo: converting %s: stat %s@%s: %v\n", file, r.Path, r.Version, err)
 			return
 		}
 		mu.Lock()
-		need[r.Path] = semver.Max(need[r.Path], info.Version)
+		path := repo.ModulePath()
+		need[path] = semver.Max(need[path], info.Version)
 		mu.Unlock()
 	})
 
@@ -82,7 +75,7 @@ func ConvertLegacyConfig(f *modfile.File, file string, data []byte) error {
 		f.AddNewRequire(path, need[path])
 	}
 
-	for _, r := range result.Replace {
+	for _, r := range mf.Replace {
 		err := f.AddReplace(r.Old.Path, r.Old.Version, r.New.Path, r.New.Version)
 		if err != nil {
 			return fmt.Errorf("add replace: %v", err)