cmd/go: declare net hosts in script tests

Although we aren't precise about enforcing the hosts just yet,
we can eventually use the declared hostnames to selectively skip
tests (for example, if an external service has an outage while
a Go release is being tested).

Also relax the constraint to [short] in tests that require only
vcs-test.golang.org, which has redirected to an in-process server
since around CL 427914.

Also enforce that tests that use the network actually use the [net]
constraint, by setting TESTGONETWORK=panic in the test environment
until the condition is evaluated.

For #52545.
For #54503.
Updates #27494.

Change-Id: I13be6b42a9beee97657eb45424882e787ac164c3
Reviewed-on: https://go-review.googlesource.com/c/go/+/473276
Run-TryBot: Bryan Mills <[email protected]>
TryBot-Bypass: Bryan Mills <[email protected]>
Reviewed-by: Russ Cox <[email protected]>
Auto-Submit: Bryan Mills <[email protected]>

src/cmd/go/internal/web/http.go

@@ -114,7 +114,7 @@ func interceptURL(u *urlpkg.URL) (*Interceptor, bool) {
 		return nil, false
 	}
 	for i, t := range testInterceptors {
-		if u.Host == t.FromHost && (t.Scheme == "" || u.Scheme == t.Scheme) {
+		if u.Host == t.FromHost && (u.Scheme == "" || u.Scheme == t.Scheme) {
 			return &testInterceptors[i], true
 		}
 	}

src/cmd/go/script_test.go

@@ -210,6 +210,7 @@ func scriptEnv(srv *vcstest.Server, srvCertFile string) ([]string, error) {
 		"GOROOT=" + testGOROOT,
 		"GOROOT_FINAL=" + testGOROOT_FINAL, // causes spurious rebuilds and breaks the "stale" built-in if not propagated
 		"GOTRACEBACK=system",
+		"TESTGONETWORK=panic", // allow only local connections by default; the [net] condition resets this
 		"TESTGO_GOROOT=" + testGOROOT,
 		"TESTGO_EXE=" + testGo,
 		"TESTGO_VCSTEST_HOST=" + httpURL.Host,
@@ -232,8 +233,11 @@ func scriptEnv(srv *vcstest.Server, srvCertFile string) ([]string, error) {
 			"GIT_TRACE_CURL_NO_DATA=1",
 			"GIT_REDACT_COOKIES=o,SSO,GSSO_Uberproxy")
 	}
-	if !testenv.HasExternalNetwork() {
-		env = append(env, "TESTGONETWORK=panic", "TESTGOVCS=panic")
+	if testing.Short() {
+		// VCS commands are always somewhat slow: they either require access to external hosts,
+		// or they require our intercepted vcs-test.golang.org to regenerate the repository.
+		// Require all tests that use VCS commands to be skipped in short mode.
+		env = append(env, "TESTGOVCS=panic")
 	}
 	if os.Getenv("CGO_ENABLED") != "" || runtime.GOOS != goHostOS || runtime.GOARCH != goHostArch {
 		// If the actual CGO_ENABLED might not match the cmd/go default, set it

src/cmd/go/scriptconds_test.go

@@ -40,6 +40,7 @@ func scriptConditions() map[string]script.Cond {
 	add("buildmode", script.PrefixCondition("go supports -buildmode=<suffix>", hasBuildmode))
 	add("case-sensitive", script.OnceCondition("$WORK filesystem is case-sensitive", isCaseSensitive))
 	add("cgo", script.BoolCondition("host CGO_ENABLED", testenv.HasCGO()))
+	add("cgolinkext", script.BoolCondition("platform requires external linking for cgo", platform.MustLinkExternal(cfg.Goos, cfg.Goarch, true)))
 	add("cross", script.BoolCondition("cmd/go GOOS/GOARCH != GOHOSTOS/GOHOSTARCH", goHostOS != runtime.GOOS || goHostArch != runtime.GOARCH))
 	add("fuzz", sysCondition("-fuzz", platform.FuzzSupported, false))
 	add("fuzz-instrumented", sysCondition("-fuzz with instrumentation", platform.FuzzInstrumented, false))
@@ -49,8 +50,7 @@ func scriptConditions() map[string]script.Cond {
 	add("link", lazyBool("testenv.HasLink()", testenv.HasLink))
 	add("mismatched-goroot", script.Condition("test's GOROOT_FINAL does not match the real GOROOT", isMismatchedGoroot))
 	add("msan", sysCondition("-msan", platform.MSanSupported, true))
-	add("cgolinkext", script.BoolCondition("platform requires external linking for cgo", platform.MustLinkExternal(cfg.Goos, cfg.Goarch, true)))
-	add("net", lazyBool("testenv.HasExternalNetwork()", testenv.HasExternalNetwork))
+	add("net", script.PrefixCondition("can connect to external network host <suffix>", hasNet))
 	add("race", sysCondition("-race", platform.RaceDetectorSupported, true))
 	add("symlink", lazyBool("testenv.HasSymlink()", testenv.HasSymlink))
 	add("trimpath", script.OnceCondition("test binary was built with -trimpath", isTrimpath))
@@ -95,6 +95,20 @@ func hasBuildmode(s *script.State, mode string) (bool, error) {
 	return platform.BuildModeSupported(runtime.Compiler, mode, GOOS, GOARCH), nil
 }
 
+func hasNet(s *script.State, host string) (bool, error) {
+	if !testenv.HasExternalNetwork() {
+		return false, nil
+	}
+
+	// TODO(bcmills): Add a flag or environment variable to allow skipping tests
+	// for specific hosts and/or skipping all net tests except for specific hosts.
+
+	// Since we have confirmed that the network is available,
+	// allow cmd/go to use it.
+	s.Setenv("TESTGONETWORK", "")
+	return true, nil
+}
+
 func hasGodebug(s *script.State, value string) (bool, error) {
 	godebug, _ := s.LookupEnv("GODEBUG")
 	for _, p := range strings.Split(godebug, ",") {

src/cmd/go/testdata/script/README

@@ -402,8 +402,8 @@ The available conditions are:
 	test's GOROOT_FINAL does not match the real GOROOT
 [msan]
 	GOOS/GOARCH supports -msan
-[net]
-	testenv.HasExternalNetwork()
+[net:*]
+	can connect to external network host <suffix>
 [race]
 	GOOS/GOARCH supports -race
 [root]

src/cmd/go/testdata/script/get_404_meta.txt

@@ -1,6 +1,6 @@
 # golang.org/issue/13037: 'go get' was not parsing <meta> tags in 404 served over HTTPS.
 
-[!net] skip
+[!net:bazil.org] skip
 [!git] skip
 
 env GONOSUMDB=bazil.org,github.com,golang.org

src/cmd/go/testdata/script/get_custom_domain_wildcard.txt

@@ -1,4 +1,4 @@
-[!net] skip
+[!net:rsc.io] skip
 [!git] skip
 env GO111MODULE=off
 

src/cmd/go/testdata/script/get_dash_t.txt

@@ -1,6 +1,6 @@
 # Tests issue 8181
 
-[!net] skip
+[!net:github.com] skip
 [!git] skip
 env GO111MODULE=off
 

src/cmd/go/testdata/script/get_domain_root.txt

@@ -1,7 +1,7 @@
 # Tests issue #9357
 # go get foo.io (not foo.io/subdir) was not working consistently.
 
-[!net] skip
+[!net:go-get-issue-9357.appspot.com] skip
 [!git] skip
 env GO111MODULE=off
 

src/cmd/go/testdata/script/get_dot_slash_download.txt

@@ -1,4 +1,4 @@
-[!net] skip
+[!net:rsc.io] skip
 [!git] skip
 env GO111MODULE=off
 

src/cmd/go/testdata/script/get_goroot.txt

@@ -1,4 +1,4 @@
-[!net] skip
+[!net:github.com] skip
 env GO111MODULE=off
 
 # Issue 4186. go get cannot be used to download packages to $GOROOT.

src/cmd/go/testdata/script/get_insecure.txt

@@ -1,6 +1,6 @@
 # TODO(matloob): Split this test into two? It's one of the slowest tests we have.
 
-[!net] skip
+[!net:insecure.go-get-issue-15410.appspot.com] skip
 [!git] skip
 
 env PATH=$WORK/tmp/bin${:}$PATH

src/cmd/go/testdata/script/get_insecure_custom_domain.txt

@@ -1,4 +1,4 @@
-[!net] skip
+[!net:insecure.go-get-issue-15410.appspot.com] skip
 [!git] skip
 env GO111MODULE=off
 

src/cmd/go/testdata/script/get_insecure_env.txt

@@ -1,4 +1,4 @@
-[!net] skip
+[!net:insecure.go-get-issue-15410.appspot.com] skip
 [!git] skip
 
 # GOPATH: Set up

src/cmd/go/testdata/script/get_insecure_redirect.txt

@@ -1,7 +1,6 @@
 # golang.org/issue/29591: 'go get' was following plain-HTTP redirects even without -insecure (now replaced by GOINSECURE).
 # golang.org/issue/34049: 'go get' would panic in case of an insecure redirect in GOPATH mode
 
-[!net] skip
 [!git] skip
 
 env GO111MODULE=off

src/cmd/go/testdata/script/get_insecure_update.txt

@@ -1,4 +1,4 @@
-[!net] skip
+[!net:github.com] skip
 [!git] skip
 env GO111MODULE=off
 

src/cmd/go/testdata/script/get_internal_wildcard.txt

@@ -1,4 +1,4 @@
-[!net] skip
+[!net:github.com] skip
 [!git] skip
 env GO111MODULE=off
 

src/cmd/go/testdata/script/get_issue11307.txt

@@ -1,6 +1,6 @@
 # go get -u was not working except in checkout directory
 
-[!net] skip
+[!net:github.com] skip
 [!git] skip
 env GO111MODULE=off
 

src/cmd/go/testdata/script/get_issue16471.txt

@@ -1,4 +1,5 @@
-[!net] skip
+[!net:rsc.io] skip
+[!net:github.com] skip
 [!git] skip
 
 env GO111MODULE=off

src/cmd/go/testdata/script/get_issue22125.txt

@@ -1,6 +1,6 @@
 # This test verifies a fix for a security issue; see https://go.dev/issue/22125.
 
-[!net] skip
+[short] skip
 [!git] skip
 [!exec:svn] skip
 

src/cmd/go/testdata/script/get_non_pkg.txt

@@ -1,4 +1,4 @@
-[!net] skip
+[!net:golang.org] skip
 [!git] skip
 
 env GOBIN=$WORK/tmp/gobin
@@ -11,4 +11,4 @@ stderr 'golang.org/x/tools: no Go files'
 stderr 'golang.org/x/tools: no Go files'
 
 ! go get -d golang.org/x/tools
-stderr 'golang.org/x/tools: no Go files'
+stderr 'golang.org/x/tools: no Go files'

src/cmd/go/testdata/script/get_race.txt

@@ -1,6 +1,6 @@
 # Tests issue #20502
 
-[!net] skip
+[!net:github.com] skip
 [!git] skip
 [!race] skip
 env GO111MODULE=off

src/cmd/go/testdata/script/get_test_only.txt

@@ -1,4 +1,4 @@
-[!net] skip
+[!net:golang.org] skip
 [!git] skip
 env GO111MODULE=off
 

src/cmd/go/testdata/script/get_tilde.txt

@@ -1,24 +1,25 @@
 env GO111MODULE=off
-[short] skip
 
 # Paths containing windows short names should be rejected before attempting to fetch.
-! go get example.com/longna~1.dir/thing
+! go get vcs-test.golang.org/longna~1.dir/thing
 stderr 'trailing tilde and digits'
-! go get example.com/longna~1/thing
+! go get vcs-test.golang.org/longna~1/thing
 stderr 'trailing tilde and digits'
-! go get example.com/~9999999/thing
+! go get vcs-test.golang.org/~9999999/thing
 stderr 'trailing tilde and digits'
 
+[short] stop
+
 # A path containing an element that is just a tilde, or a tilde followed by non-digits,
 # should attempt to resolve.
-! go get example.com/~glenda/notfound
+! go get vcs-test.golang.org/~glenda/notfound
 ! stderr 'trailing tilde and digits'
 stderr 'unrecognized import path'
 
-! go get example.com/~glenda2/notfound
+! go get vcs-test.golang.org/~glenda2/notfound
 ! stderr 'trailing tilde and digits'
 stderr 'unrecognized import path'
 
-! go get example.com/~/notfound
+! go get vcs-test.golang.org/~/notfound
 ! stderr 'trailing tilde and digits'
 stderr 'unrecognized import path'

src/cmd/go/testdata/script/get_update.txt

@@ -2,7 +2,7 @@
 # The recursive updating was trying to walk to
 # former dependencies, not current ones.
 
-[!net] skip
+[!net:github.com] skip
 [!git] skip
 env GO111MODULE=off
 

src/cmd/go/testdata/script/get_update_all.txt

@@ -1,8 +1,6 @@
 # Issue 14444: go get -u .../ duplicate loads errors
 # Check that go get update -u ... does not try to load duplicates
 
-[!net] skip
-
 env GO111MODULE=off
 
 go get -u -n .../

src/cmd/go/testdata/script/get_update_unknown_protocol.txt

@@ -1,4 +1,4 @@
-[!net] skip
+[!net:github.com] skip
 [!git] skip
 env GO111MODULE=off
 

src/cmd/go/testdata/script/get_update_wildcard.txt

@@ -1,6 +1,6 @@
 # Issue 14450: go get -u .../ tried to import not downloaded package
 
-[!net] skip
+[!net:github.com] skip
 [!git] skip
 env GO111MODULE=off
 

src/cmd/go/testdata/script/get_vendor.txt

@@ -11,7 +11,7 @@ stdout 'v/vendor/vendor.org/p'
 go get -d
 go get -t -d
 
-[!net] stop
+[!net:github.com] stop
 [!git] stop
 
 cd $GOPATH/src

src/cmd/go/testdata/script/get_with_git_trace.txt

@@ -2,7 +2,7 @@ env GO111MODULE=off
 
 env GIT_TRACE=1
 
-[!net] skip
+[!net:golang.org] skip
 [!git] skip
 
 # go get should be success when GIT_TRACE set

src/cmd/go/testdata/script/gopath_moved_repo.txt

@@ -3,7 +3,7 @@ env GO111MODULE=off
 # Test that 'go get -u' reports packages whose VCS configurations do not
 # match their import paths.
 
-[!net] skip
+[!net:rsc.io] skip
 [short] skip
 
 # We need to execute a custom Go program to break the config files.

src/cmd/go/testdata/script/gopath_vendor_dup_err.txt

@@ -1,4 +1,3 @@
-[!net] skip
 env GO111MODULE=off
 
 # Issue 17119: Test more duplicate load errors.

src/cmd/go/testdata/script/govcs.txt

@@ -67,12 +67,12 @@ stderr '^go: rsc.io/nonexist.bzr/hello: GOVCS disallows using bzr for public rsc
 # git is OK by default
 env GOVCS=
 env GONOSUMDB='*'
-[net] [git] [!short] go get rsc.io/sampler
+[net:rsc.io] [git] [!short] go get rsc.io/sampler
 
 # hg is OK by default
 env GOVCS=
 env GONOSUMDB='*'
-[net] [exec:hg] [!short] go get vcs-test.golang.org/go/custom-hg-hello
+[exec:hg] [!short] go get vcs-test.golang.org/go/custom-hg-hello
 
 # git can be disallowed
 env GOVCS=public:hg
@@ -150,12 +150,12 @@ stderr '^package rsc.io/nonexist.bzr/hello: GOVCS disallows using bzr for public
 # git is OK by default
 env GOVCS=
 env GONOSUMDB='*'
-[net] [git] [!short] go get rsc.io/sampler
+[net:rsc.io] [git] [!short] go get rsc.io/sampler
 
 # hg is OK by default
 env GOVCS=
 env GONOSUMDB='*'
-[net] [exec:hg] [!short] go get vcs-test.golang.org/go/custom-hg-hello
+[exec:hg] [!short] go get vcs-test.golang.org/go/custom-hg-hello
 
 # git can be disallowed
 env GOVCS=public:hg

src/cmd/go/testdata/script/install_shadow_gopath.txt

@@ -1,8 +1,6 @@
 # Tests Issue #3562
 # go get foo.io (not foo.io/subdir) was not working consistently.
 
-[!net] skip
-
 env GO111MODULE=off
 env GOPATH=$WORK/gopath1${:}$WORK/gopath2
 

src/cmd/go/testdata/script/list_std_vendor.txt

@@ -25,7 +25,7 @@ cmp stdout $WORK/net-deps.txt
 # However, 'go mod' and 'go get' subcommands should report the original module
 # dependencies, not the vendored packages.
 
-[!net] stop
+[!net:golang.org] stop
 
 env GOPROXY=
 env GOWORK=off

src/cmd/go/testdata/script/mod_auth.txt

@@ -1,4 +1,4 @@
-[!net] skip
+[short] skip
 
 env GO111MODULE=on
 env GOPROXY=direct

src/cmd/go/testdata/script/mod_convert.txt

@@ -1,5 +1,6 @@
-[short] skip
-[!net] skip
+[!net:github.com] skip
+[!net:golang.org] skip
+[!net:gopkg.in] skip
 [!git] skip
 
 env GO111MODULE=on
@@ -13,6 +14,9 @@ cd x
 go mod init github.com/docker/distribution
 cmpenv go.mod go.mod.want
 
+[!net:google.golang.org] skip
+[!net:cloud.google.com] skip
+
 go mod download github.com/fishy/[email protected]
 cp $GOPATH/pkg/mod/github.com/fishy/[email protected]/Gopkg.lock ../y
 cd ../y

src/cmd/go/testdata/script/mod_convert_tsv_insecure.txt

@@ -2,7 +2,7 @@ env GO111MODULE=on
 env GOPROXY=direct
 env GOSUMDB=off
 
-[!net] skip
+[short] skip
 [!git] skip
 
 # secure fetch should report insecure warning