Fix flaky tests by disabling automatic plugin acquisition for all tes…

…ts (#2190)

Disable automatic acquisition globally in the repository - this should
ensure we do not use the GH API during tests and make our tests here a
lot more robust.

Instead manually install all required plugins in the Makefile as a
dependency of the tests. This ensures we only do it once and do not spam
the GH API with redundant requests causing test flakes.

I've hit this more than 5 times on separate PRs in the last few days.
It's seriously slowing down progress in the repo.

When we do hit the GH rate limit, we have to wait for up to an hour for
it to reset.

~Skip `TestConvertViaPulumiCLI` until
pulumi/pulumi#16469 is addressed.~
~Skip `TestResourceWithoutID` until
#2191 is
addressed.~

Looks like we were using the wrong env variable name...

One of the tests was also absolutely spamming the GH API:
https://github.com/pulumi/pulumi-terraform-bridge/actions/runs/9973182117/job/27557985796?pr=2190

Latest failures:
https://github.com/pulumi/pulumi-terraform-bridge/actions/runs/9972810153/job/27556840950?pr=2186

fixes #2191

Makefile

@@ -1,9 +1,18 @@
 SHELL            := sh
 PROJECT          := github.com/pulumi/pulumi-terraform-bridge
 TESTPARALLELISM  := 10
+export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION := true
 
 PROJECT_DIR := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
 
+install_plugins::
+	pulumi plugin install converter terraform 1.0.18
+	pulumi plugin install resource random 4.16.3
+	pulumi plugin install resource aws 6.22.2
+	pulumi plugin install resource archive 0.0.4
+	pulumi plugin install resource wavefront 3.0.0
+	pulumi plugin install resource equinix 0.6.0 --server github://api.github.com/equinix
+
 build::
 	go mod tidy
 	go build ${PROJECT}/v3/pkg/...
@@ -19,7 +28,7 @@ lint:
 lint_fix:
 	go run scripts/build.go fix-lint
 
-test::
+test:: install_plugins
 	@mkdir -p bin
 	go build -o bin ./internal/testing/pulumi-terraform-bridge-test-provider
 	PULUMI_TERRAFORM_BRIDGE_TEST_PROVIDER=$(shell pwd)/bin/pulumi-terraform-bridge-test-provider \

pf/Makefile

@@ -1,6 +1,10 @@
 TESTPARALLELISM := 10
+export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION := true
 
-build.tests::
+install_plugins::
+	pulumi plugin install resource random 4.16.3
+
+build.tests:: install_plugins
 	go test -test.run NONE
 	cd tests && go test -test.run NONE
 	cd tests/integration && go test -test.run NONE

pf/tests/integration/program_test.go

@@ -30,6 +30,8 @@ import (
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
 	"github.com/stretchr/testify/require"
 	"sourcegraph.com/sourcegraph/appdash"
+
+	"github.com/pulumi/pulumi-terraform-bridge/v3/unstable/testutil"
 )
 
 func TestBasicProgram(t *testing.T) {
@@ -223,6 +225,9 @@ func TestResourceWithoutID(t *testing.T) {
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
 		Env: []string{fmt.Sprintf("PATH=%s", bin)},
 		Dir: filepath.Join("..", "testdata", "resource-without-id"),
+		LocalProviders: []integration.LocalDependency{
+			testutil.RandomProvider(t),
+		},
 	})
 }
 

pkg/tests/acc_provider_config_test.go

@@ -19,16 +19,20 @@ import (
 
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/pulumi/pulumi-terraform-bridge/v3/unstable/testutil"
 )
 
 func TestAccProviderConfig(t *testing.T) {
 	opts := accTestOptions(t).With(integration.ProgramTestOptions{
 		Dir: "provider-config",
-
 		ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
 			assert.Equal(t, stack.Outputs["generatedRandomString"],
 				stack.Outputs["providerRandomString"])
 		},
+		LocalProviders: []integration.LocalDependency{
+			testutil.RandomProvider(t),
+		},
 	})
 	integration.ProgramTest(t, &opts)
 }

pkg/tests/internal/pulcheck/pulcheck.go

@@ -185,7 +185,7 @@ func PulCheck(t T, bridgedProvider info.Provider, program string) *pulumitest.Pu
 	require.NoError(t, err)
 
 	opts := []opttest.Option{
-		opttest.Env("DISABLE_AUTOMATIC_PLUGIN_ACQUISITION", "true"),
+		opttest.Env("PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION", "true"),
 		opttest.TestInPlace(),
 		opttest.SkipInstall(),
 		opttest.AttachProvider(

pkg/tests/main_test.go

@@ -28,6 +28,8 @@ import (
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
 )
 
+var localTestProviders []integration.LocalDependency
+
 func TestMain(m *testing.M) {
 	if err := setupIntegrationTests(); err != nil {
 		log.Fatal(err)
@@ -56,7 +58,9 @@ func accTestOptions(t *testing.T) *integration.ProgramTestOptions {
 	return &integration.ProgramTestOptions{
 		Env: []string{
 			fmt.Sprintf("PATH=%s", filepath.Join(cwd, "..", "..", "bin")),
+			"PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=true",
 		},
+		LocalProviders: localTestProviders,
 	}
 }
 
@@ -72,16 +76,21 @@ func ensureCompiledTestProviders(wd string) error {
 
 	internalErrorMsg := "Internal validation of the provider failed"
 
+	internal := func(segments ...string) string {
+		return filepath.Join(append([]string{wd, "..", "..", "internal"}, segments...)...)
+	}
 	testProviders := []testProvider{
 		{
 			"tpsdkv2",
-			filepath.Join(wd, "..", "..", "internal", "testprovider_sdkv2",
-				"cmd", "pulumi-resource-tpsdkv2"),
-			filepath.Join(wd, "..", "..", "internal", "testprovider_sdkv2",
-				"cmd", "pulumi-tfgen-tpsdkv2"), nil,
+			internal("testprovider_sdkv2", "cmd", "pulumi-resource-tpsdkv2"),
+			internal("testprovider_sdkv2", "cmd", "pulumi-tfgen-tpsdkv2"),
+			nil,
 		},
 		{
-			"testprovider_invschema", filepath.Join(wd, "..", "..", "internal", "testprovider_invalid_schema", "cmd", "pulumi-resource-tpinvschema"), filepath.Join(wd, "..", "..", "internal", "testprovider_invalid_schema", "cmd", "pulumi-tfgen-tpinvschema"), &internalErrorMsg,
+			"testprovider_invschema",
+			internal("testprovider_invalid_schema", "cmd", "pulumi-resource-tpinvschema"),
+			internal("testprovider_invalid_schema", "cmd", "pulumi-tfgen-tpinvschema"),
+			&internalErrorMsg,
 		},
 	}
 
@@ -143,6 +152,10 @@ func ensureCompiledTestProviders(wd string) error {
 				fmt.Println(stderr)
 				return fmt.Errorf("provider build failed for %s: %w", p.name, err)
 			}
+			localTestProviders = append(localTestProviders, integration.LocalDependency{
+				Package: p.name,
+				Path:    bin, // The path to the directory that contains the binary, not the binary
+			})
 		}
 	}
 

pkg/tfgen/convert_cli_test.go

@@ -56,7 +56,6 @@ func TestConvertViaPulumiCLI(t *testing.T) {
 		t.Skipf("Skipping on Windows due to a test setup issue")
 	}
 	t.Setenv("PULUMI_CONVERT", "1")
-	t.Setenv("DISABLE_AUTOMATIC_PLUGIN_ACQUISITION", "true")
 
 	simpleResourceTF := `
 resource "simple_resource" "a_resource" {

pkg/tfgen/docs_test.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -1290,8 +1289,6 @@ func TestConvertExamples(t *testing.T) {
 		name string
 		path examplePath
 
-		needsProviders map[string]pluginDesc
-
 		language *Language
 	}
 
@@ -1302,19 +1299,13 @@ func TestConvertExamples(t *testing.T) {
 				fullPath: "#/resources/wavefront:index/dashboardJson:DashboardJson",
 				token:    "wavefront:index/dashboardJson:DashboardJson",
 			},
-			needsProviders: map[string]pluginDesc{
-				"wavefront": {version: "3.0.0"},
-			},
 		},
 		{
 			name: "golang_wavefront_dashboard_json",
 			path: examplePath{
 				fullPath: "#/resources/wavefront:index/dashboardJson:DashboardJson",
 				token:    "wavefront:index/dashboardJson:DashboardJson",
 			},
-			needsProviders: map[string]pluginDesc{
-				"wavefront": {version: "3.0.0"},
-			},
 			language: ref(Golang),
 		},
 		{
@@ -1323,39 +1314,19 @@ func TestConvertExamples(t *testing.T) {
 				fullPath: "#/resources/equinix:fabric:Connection",
 				token:    "equinix:fabric:Connection",
 			},
-			needsProviders: map[string]pluginDesc{
-				"equinix": {
-					pluginDownloadURL: "github://api.github.com/equinix",
-					version:           "0.6.0",
-				},
-			},
 		},
 		{
 			name: "aws_lambda_function",
 			path: examplePath{
 				fullPath: "#/resources/aws:lambda/function:Function",
 				token:    "aws:lambda/function:Function",
 			},
-			needsProviders: map[string]pluginDesc{
-				"aws": {
-					pluginDownloadURL: "github://api.github.com/pulumi",
-					version:           "6.22.2",
-				},
-				"archive": {
-					pluginDownloadURL: "github://api.github.com/pulumi",
-					version:           "0.0.4",
-				},
-			},
 		},
 	}
 
 	for _, tc := range testCases {
 		tc := tc
 
-		t.Run(fmt.Sprintf("%s/setup", tc.name), func(t *testing.T) {
-			ensureProvidersInstalled(t, tc.needsProviders)
-		})
-
 		t.Run(tc.name, func(t *testing.T) {
 			inmem := afero.NewMemMapFs()
 			info := testprovider.ProviderMiniRandom()
@@ -1413,9 +1384,8 @@ func TestConvertExamplesInner(t *testing.T) {
 	assert.NoError(t, err)
 
 	type testCase struct {
-		name           string
-		path           examplePath
-		needsProviders map[string]pluginDesc
+		name string
+		path examplePath
 	}
 
 	testCases := []testCase{
@@ -1438,10 +1408,6 @@ func TestConvertExamplesInner(t *testing.T) {
 	for _, tc := range testCases {
 		tc := tc
 
-		t.Run(fmt.Sprintf("%s/setup", tc.name), func(t *testing.T) {
-			ensureProvidersInstalled(t, tc.needsProviders)
-		})
-
 		t.Run(tc.name, func(t *testing.T) {
 			docs, err := os.ReadFile(filepath.Join("test_data", "convertExamples",
 				fmt.Sprintf("%s.md", tc.name)))
@@ -1461,11 +1427,6 @@ func TestConvertExamplesInner(t *testing.T) {
 	}
 }
 
-type pluginDesc struct {
-	version           string
-	pluginDownloadURL string
-}
-
 func TestFindFencesAndHeaders(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skipf("Skipping on windows to avoid failing on incorrect newline handling")
@@ -1530,61 +1491,6 @@ func TestFindFencesAndHeaders(t *testing.T) {
 
 }
 
-func ensureProvidersInstalled(t *testing.T, needsProviders map[string]pluginDesc) {
-	pulumi, err := exec.LookPath("pulumi")
-	require.NoError(t, err)
-
-	t.Logf("pulumi plugin ls --json")
-	cmd := exec.Command(pulumi, "plugin", "ls", "--json")
-	var buf bytes.Buffer
-	cmd.Stdout = &buf
-	err = cmd.Run()
-	require.NoError(t, err)
-
-	type plugin struct {
-		Name    string `json:"name"`
-		Version string `json:"version"`
-	}
-
-	var installedPlugins []plugin
-	err = json.Unmarshal(buf.Bytes(), &installedPlugins)
-	require.NoError(t, err)
-
-	for name, desc := range needsProviders {
-		count := 0
-		matched := false
-
-		for _, p := range installedPlugins {
-			if p.Name == name {
-				count++
-			}
-			if p.Name == name && p.Version == desc.version {
-				matched = true
-			}
-		}
-
-		alreadyInstalled := count == 1 && matched
-		if alreadyInstalled {
-			continue
-		}
-
-		if count > 0 {
-			t.Logf("pulumi plugin rm resource %s", name)
-			err = exec.Command(pulumi, "plugin", "rm", "resource", name).Run()
-			require.NoError(t, err)
-		}
-
-		args := []string{"plugin", "install", "resource", name, desc.version}
-		if desc.pluginDownloadURL != "" {
-			args = append(args, "--server", desc.pluginDownloadURL)
-		}
-		cmd := exec.Command(pulumi, args...)
-		t.Logf("Exec: %s", cmd)
-		err = cmd.Run()
-		require.NoError(t, err)
-	}
-}
-
 func TestExampleGeneration(t *testing.T) {
 	info := testprovider.ProviderMiniRandom()
 

unstable/testutil/installed_providers.go

@@ -0,0 +1,49 @@
+// Copyright 2016-2023, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testutil
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/blang/semver"
+	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
+	"github.com/stretchr/testify/require"
+)
+
+// RandomProvider returns a [integration.LocalDependency] reference to the random provider
+// installed via `make install_plugins`.
+func RandomProvider(t *testing.T) integration.LocalDependency {
+	return pluginDependency(t, "random", semver.Version{Major: 4, Minor: 16, Patch: 3})
+}
+
+func pluginDependency(t *testing.T, name string, version semver.Version) integration.LocalDependency {
+	path, err := workspace.GetPluginPath(
+		diag.DefaultSink(os.Stdout, os.Stderr, diag.FormatOptions{
+			Color: colors.Never,
+		}),
+		apitype.ResourcePlugin, name, &version, nil)
+	require.NoError(t, err,
+		`The %s provider at this version should have been installed by "make install_plugins"`, name)
+	return integration.LocalDependency{
+		Package: name,
+		Path:    filepath.Dir(path),
+	}
+}