Fix flaky tests by disabling automatic plugin acquisition for all tests

Makefile

@@ -1,9 +1,18 @@
 SHELL            := sh
 PROJECT          := github.com/pulumi/pulumi-terraform-bridge
 TESTPARALLELISM  := 10
+export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION := true
 
 PROJECT_DIR := $(patsubst %/,%,$(dir $(abspath $(lastword $(MAKEFILE_LIST)))))
 
+install_plugins::
+	pulumi plugin install converter terraform 1.0.18
+	pulumi plugin install resource random 4.16.3
+	pulumi plugin install resource aws 6.22.2
+	pulumi plugin install resource archive 0.0.4
+	pulumi plugin install resource wavefront 3.0.0
+	pulumi plugin install resource equinix 0.6.0 --server github://api.github.com/equinix
+
 build::
 	go mod tidy
 	go build ${PROJECT}/v3/pkg/...
@@ -19,7 +28,7 @@ lint:
 lint_fix:
 	go run scripts/build.go fix-lint
 
-test::
+test:: install_plugins
 	@mkdir -p bin
 	go build -o bin ./internal/testing/pulumi-terraform-bridge-test-provider
 	PULUMI_TERRAFORM_BRIDGE_TEST_PROVIDER=$(shell pwd)/bin/pulumi-terraform-bridge-test-provider \

pf/Makefile

@@ -1,6 +1,10 @@
 TESTPARALLELISM := 10
+export PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION := true
 
-build.tests::
+install_plugins::
+	pulumi plugin install resource random 4.16.3
+
+build.tests:: install_plugins
 	go test -test.run NONE
 	cd tests && go test -test.run NONE
 	cd tests/integration && go test -test.run NONE

pf/tests/integration/program_test.go

@@ -30,6 +30,8 @@ import (
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
 	"github.com/stretchr/testify/require"
 	"sourcegraph.com/sourcegraph/appdash"
+
+	"github.com/pulumi/pulumi-terraform-bridge/v3/unstable/testutil"
 )
 
 func TestBasicProgram(t *testing.T) {
@@ -223,6 +225,9 @@ func TestResourceWithoutID(t *testing.T) {
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
 		Env: []string{fmt.Sprintf("PATH=%s", bin)},
 		Dir: filepath.Join("..", "testdata", "resource-without-id"),
+		LocalProviders: []integration.LocalDependency{
+			testutil.RandomProvider(t),
+		},
 	})
 }
 

pkg/tests/acc_provider_config_test.go

@@ -19,16 +19,20 @@ import (
 
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
 	"github.com/stretchr/testify/assert"
+
+	"github.com/pulumi/pulumi-terraform-bridge/v3/unstable/testutil"
 )
 
 func TestAccProviderConfig(t *testing.T) {
 	opts := accTestOptions(t).With(integration.ProgramTestOptions{
 		Dir: "provider-config",
-
 		ExtraRuntimeValidation: func(t *testing.T, stack integration.RuntimeValidationStackInfo) {
 			assert.Equal(t, stack.Outputs["generatedRandomString"],
 				stack.Outputs["providerRandomString"])
 		},
+		LocalProviders: []integration.LocalDependency{
+			testutil.RandomProvider(t),
+		},
 	})
 	integration.ProgramTest(t, &opts)
 }

pkg/tests/internal/pulcheck/pulcheck.go

@@ -163,7 +163,7 @@ func PulCheck(t T, bridgedProvider info.Provider, program string) *pulumitest.Pu
 	require.NoError(t, err)
 
 	opts := []opttest.Option{
-		opttest.Env("DISABLE_AUTOMATIC_PLUGIN_ACQUISITION", "true"),
+		opttest.Env("PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION", "true"),
 		opttest.TestInPlace(),
 		opttest.SkipInstall(),
 		opttest.AttachProvider(

pkg/tests/main_test.go

@@ -28,6 +28,8 @@ import (
 	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
 )
 
+var localTestProviders []integration.LocalDependency
+
 func TestMain(m *testing.M) {
 	if err := setupIntegrationTests(); err != nil {
 		log.Fatal(err)
@@ -56,7 +58,9 @@ func accTestOptions(t *testing.T) *integration.ProgramTestOptions {
 	return &integration.ProgramTestOptions{
 		Env: []string{
 			fmt.Sprintf("PATH=%s", filepath.Join(cwd, "..", "..", "bin")),
+			"PULUMI_DISABLE_AUTOMATIC_PLUGIN_ACQUISITION=true",
 		},
+		LocalProviders: localTestProviders,
 	}
 }
 
@@ -72,16 +76,21 @@ func ensureCompiledTestProviders(wd string) error {
 
 	internalErrorMsg := "Internal validation of the provider failed"
 
+	internal := func(segments ...string) string {
+		return filepath.Join(append([]string{wd, "..", "..", "internal"}, segments...)...)
+	}
 	testProviders := []testProvider{
 		{
 			"tpsdkv2",
-			filepath.Join(wd, "..", "..", "internal", "testprovider_sdkv2",
-				"cmd", "pulumi-resource-tpsdkv2"),
-			filepath.Join(wd, "..", "..", "internal", "testprovider_sdkv2",
-				"cmd", "pulumi-tfgen-tpsdkv2"), nil,
+			internal("testprovider_sdkv2", "cmd", "pulumi-resource-tpsdkv2"),
+			internal("testprovider_sdkv2", "cmd", "pulumi-tfgen-tpsdkv2"),
+			nil,
 		},
 		{
-			"testprovider_invschema", filepath.Join(wd, "..", "..", "internal", "testprovider_invalid_schema", "cmd", "pulumi-resource-tpinvschema"), filepath.Join(wd, "..", "..", "internal", "testprovider_invalid_schema", "cmd", "pulumi-tfgen-tpinvschema"), &internalErrorMsg,
+			"testprovider_invschema",
+			internal("testprovider_invalid_schema", "cmd", "pulumi-resource-tpinvschema"),
+			internal("testprovider_invalid_schema", "cmd", "pulumi-tfgen-tpinvschema"),
+			&internalErrorMsg,
 		},
 	}
 
@@ -143,6 +152,10 @@ func ensureCompiledTestProviders(wd string) error {
 				fmt.Println(stderr)
 				return fmt.Errorf("provider build failed for %s: %w", p.name, err)
 			}
+			localTestProviders = append(localTestProviders, integration.LocalDependency{
+				Package: p.name,
+				Path:    bin, // The path to the directory that contains the binary, not the binary
+			})
 		}
 	}
 

pkg/tfgen/convert_cli_test.go

@@ -56,7 +56,6 @@ func TestConvertViaPulumiCLI(t *testing.T) {
 		t.Skipf("Skipping on Windows due to a test setup issue")
 	}
 	t.Setenv("PULUMI_CONVERT", "1")
-	t.Setenv("DISABLE_AUTOMATIC_PLUGIN_ACQUISITION", "true")
 
 	simpleResourceTF := `
 resource "simple_resource" "a_resource" {

pkg/tfgen/docs_test.go

@@ -22,7 +22,6 @@ import (
 	"fmt"
 	"io"
 	"os"
-	"os/exec"
 	"path/filepath"
 	"runtime"
 	"strings"
@@ -1290,8 +1289,6 @@ func TestConvertExamples(t *testing.T) {
 		name string
 		path examplePath
 
-		needsProviders map[string]pluginDesc
-
 		language *Language
 	}
 
@@ -1302,19 +1299,13 @@ func TestConvertExamples(t *testing.T) {
 				fullPath: "#/resources/wavefront:index/dashboardJson:DashboardJson",
 				token:    "wavefront:index/dashboardJson:DashboardJson",
 			},
-			needsProviders: map[string]pluginDesc{
-				"wavefront": {version: "3.0.0"},
-			},
 		},
 		{
 			name: "golang_wavefront_dashboard_json",
 			path: examplePath{
 				fullPath: "#/resources/wavefront:index/dashboardJson:DashboardJson",
 				token:    "wavefront:index/dashboardJson:DashboardJson",
 			},
-			needsProviders: map[string]pluginDesc{
-				"wavefront": {version: "3.0.0"},
-			},
 			language: ref(Golang),
 		},
 		{
@@ -1323,39 +1314,19 @@ func TestConvertExamples(t *testing.T) {
 				fullPath: "#/resources/equinix:fabric:Connection",
 				token:    "equinix:fabric:Connection",
 			},
-			needsProviders: map[string]pluginDesc{
-				"equinix": {
-					pluginDownloadURL: "github://api.github.com/equinix",
-					version:           "0.6.0",
-				},
-			},
 		},
 		{
 			name: "aws_lambda_function",
 			path: examplePath{
 				fullPath: "#/resources/aws:lambda/function:Function",
 				token:    "aws:lambda/function:Function",
 			},
-			needsProviders: map[string]pluginDesc{
-				"aws": {
-					pluginDownloadURL: "github://api.github.com/pulumi",
-					version:           "6.22.2",
-				},
-				"archive": {
-					pluginDownloadURL: "github://api.github.com/pulumi",
-					version:           "0.0.4",
-				},
-			},
 		},
 	}
 
 	for _, tc := range testCases {
 		tc := tc
 
-		t.Run(fmt.Sprintf("%s/setup", tc.name), func(t *testing.T) {
-			ensureProvidersInstalled(t, tc.needsProviders)
-		})
-
 		t.Run(tc.name, func(t *testing.T) {
 			inmem := afero.NewMemMapFs()
 			info := testprovider.ProviderMiniRandom()
@@ -1413,9 +1384,8 @@ func TestConvertExamplesInner(t *testing.T) {
 	assert.NoError(t, err)
 
 	type testCase struct {
-		name           string
-		path           examplePath
-		needsProviders map[string]pluginDesc
+		name string
+		path examplePath
 	}
 
 	testCases := []testCase{
@@ -1438,10 +1408,6 @@ func TestConvertExamplesInner(t *testing.T) {
 	for _, tc := range testCases {
 		tc := tc
 
-		t.Run(fmt.Sprintf("%s/setup", tc.name), func(t *testing.T) {
-			ensureProvidersInstalled(t, tc.needsProviders)
-		})
-
 		t.Run(tc.name, func(t *testing.T) {
 			docs, err := os.ReadFile(filepath.Join("test_data", "convertExamples",
 				fmt.Sprintf("%s.md", tc.name)))
@@ -1461,11 +1427,6 @@ func TestConvertExamplesInner(t *testing.T) {
 	}
 }
 
-type pluginDesc struct {
-	version           string
-	pluginDownloadURL string
-}
-
 func TestFindFencesAndHeaders(t *testing.T) {
 	if runtime.GOOS == "windows" {
 		t.Skipf("Skipping on windows to avoid failing on incorrect newline handling")
@@ -1530,61 +1491,6 @@ func TestFindFencesAndHeaders(t *testing.T) {
 
 }
 
-func ensureProvidersInstalled(t *testing.T, needsProviders map[string]pluginDesc) {
-	pulumi, err := exec.LookPath("pulumi")
-	require.NoError(t, err)
-
-	t.Logf("pulumi plugin ls --json")
-	cmd := exec.Command(pulumi, "plugin", "ls", "--json")
-	var buf bytes.Buffer
-	cmd.Stdout = &buf
-	err = cmd.Run()
-	require.NoError(t, err)
-
-	type plugin struct {
-		Name    string `json:"name"`
-		Version string `json:"version"`
-	}
-
-	var installedPlugins []plugin
-	err = json.Unmarshal(buf.Bytes(), &installedPlugins)
-	require.NoError(t, err)
-
-	for name, desc := range needsProviders {
-		count := 0
-		matched := false
-
-		for _, p := range installedPlugins {
-			if p.Name == name {
-				count++
-			}
-			if p.Name == name && p.Version == desc.version {
-				matched = true
-			}
-		}
-
-		alreadyInstalled := count == 1 && matched
-		if alreadyInstalled {
-			continue
-		}
-
-		if count > 0 {
-			t.Logf("pulumi plugin rm resource %s", name)
-			err = exec.Command(pulumi, "plugin", "rm", "resource", name).Run()
-			require.NoError(t, err)
-		}
-
-		args := []string{"plugin", "install", "resource", name, desc.version}
-		if desc.pluginDownloadURL != "" {
-			args = append(args, "--server", desc.pluginDownloadURL)
-		}
-		cmd := exec.Command(pulumi, args...)
-		t.Logf("Exec: %s", cmd)
-		err = cmd.Run()
-		require.NoError(t, err)
-	}
-}
-
 func TestExampleGeneration(t *testing.T) {
 	info := testprovider.ProviderMiniRandom()
 

unstable/testutil/installed_providers.go

@@ -0,0 +1,49 @@
+// Copyright 2016-2023, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+package testutil
+
+import (
+	"os"
+	"path/filepath"
+	"testing"
+
+	"github.com/blang/semver"
+	"github.com/pulumi/pulumi/pkg/v3/testing/integration"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/apitype"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/diag"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/diag/colors"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/workspace"
+	"github.com/stretchr/testify/require"
+)
+
+// RandomProvider returns a [integration.LocalDependency] reference to the random provider
+// installed via `make install_plugins`.
+func RandomProvider(t *testing.T) integration.LocalDependency {
+	return pluginDependency(t, "random", semver.Version{Major: 4, Minor: 16, Patch: 3})
+}
+
+func pluginDependency(t *testing.T, name string, version semver.Version) integration.LocalDependency {
+	path, err := workspace.GetPluginPath(
+		diag.DefaultSink(os.Stdout, os.Stderr, diag.FormatOptions{
+			Color: colors.Never,
+		}),
+		apitype.ResourcePlugin, name, &version, nil)
+	require.NoError(t, err,
+		`The %s provider at this version should have been installed by "make install_plugins"`, name)
+	return integration.LocalDependency{
+		Package: name,
+		Path:    filepath.Dir(path),
+	}
+}