Fixing broken tests (#326)

After an update to pulumi CLI, tests started failing. Most of the issues
were on refresh, and refresh used to be skipped for most tests, which
made it more confusing why the same test failed in yaml, but succeeded
in other languages.

Summary
- Removed SkipRefresh flags everywhere except Deployment Settings,
because this is just a bandaid to make broken resources test pass
(Deployment Settings are currently broken, with fix PR blocked by these
tests failing)
- Reverted `req.GetOldInputs()` back into `req.GetOlds()`, which is not
a good change, but the proper way would be to overhaul how AccessTokens
work (currently Diff uses internal `__inputs` property which should no
longer be used)
- Adjusted go and python environment yaml string to make refresh work
(it's quite ugly, but we do have an issue to add auto-formatting)
- Fixed Schedules and Environment Read() outputting properties instead
of inputs, causing noisy refresh diff due to discrepancy in input and
output schema (this used to be silent, became noisy after a pulumi
engine update)
- Added logic to set `displayName` of Team resource input to name if not
supplied. This is what Pulumi Service does anyway, which caused a noisy
diff

CHANGELOG_PENDING.md

@@ -3,3 +3,5 @@
 ### Bug Fixes
 
 ### Miscellaneous
+
+- Fixed integ tests [#328](https://github.com/pulumi/pulumi-pulumiservice/issues/328)

examples/cs-environments/MyStack.cs

@@ -12,7 +12,7 @@ public MyStack()
           myKey1: "myValue1"
           myNestedKey:
             myKey2: "myValue2"
-            myNumber: 1 
+            myNumber: 1
         """;
 
         var environment = new Pulumi.PulumiService.Environment(

examples/examples_dotnet_test.go

@@ -11,8 +11,7 @@ import (
 
 func TestDotnetTeamsExamples(t *testing.T) {
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
-		Dir:         path.Join(getCwd(t), "cs-teams"),
-		SkipRefresh: true,
+		Dir: path.Join(getCwd(t), "cs-teams"),
 		Dependencies: []string{
 			"Pulumi.PulumiService",
 		},
@@ -22,9 +21,8 @@ func TestDotnetTeamsExamples(t *testing.T) {
 func TestDotnetSchedulesExamples(t *testing.T) {
 	digits := generateRandomFiveDigits()
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
-		Dir:         path.Join(getCwd(t), "cs-schedules"),
-		StackName:   "test-stack-" + digits,
-		SkipRefresh: true,
+		Dir:       path.Join(getCwd(t), "cs-schedules"),
+		StackName: "test-stack-" + digits,
 		Config: map[string]string{
 			"digits": digits,
 		},
@@ -37,8 +35,7 @@ func TestDotnetSchedulesExamples(t *testing.T) {
 func TestDotnetEnvironmentsExamples(t *testing.T) {
 	digits := generateRandomFiveDigits()
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
-		Dir:         path.Join(getCwd(t), "cs-environments"),
-		SkipRefresh: true,
+		Dir: path.Join(getCwd(t), "cs-environments"),
 		Config: map[string]string{
 			"digits": digits,
 		},

examples/examples_go_test.go

@@ -12,19 +12,17 @@ import (
 
 func TestGoTeamsExample(t *testing.T) {
 	testOpts := getGoBaseOptions(t).With(integration.ProgramTestOptions{
-		Verbose:     true,
-		Dir:         filepath.Join(getCwd(t), "go-teams"),
-		SkipRefresh: true,
+		Verbose: true,
+		Dir:     filepath.Join(getCwd(t), "go-teams"),
 	})
 	integration.ProgramTest(t, &testOpts)
 }
 
 func TestGoEnvironmentsExample(t *testing.T) {
 	digits := generateRandomFiveDigits()
 	testOpts := getGoBaseOptions(t).With(integration.ProgramTestOptions{
-		Verbose:     true,
-		Dir:         filepath.Join(getCwd(t), "go-environments"),
-		SkipRefresh: true,
+		Verbose: true,
+		Dir:     filepath.Join(getCwd(t), "go-environments"),
 		Config: map[string]string{
 			"digits": digits,
 		},

examples/examples_java_test.go

@@ -13,9 +13,7 @@ import (
 )
 
 func TestJavaTeamsExamples(t *testing.T) {
-	test := getJavaBase(t, "java-teams", integration.ProgramTestOptions{
-		SkipRefresh: true,
-	})
+	test := getJavaBase(t, "java-teams", integration.ProgramTestOptions{})
 
 	integration.ProgramTest(t, &test)
 }

examples/examples_nodejs_test.go

@@ -14,9 +14,8 @@ import (
 func TestAccessTokenExample(t *testing.T) {
 	cwd, _ := os.Getwd()
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
-		Quick:       true,
-		SkipRefresh: true,
-		Dir:         path.Join(cwd, ".", "ts-access-tokens"),
+		Quick: true,
+		Dir:   path.Join(cwd, ".", "ts-access-tokens"),
 		Dependencies: []string{
 			"@pulumi/pulumiservice",
 		},
@@ -26,9 +25,8 @@ func TestAccessTokenExample(t *testing.T) {
 func TestStackTagsExample(t *testing.T) {
 	cwd, _ := os.Getwd()
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
-		Quick:       true,
-		SkipRefresh: true,
-		Dir:         path.Join(cwd, ".", "ts-stack-tags"),
+		Quick: true,
+		Dir:   path.Join(cwd, ".", "ts-stack-tags"),
 		Dependencies: []string{
 			"@pulumi/pulumiservice",
 		},
@@ -54,9 +52,8 @@ func TestDeploymentSettingsExample(t *testing.T) {
 func TestTeamStackPermissionsExample(t *testing.T) {
 	cwd, _ := os.Getwd()
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
-		Quick:       true,
-		SkipRefresh: true,
-		Dir:         path.Join(cwd, ".", "ts-team-stack-permissions"),
+		Quick: true,
+		Dir:   path.Join(cwd, ".", "ts-team-stack-permissions"),
 		Dependencies: []string{
 			"@pulumi/pulumiservice",
 		},
@@ -66,9 +63,8 @@ func TestTeamStackPermissionsExample(t *testing.T) {
 func TestTeamsExample(t *testing.T) {
 	cwd, _ := os.Getwd()
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
-		Quick:       true,
-		SkipRefresh: true,
-		Dir:         path.Join(cwd, ".", "ts-teams"),
+		Quick: true,
+		Dir:   path.Join(cwd, ".", "ts-teams"),
 		Dependencies: []string{
 			"@pulumi/pulumiservice",
 		},

examples/examples_python_test.go

@@ -12,8 +12,7 @@ import (
 
 func TestPythonTeamsExample(t *testing.T) {
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
-		Dir:         path.Join(getCwd(t), "py-teams"),
-		SkipRefresh: true,
+		Dir: path.Join(getCwd(t), "py-teams"),
 		Dependencies: []string{
 			filepath.Join("..", "sdk", "python", "bin"),
 		},
@@ -36,8 +35,7 @@ func TestPythonDeploymentSettingsExample(t *testing.T) {
 func TestPythonEnvironmentsExample(t *testing.T) {
 	digits := generateRandomFiveDigits()
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
-		Dir:         path.Join(getCwd(t), "py-environments"),
-		SkipRefresh: true,
+		Dir: path.Join(getCwd(t), "py-environments"),
 		Config: map[string]string{
 			"digits": digits,
 		},

examples/examples_yaml_test.go

@@ -141,9 +141,8 @@ func TestYamlStackTagsExample(t *testing.T) {
 	require.NoError(t, err)
 
 	integration.ProgramTest(t, &integration.ProgramTestOptions{
-		Quick:       true,
-		SkipRefresh: true,
-		Dir:         tmpdir,
+		Quick: true,
+		Dir:   tmpdir,
 		EditDirs: []integration.EditDir{
 			{
 				Dir: path.Join(cwd, ".", "yaml-stack-tags"),

examples/go-environments/main.go

@@ -11,15 +11,16 @@ import (
 func main() {
 	pulumi.Run(func(ctx *pulumi.Context) error {
 		conf := config.New(ctx, "")
+		yaml := `values:
+  myKey1: "myValue1"
+  myNestedKey:
+    myKey2: "myValue2"
+    myNumber: 1`
+
 		environment, err := pulumiservice.NewEnvironment(ctx, "testing-environment", &pulumiservice.EnvironmentArgs{
 			Name:         pulumi.String("testing-environment-go-" + conf.Require("digits")),
 			Organization: pulumi.String("service-provider-test-org"),
-			Yaml: pulumi.NewStringAsset(`
-            values:
-              myKey1: "myValue1"
-              myNestedKey:
-                myKey2: "myValue2"
-                myNumber: 1`),
+			Yaml:         pulumi.NewStringAsset(yaml),
 		})
 		if err != nil {
 			return err

examples/py-environments/__main__.py

@@ -10,13 +10,11 @@
     "testing-environment",
     organization="service-provider-test-org",
     name="testing-environment-py-"+config.require('digits'),
-    yaml=pulumi.StringAsset("""
-        values:
-          myKey1: "myValue1"
-          myNestedKey:
-            myKey2: "myValue2"
-            myNumber: 1
-    """)
+    yaml=pulumi.StringAsset("""values:
+  myKey1: "myValue1"
+  myNestedKey:
+    myKey2: "myValue2"
+    myNumber: 1""")
 )
 
 # A tag that will always be placed on the latest revision of the environment

provider/pkg/provider/access_token.go

@@ -135,7 +135,7 @@ func (at *PulumiServiceAccessTokenResource) deleteAccessToken(ctx context.Contex
 }
 
 func diffAccessTokenProperties(req *pulumirpc.DiffRequest, replaceProps []string) (*pulumirpc.DiffResponse, error) {
-	olds, err := plugin.UnmarshalProperties(req.GetOldInputs(), plugin.MarshalOptions{KeepUnknowns: false, SkipNulls: true})
+	olds, err := plugin.UnmarshalProperties(req.GetOlds(), plugin.MarshalOptions{KeepUnknowns: false, SkipNulls: true})
 	if err != nil {
 		return nil, err
 	}

provider/pkg/provider/deployment_schedules.go

@@ -352,6 +352,16 @@ func (st *PulumiServiceDeploymentScheduleResource) Read(req *pulumirpc.ReadReque
 		PulumiOperation: scheduleResponse.Definition.Request.PulumiOperation,
 	}
 
+	inputs, err := plugin.MarshalProperties(
+		input.ToPropertyMap(),
+		plugin.MarshalOptions{
+			KeepUnknowns: true,
+			SkipNulls:    true,
+		},
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read DeploymentSchedule (%q): %w", req.Id, err)
+	}
 	outputProperties, err := plugin.MarshalProperties(
 		AddScheduleIdToPropertyMap(*scheduleID, input.ToPropertyMap()),
 		plugin.MarshalOptions{
@@ -366,7 +376,7 @@ func (st *PulumiServiceDeploymentScheduleResource) Read(req *pulumirpc.ReadReque
 	return &pulumirpc.ReadResponse{
 		Id:         req.Id,
 		Properties: outputProperties,
-		Inputs:     outputProperties,
+		Inputs:     inputs,
 	}, nil
 }
 

provider/pkg/provider/drift_schedules.go

@@ -197,6 +197,16 @@ func (st *PulumiServiceDriftScheduleResource) Read(req *pulumirpc.ReadRequest) (
 		AutoRemediate: scheduleResponse.Definition.Request.OperationContext.Options.AutoRemediate,
 	}
 
+	inputs, err := plugin.MarshalProperties(
+		input.ToPropertyMap(),
+		plugin.MarshalOptions{
+			KeepUnknowns: true,
+			SkipNulls:    true,
+		},
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read DriftSchedule (%q): %w", req.Id, err)
+	}
 	outputProperties, err := plugin.MarshalProperties(
 		AddScheduleIdToPropertyMap(*scheduleID, input.ToPropertyMap()),
 		plugin.MarshalOptions{
@@ -211,7 +221,7 @@ func (st *PulumiServiceDriftScheduleResource) Read(req *pulumirpc.ReadRequest) (
 	return &pulumirpc.ReadResponse{
 		Id:         req.Id,
 		Properties: outputProperties,
-		Inputs:     outputProperties,
+		Inputs:     inputs,
 	}, nil
 }
 

provider/pkg/provider/environment.go

@@ -256,25 +256,43 @@ func (st *PulumiServiceEnvironmentResource) Update(req *pulumirpc.UpdateRequest)
 }
 
 func (st *PulumiServiceEnvironmentResource) Read(req *pulumirpc.ReadRequest) (*pulumirpc.ReadResponse, error) {
-	input, err := ToPulumiServiceEnvironmentInput(req.GetProperties())
-	if err != nil {
-		return nil, err
+	splitID := strings.Split(req.Id, "/")
+	if len(splitID) < 2 {
+		return nil, fmt.Errorf("invalid environment id: %s", req.Id)
 	}
+	orgName := splitID[0]
+	envName := splitID[1]
 
-	retrievedYaml, _, revision, err := st.client.GetEnvironment(context.Background(), input.OrgName, input.EnvName, "", false)
+	retrievedYaml, _, revision, err := st.client.GetEnvironment(context.Background(), orgName, envName, "", false)
 	if err != nil {
 		return &pulumirpc.ReadResponse{Id: "", Properties: nil}, nil
 	}
 
+	input := PulumiServiceEnvironmentInput{
+		OrgName: orgName,
+		EnvName: envName,
+		Yaml:    retrievedYaml,
+	}
+
 	result := PulumiServiceEnvironmentOutput{
-		input: PulumiServiceEnvironmentInput{
-			OrgName: input.OrgName,
-			EnvName: input.EnvName,
-			Yaml:    retrievedYaml,
-		},
+		input:    input,
 		revision: revision,
 	}
 
+	inputMap, err := input.ToPropertyMap()
+	if err != nil {
+		return nil, err
+	}
+	inputs, err := plugin.MarshalProperties(
+		inputMap,
+		plugin.MarshalOptions{
+			KeepSecrets: true,
+		},
+	)
+	if err != nil {
+		return nil, err
+	}
+
 	propertyMap, err := result.ToPropertyMap()
 	if err != nil {
 		return nil, err
@@ -292,7 +310,7 @@ func (st *PulumiServiceEnvironmentResource) Read(req *pulumirpc.ReadRequest) (*p
 	return &pulumirpc.ReadResponse{
 		Id:         req.Id,
 		Properties: properties,
-		Inputs:     properties,
+		Inputs:     inputs,
 	}, nil
 }
 

provider/pkg/provider/team.go

@@ -148,7 +148,16 @@ func (t *PulumiServiceTeamResource) Check(req *pulumirpc.CheckRequest) (*pulumir
 		})
 	}
 
-	return &pulumirpc.CheckResponse{Inputs: news, Failures: failures}, nil
+	if !newsMap["displayName"].HasValue() {
+		newsMap["displayName"] = newsMap["name"]
+	}
+
+	inputs, err := plugin.MarshalProperties(newsMap, plugin.MarshalOptions{KeepUnknowns: true, SkipNulls: true})
+	if err != nil {
+		return nil, err
+	}
+
+	return &pulumirpc.CheckResponse{Inputs: inputs, Failures: failures}, nil
 }
 
 func (t *PulumiServiceTeamResource) Delete(req *pulumirpc.DeleteRequest) (*pbempty.Empty, error) {

provider/pkg/provider/team_access_token.go

@@ -123,6 +123,9 @@ func (t *PulumiServiceTeamAccessTokenResource) Read(req *pulumirpc.ReadRequest)
 	urn := req.GetId()
 
 	orgName, teamName, _, tokenId, err := splitTeamAccessTokenId(urn)
+	if err != nil {
+		return nil, err
+	}
 
 	// the team access token is immutable; if we get nil it got deleted, otherwise all data is the same
 	accessToken, err := t.client.GetTeamAccessToken(ctx, tokenId, orgName, teamName)
@@ -136,6 +139,7 @@ func (t *PulumiServiceTeamAccessTokenResource) Read(req *pulumirpc.ReadRequest)
 	return &pulumirpc.ReadResponse{
 		Id:         req.GetId(),
 		Properties: req.GetProperties(),
+		Inputs:     req.GetProperties(),
 	}, nil
 }
 

provider/pkg/provider/ttl_schedules.go

@@ -205,6 +205,16 @@ func (st *PulumiServiceTtlScheduleResource) Read(req *pulumirpc.ReadRequest) (*p
 		DeleteAfterDestroy: scheduleResponse.Definition.Request.OperationContext.Options.DeleteAfterDestroy,
 	}
 
+	inputs, err := plugin.MarshalProperties(
+		input.ToPropertyMap(),
+		plugin.MarshalOptions{
+			KeepUnknowns: true,
+			SkipNulls:    true,
+		},
+	)
+	if err != nil {
+		return nil, fmt.Errorf("failed to read TtlSchedule (%q): %w", req.Id, err)
+	}
 	outputProperties, err := plugin.MarshalProperties(
 		AddScheduleIdToPropertyMap(*scheduleID, input.ToPropertyMap()),
 		plugin.MarshalOptions{
@@ -219,7 +229,7 @@ func (st *PulumiServiceTtlScheduleResource) Read(req *pulumirpc.ReadRequest) (*p
 	return &pulumirpc.ReadResponse{
 		Id:         req.Id,
 		Properties: outputProperties,
-		Inputs:     outputProperties,
+		Inputs:     inputs,
 	}, nil
 }