Rapid generator for schema-value pairs (#1801)

Fixes #1790 by
building a rapid generator for schemas and associated values.

Large-ish problem 1: I do not have it figured out how to test unknown
values. TF literals as unknown values are forbidden and do not make
sense. We might need a helper resource so that testing unknown values
generates references to an output of the helper resource. This is logged
for future work.

Large-ish problem 2: iteration is pretty slow (x-proc). Normal n=100
rapid tests can take up to 10min. Could try batching so several
resources are tried in one shot say 100 resources.

Large-ish problem 3: I'm not sure if no-op Update and Create
implementations are acceptable. There is something to testing Computed
attributes where provider has to set values. Possibly Update also needs
to set values? Possibly not.

Small problems:

- [x] Using TF JSON syntax didn't handle null/empty correctly; that is
now discarded, using actual HCL syntax
- [x] TF representations are difficult to visualize in failing tests and
difficult to assert against
- [x] Lots of lost-in-translation papercuts possible between
representations (cty.Value, resource.PropertyValue, tftypes.Value)
- [x] this requires a change to providertest to abstract from testing.T
so we can pass rapid.T
- [x] it's very hard to disable annoying TF logging, using env vars for
now

We are starting to find bugs and discrepancies from this work:

- #1856 panic
corner-case
- #1852 need to
InternalValidate
- #1828

Future work:

- #1856 
- #1857 
- #1858 
- #1859 
- #1860 
- #1861 
- #1862 
- #1863 
- #1864 
- #1865 
- #1866 
- #1867

pkg/tests/cross-tests/adapt.go

@@ -0,0 +1,201 @@
+// Copyright 2016-2024, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+// limitations under the License.
+
+// Adapters for converting morally equivalent typed representations of TF values for integrating with all the libraries
+// cross-testing is using.
+package crosstests
+
+import (
+	"math/big"
+
+	"github.com/hashicorp/terraform-plugin-go/tftypes"
+	"github.com/pulumi/pulumi/sdk/v3/go/common/util/contract"
+	"github.com/zclconf/go-cty/cty"
+)
+
+type typeAdapter struct {
+	typ tftypes.Type
+}
+
+func (ta *typeAdapter) ToCty() cty.Type {
+	t := ta.typ
+	switch {
+	case t.Is(tftypes.String):
+		return cty.String
+	case t.Is(tftypes.Number):
+		return cty.Number
+	case t.Is(tftypes.Bool):
+		return cty.Bool
+	case t.Is(tftypes.List{}):
+		return cty.List(fromType(t.(tftypes.List).ElementType).ToCty())
+	case t.Is(tftypes.Set{}):
+		return cty.Set(fromType(t.(tftypes.Set).ElementType).ToCty())
+	case t.Is(tftypes.Map{}):
+		return cty.Map(fromType(t.(tftypes.Map).ElementType).ToCty())
+	case t.Is(tftypes.Object{}):
+		fields := map[string]cty.Type{}
+		for k, v := range t.(tftypes.Object).AttributeTypes {
+			fields[k] = fromType(v).ToCty()
+		}
+		return cty.Object(fields)
+	default:
+		contract.Failf("unexpected type %v", t)
+		return cty.NilType
+	}
+}
+
+func (ta *typeAdapter) NewValue(value any) tftypes.Value {
+	t := ta.typ
+	if value == nil {
+		return tftypes.NewValue(t, nil)
+	}
+	switch t := value.(type) {
+	case tftypes.Value:
+		return t
+	case *tftypes.Value:
+		return *t
+	}
+	switch {
+	case t.Is(tftypes.List{}):
+		elT := t.(tftypes.List).ElementType
+		switch v := value.(type) {
+		case []any:
+			values := []tftypes.Value{}
+			for _, el := range v {
+				values = append(values, fromType(elT).NewValue(el))
+			}
+			return tftypes.NewValue(t, values)
+		}
+	case t.Is(tftypes.Set{}):
+		elT := t.(tftypes.Set).ElementType
+		switch v := value.(type) {
+		case []any:
+			values := []tftypes.Value{}
+			for _, el := range v {
+				values = append(values, fromType(elT).NewValue(el))
+			}
+			return tftypes.NewValue(t, values)
+		}
+	case t.Is(tftypes.Map{}):
+		elT := t.(tftypes.Map).ElementType
+		switch v := value.(type) {
+		case map[string]any:
+			values := map[string]tftypes.Value{}
+			for k, el := range v {
+				values[k] = fromType(elT).NewValue(el)
+			}
+			return tftypes.NewValue(t, values)
+		}
+	case t.Is(tftypes.Object{}):
+		aT := t.(tftypes.Object).AttributeTypes
+		switch v := value.(type) {
+		case map[string]any:
+			values := map[string]tftypes.Value{}
+			for k, el := range v {
+				values[k] = fromType(aT[k]).NewValue(el)
+			}
+			return tftypes.NewValue(t, values)
+		}
+	}
+	return tftypes.NewValue(t, value)
+}
+
+func fromType(t tftypes.Type) *typeAdapter {
+	return &typeAdapter{t}
+}
+
+type valueAdapter struct {
+	value tftypes.Value
+}
+
+func (va *valueAdapter) ToCty() cty.Value {
+	v := va.value
+	t := v.Type()
+	switch {
+	case v.IsNull():
+		return cty.NullVal(fromType(t).ToCty())
+	case !v.IsKnown():
+		return cty.UnknownVal(fromType(t).ToCty())
+	case t.Is(tftypes.String):
+		var s string
+		err := v.As(&s)
+		contract.AssertNoErrorf(err, "unexpected error converting string")
+		return cty.StringVal(s)
+	case t.Is(tftypes.Number):
+		var n *big.Float
+		err := v.As(&n)
+		contract.AssertNoErrorf(err, "unexpected error converting number")
+		return cty.NumberVal(n)
+	case t.Is(tftypes.Bool):
+		var b bool
+		err := v.As(&b)
+		contract.AssertNoErrorf(err, "unexpected error converting bool")
+		return cty.BoolVal(b)
+	case t.Is(tftypes.List{}):
+		var vals []tftypes.Value
+		err := v.As(&vals)
+		contract.AssertNoErrorf(err, "unexpected error converting list")
+		if len(vals) == 0 {
+			return cty.ListValEmpty(fromType(t).ToCty())
+		}
+		outVals := make([]cty.Value, len(vals))
+		for i, el := range vals {
+			outVals[i] = fromValue(el).ToCty()
+		}
+		return cty.ListVal(outVals)
+	case t.Is(tftypes.Set{}):
+		var vals []tftypes.Value
+		err := v.As(&vals)
+		if len(vals) == 0 {
+			return cty.SetValEmpty(fromType(t).ToCty())
+		}
+		contract.AssertNoErrorf(err, "unexpected error converting set")
+		outVals := make([]cty.Value, len(vals))
+		for i, el := range vals {
+			outVals[i] = fromValue(el).ToCty()
+		}
+		return cty.SetVal(outVals)
+	case t.Is(tftypes.Map{}):
+		var vals map[string]tftypes.Value
+		err := v.As(&vals)
+		if len(vals) == 0 {
+			return cty.MapValEmpty(fromType(t).ToCty())
+		}
+		contract.AssertNoErrorf(err, "unexpected error converting map")
+		outVals := make(map[string]cty.Value, len(vals))
+		for k, el := range vals {
+			outVals[k] = fromValue(el).ToCty()
+		}
+		return cty.MapVal(outVals)
+	case t.Is(tftypes.Object{}):
+		var vals map[string]tftypes.Value
+		err := v.As(&vals)
+		if len(vals) == 0 {
+			return cty.EmptyObjectVal
+		}
+		contract.AssertNoErrorf(err, "unexpected error converting object")
+		outVals := make(map[string]cty.Value, len(vals))
+		for k, el := range vals {
+			outVals[k] = fromValue(el).ToCty()
+		}
+		return cty.ObjectVal(outVals)
+	default:
+		contract.Failf("unexpected type %v", t)
+		return cty.NilVal
+	}
+}
+
+func fromValue(v tftypes.Value) *valueAdapter {
+	return &valueAdapter{v}
+}

pkg/tests/cross-tests/ci.go

@@ -0,0 +1,28 @@
+// Copyright 2016-2024, Pulumi Corporation.
+//
+// Licensed under the Apache License, Version 2.0 (the "License");
+// you may not use this file except in compliance with the License.
+// You may obtain a copy of the License at
+//
+//     http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing, software
+// distributed under the License is distributed on an "AS IS" BASIS,
+// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+// See the License for the specific language governing permissions and
+
+// Helpers to disable failing CI runs.
+package crosstests
+
+import (
+	"os"
+	"runtime"
+	"strings"
+	"testing"
+)
+
+func skipUnlessLinux(t *testing.T) {
+	if ci, ok := os.LookupEnv("CI"); ok && ci == "true" && !strings.Contains(strings.ToLower(runtime.GOOS), "linux") {
+		t.Skip("Skipping on non-Linux platforms as our CI does not yet install Terraform CLI required for these tests")
+	}
+}