Move DCL resources to the service packages (#8031)

* Move DCL resources to the service package

* Copy DCL helper files into the folder tpgdclresource

tpgtools/handwritten.go

@@ -89,7 +89,7 @@ func copyHandwrittenFiles(inPath string, outPath string) {
 		}
 
 		// Write copied file.
-		err = ioutil.WriteFile(path.Join(outPath, terraformResourceDirectory, f.Name()), b, 0644)
+		err = ioutil.WriteFile(path.Join(outPath, terraformResourceDirectory, "tpgdclresource", f.Name()), b, 0644)
 		if err != nil {
 			glog.Exit(err)
 		}

tpgtools/handwritten/dcl.go

@@ -1,4 +1,4 @@
-package google
+package tpgdclresource
 
 import (
 	dcl "github.com/GoogleCloudPlatform/declarative-resource-client-library/dcl"

tpgtools/handwritten/expanders.go

@@ -1,31 +1,32 @@
-package google
+package tpgdclresource
 
 import (
 	"strings"
 
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	"github.com/hashicorp/terraform-provider-google/google/tpgresource"
 )
 
-func expandStringArray(v interface{}) []string {
+func ExpandStringArray(v interface{}) []string {
 	arr, ok := v.([]string)
 
 	if ok {
 		return arr
 	}
 
 	if arr, ok := v.(*schema.Set); ok {
-		return convertStringSet(arr)
+		return tpgresource.ConvertStringSet(arr)
 	}
 
-	arr = convertStringArr(v.([]interface{}))
+	arr = tpgresource.ConvertStringArr(v.([]interface{}))
 	if arr == nil {
 		// Send empty array specifically instead of nil
 		return make([]string, 0)
 	}
 	return arr
 }
 
-func expandIntegerArray(v interface{}) []int64 {
+func ExpandIntegerArray(v interface{}) []int64 {
 	arr, ok := v.([]int64)
 
 	if ok {
@@ -52,7 +53,7 @@ func convertIntegerArr(v []interface{}) []int64 {
 }
 
 // Returns the DCL representation of a three-state boolean value represented by a string in terraform.
-func expandEnumBool(v interface{}) *bool {
+func ExpandEnumBool(v interface{}) *bool {
 	s, ok := v.(string)
 	if !ok {
 		return nil

tpgtools/handwritten/expanders_test.go

@@ -1,4 +1,4 @@
-package google
+package tpgdclresource
 
 import (
 	"reflect"
@@ -66,7 +66,7 @@ func TestExpandEnumBool(t *testing.T) {
 		t.Run(tc.name, func(t *testing.T) {
 			t.Parallel()
 
-			if got, want := expandEnumBool(tc.input), tc.exp; !reflect.DeepEqual(got, want) {
+			if got, want := ExpandEnumBool(tc.input), tc.exp; !reflect.DeepEqual(got, want) {
 				t.Errorf("expected %v to be %v", got, want)
 			}
 		})

tpgtools/handwritten/flatteners.go

@@ -1,7 +1,7 @@
-package google
+package tpgdclresource
 
 // Returns the terraform representation of a three-state boolean value represented by a pointer to bool in DCL.
-func flattenEnumBool(v interface{}) string {
+func FlattenEnumBool(v interface{}) string {
 	b, ok := v.(*bool)
 	if !ok || b == nil {
 		return ""

tpgtools/handwritten/orgpolicy_utils.go

@@ -1,4 +1,4 @@
-package google
+package tpgdclresource
 
 import (
 	"fmt"
@@ -10,7 +10,7 @@ import (
 
 // OrgPolicyPolicy has a custom import method because the parent field needs to allow an additional forward slash
 // to represent the type of parent (e.g. projects/{project_id}).
-func resourceOrgPolicyPolicyCustomImport(d *schema.ResourceData, meta interface{}) error {
+func ResourceOrgPolicyPolicyCustomImport(d *schema.ResourceData, meta interface{}) error {
 	config := meta.(*transport_tpg.Config)
 	if err := tpgresource.ParseImportId([]string{
 		"^(?P<parent>[^/]+/?[^/]*)/policies/(?P<name>[^/]+)",

tpgtools/handwritten/tpgtools_utils.go

@@ -1,4 +1,4 @@
-package google
+package tpgdclresource
 
 import (
 	"fmt"
@@ -9,11 +9,11 @@ import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
 )
 
-func oldValue(old, new interface{}) interface{} {
+func OldValue(old, new interface{}) interface{} {
 	return old
 }
 
-func handleNotFoundDCLError(err error, d *schema.ResourceData, resourceName string) error {
+func HandleNotFoundDCLError(err error, d *schema.ResourceData, resourceName string) error {
 	if dcl.IsNotFound(err) {
 		log.Printf("[WARN] Removing %s because it's gone", resourceName)
 		// The resource doesn't exist anymore

tpgtools/main.go

@@ -123,6 +123,12 @@ func main() {
 		return
 	}
 
+	// Copy DCL helper files into the folder tpgdclresource to make it easier to remove these files later.
+	dirPath := path.Join(*oPath, terraformResourceDirectory, "tpgdclresource")
+	if err := os.MkdirAll(dirPath, os.ModePerm); err != nil {
+		glog.Error(fmt.Errorf("error creating Terraform tpgdclresource directory %v: %v", dirPath, err))
+	}
+
 	copyHandwrittenFiles(*cPath, *oPath)
 }
 
@@ -360,6 +366,14 @@ func loadOverrides(packagePath Filepath, fileName string) Overrides {
 	return overrides
 }
 
+func getParentDir(res *Resource) string {
+	servicePath := path.Join(*oPath, terraformResourceDirectory, "services", string(res.Package()))
+	if err := os.MkdirAll(servicePath, os.ModePerm); err != nil {
+		glog.Error(fmt.Errorf("error creating Terraform the service directory %v: %v", servicePath, err))
+	}
+	return servicePath
+}
+
 func generateResourceFile(res *Resource) {
 	// Generate resource file
 	tmplInput := ResourceInput{
@@ -391,7 +405,8 @@ func generateResourceFile(res *Resource) {
 		fmt.Printf("%v", string(formatted))
 	} else {
 		outname := fmt.Sprintf("resource_%s_%s.go", res.ProductName(), res.Name())
-		err := ioutil.WriteFile(path.Join(*oPath, terraformResourceDirectory, outname), formatted, 0644)
+		parentDir := getParentDir(res)
+		err = ioutil.WriteFile(path.Join(parentDir, outname), formatted, 0644)
 		if err != nil {
 			glog.Exit(err)
 		}

tpgtools/overrides/cloudbuild/beta/worker_pool.yaml

@@ -1,6 +1,6 @@
 - type: DIFF_SUPPRESS_FUNC
   field: network_config.peered_network
   details:
-    diffsuppressfunc: compareResourceNames
+    diffsuppressfunc: tpgresource.CompareResourceNames
 - type: EXCLUDE
   field: etag

tpgtools/overrides/cloudbuild/worker_pool.yaml

@@ -1,6 +1,6 @@
 - type: DIFF_SUPPRESS_FUNC
   field: network_config.peered_network
   details:
-    diffsuppressfunc: compareResourceNames
+    diffsuppressfunc: tpgresource.CompareResourceNames
 - type: EXCLUDE
   field: etag

tpgtools/overrides/compute/beta/firewall_policy_rule.yaml

@@ -1,4 +1,4 @@
 - type: DIFF_SUPPRESS_FUNC
   field: target_resources
   details:
-    diffsuppressfunc: compareSelfLinkOrResourceName
+    diffsuppressfunc: tpgresource.CompareSelfLinkOrResourceName

tpgtools/overrides/compute/firewall_policy_rule.yaml

@@ -1,4 +1,4 @@
 - type: DIFF_SUPPRESS_FUNC
   field: target_resources
   details:
-    diffsuppressfunc: compareSelfLinkOrResourceName
+    diffsuppressfunc: tpgresource.CompareSelfLinkOrResourceName

tpgtools/overrides/orgpolicy/beta/policy.yaml

@@ -1,6 +1,6 @@
 - type: CUSTOM_IMPORT_FUNCTION
   details:
-    function: resourceOrgPolicyPolicyCustomImport
+    function: tpgdclresource.ResourceOrgPolicyPolicyCustomImport
 - type: ENUM_BOOL
   field: spec.rules.allow_all
 - type: ENUM_BOOL

tpgtools/overrides/orgpolicy/policy.yaml

@@ -1,6 +1,6 @@
 - type: CUSTOM_IMPORT_FUNCTION
   details:
-    function: resourceOrgPolicyPolicyCustomImport
+    function: tpgdclresource.ResourceOrgPolicyPolicyCustomImport
 - type: ENUM_BOOL
   field: spec.rules.allow_all
 - type: ENUM_BOOL

tpgtools/property.go

@@ -222,7 +222,7 @@ func (p Property) DefaultStateGetter() string {
 }
 
 func (p Property) ChangeStateGetter() string {
-	return buildGetter(p, fmt.Sprintf("oldValue(d.GetChange(%q))", p.Name()))
+	return buildGetter(p, fmt.Sprintf("tpgdclresource.OldValue(d.GetChange(%q))", p.Name()))
 }
 
 // Builds a Getter for constructing a shallow
@@ -256,7 +256,7 @@ func buildGetter(p Property, rawGetter string) string {
 			return fmt.Sprintf("%s.%sEnumRef(%s.(string))", p.resource.Package(), p.ObjectType(), rawGetter)
 		}
 		if p.EnumBool {
-			return fmt.Sprintf("expandEnumBool(%s.(string))", rawGetter)
+			return fmt.Sprintf("tpgdclresource.ExpandEnumBool(%s.(string))", rawGetter)
 		}
 		if p.Computed {
 			return fmt.Sprintf("dcl.StringOrNil(%s.(string))", rawGetter)
@@ -279,11 +279,11 @@ func buildGetter(p Property, rawGetter string) string {
 			return fmt.Sprintf("expand%s%sArray(%s)", p.resource.PathType(), p.PackagePath(), rawGetter)
 		}
 		if p.Type.typ.Items != nil && p.Type.typ.Items.Type == "string" {
-			return fmt.Sprintf("expandStringArray(%s)", rawGetter)
+			return fmt.Sprintf("tpgdclresource.ExpandStringArray(%s)", rawGetter)
 		}
 
 		if p.Type.typ.Items != nil && p.Type.typ.Items.Type == "integer" {
-			return fmt.Sprintf("expandIntegerArray(%s)", rawGetter)
+			return fmt.Sprintf("tpgdclresource.ExpandIntegerArray(%s)", rawGetter)
 		}
 
 		if p.Type.typ.Items != nil && len(p.Properties) > 0 {
@@ -354,7 +354,7 @@ func (p Property) flattenGetterWithParent(parent string) string {
 		fallthrough
 	case SchemaTypeMap:
 		if p.EnumBool {
-			return fmt.Sprintf("flattenEnumBool(%s.%s)", parent, p.PackageName)
+			return fmt.Sprintf("tpgdclresource.FlattenEnumBool(%s.%s)", parent, p.PackageName)
 		}
 		return fmt.Sprintf("%s.%s", parent, p.PackageName)
 	case SchemaTypeList, SchemaTypeSet:
@@ -396,7 +396,7 @@ func (p Property) DefaultDiffSuppress() *string {
 	case SchemaTypeString:
 		// Field is reference to another resource
 		if _, ok := p.typ.Extension["x-dcl-references"]; ok {
-			dsf := "compareSelfLinkOrResourceName"
+			dsf := "tpgresource.CompareSelfLinkOrResourceName"
 			return &dsf
 		}
 	}
@@ -629,7 +629,7 @@ func createPropertiesFromSchema(schema *openapi.Schema, typeFetcher *TypeFetcher
 				i := Type{typ: v.Items}
 				e := fmt.Sprintf("&schema.Schema{Type: schema.%s}", i.String())
 				if _, ok := v.Extension["x-dcl-references"]; ok {
-					e = fmt.Sprintf("&schema.Schema{Type: schema.%s, DiffSuppressFunc: compareSelfLinkOrResourceName, }", i.String())
+					e = fmt.Sprintf("&schema.Schema{Type: schema.%s, DiffSuppressFunc: tpgresource.CompareSelfLinkOrResourceName, }", i.String())
 				}
 				p.Elem = &e
 				p.ElemIsBasicType = true
@@ -697,7 +697,7 @@ func createPropertiesFromSchema(schema *openapi.Schema, typeFetcher *TypeFetcher
 				if p.customName != "" {
 					propertyName = p.customName
 				}
-				ig := fmt.Sprintf("get%s(d, config)", renderSnakeAsTitle(miscellaneousNameSnakeCase(propertyName)))
+				ig := fmt.Sprintf("tpgresource.Get%s(d, config)", renderSnakeAsTitle(miscellaneousNameSnakeCase(propertyName)))
 				if cigOk {
 					ig = fmt.Sprintf("%s(d, config)", cig.Function)
 				}
@@ -837,9 +837,9 @@ func createPropertiesFromSchema(schema *openapi.Schema, typeFetcher *TypeFetcher
 			} else {
 				parent = "obj"
 			}
-			enumBoolSS := fmt.Sprintf("d.Set(%q, flattenEnumBool(%s.%s))", p.Name(), parent, p.PackageName)
+			enumBoolSS := fmt.Sprintf("d.Set(%q, tpgdclresource.FlattenEnumBool(%s.%s))", p.Name(), parent, p.PackageName)
 			p.StateSetter = &enumBoolSS
-			enumBoolSG := fmt.Sprintf("expandEnumBool(d.Get(%q))", p.Name())
+			enumBoolSG := fmt.Sprintf("tpgdclresource.ExpandEnumBool(d.Get(%q))", p.Name())
 			p.StateGetter = &enumBoolSG
 		}
 

tpgtools/templates/provider_dcl_resources.go.tmpl

@@ -33,12 +33,17 @@ package google
 
 import (
 	"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+	{{- range $res := . }}
+		{{- if not $res.SkipInProvider }}
+			"github.com/hashicorp/terraform-provider-google/google/services/{{$res.Package}}"
+		{{- end }}
+	{{- end }}
 )
 
 var dclResources = map[string]*schema.Resource{
 {{- range $res := . }}
 	{{- if not $res.SkipInProvider }}
-	"{{$res.TerraformName}}": Resource{{$res.PathType}}(),
+	"{{$res.TerraformName}}": {{$res.Package}}.Resource{{$res.PathType}}(),
 	{{- end }}
 {{- end }}
 }

tpgtools/templates/resource.go.tmpl

@@ -29,7 +29,7 @@
 //
 // ----------------------------------------------------------------------------
 
-package google
+package {{$.Package}}
 
 import(
 	"context"
@@ -51,6 +51,7 @@ import(
 	dcl "github.com/GoogleCloudPlatform/declarative-resource-client-library/dcl"
 	{{$.Package}} "github.com/GoogleCloudPlatform/declarative-resource-client-library/services/google/{{$.DCLPackage}}"
 
+	"github.com/hashicorp/terraform-provider-google/google/tpgdclresource"
 	"github.com/hashicorp/terraform-provider-google/google/tpgresource"
 	transport_tpg "github.com/hashicorp/terraform-provider-google/google/transport"
 )
@@ -258,10 +259,10 @@ should be converted to use the DCL's ID method, so normalization can be uniform.
 {{- if $.CustomCreateDirectiveFunction }}
 	directive := {{ $.CustomCreateDirectiveFunction }}(obj)
 {{- else if $.HasCreate }}
-	directive := CreateDirective
+	directive := tpgdclresource.CreateDirective
 {{- else }}
 	{{/* Resource has no create method, so we skip the BlockModification parameter. */}}
-	directive := UpdateDirective
+	directive := tpgdclresource.UpdateDirective
 {{- end }}
 	userAgent, err :=  tpgresource.GenerateUserAgentString(d, config.UserAgent)
 	if err != nil {
@@ -383,7 +384,7 @@ func resource{{$.PathType}}Read(d *schema.ResourceData, meta interface{}) error
 	res, err := client.Get{{$.DCLTitle}}(context.Background(), obj)
 	if err != nil {
 		resourceName := fmt.Sprintf("{{$.PathType}} %q", d.Id())
-		return handleNotFoundDCLError(err, d, resourceName)
+		return tpgdclresource.HandleNotFoundDCLError(err, d, resourceName)
 	}
 
 {{ range $v := .Properties -}}
@@ -438,7 +439,7 @@ func resource{{$.PathType}}Update(d *schema.ResourceData, meta interface{}) erro
 	defer transport_tpg.MutexStore.Unlock(lockName)
 
 {{ end }}
-	directive := UpdateDirective
+	directive := tpgdclresource.UpdateDirective
 	{{- if $.StateHint }}
 	directive = append(directive, dcl.WithStateHint(old))
 	{{- end }}