Merge pull request #351 from accurics/feature/support-resolve-variabl…

…e-references

Feature/support resolve variable references

pkg/iac-providers/terraform/v12/cty-converters.go

@@ -0,0 +1,96 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+package tfv12
+
+import (
+	"fmt"
+	"github.com/pkg/errors"
+	"github.com/zclconf/go-cty/cty"
+	"github.com/zclconf/go-cty/cty/gocty"
+)
+
+// list of available cty to golang type converters
+var (
+	ctyConverterFuncs       = []func(cty.Value) (interface{}, error){ctyToStr, ctyToInt, ctyToBool, ctyToSlice, ctyToMap}
+	ctyNativeConverterFuncs = []func(cty.Value) (interface{}, error){ctyToStr, ctyToInt, ctyToBool, ctyToSlice}
+)
+
+// ctyToStr tries to convert the given cty.Value into golang string type
+func ctyToStr(ctyVal cty.Value) (interface{}, error) {
+	var val string
+	err := gocty.FromCtyValue(ctyVal, &val)
+	return val, err
+}
+
+// ctyToInt tries to convert the given cty.Value into golang int type
+func ctyToInt(ctyVal cty.Value) (interface{}, error) {
+	var val int
+	err := gocty.FromCtyValue(ctyVal, &val)
+	return val, err
+}
+
+// ctyToBool tries to convert the given cty.Value into golang bool type
+func ctyToBool(ctyVal cty.Value) (interface{}, error) {
+	var val bool
+	err := gocty.FromCtyValue(ctyVal, &val)
+	return val, err
+}
+
+// ctyToSlice tries to convert the given cty.Value into golang slice of
+// interfce{}
+func ctyToSlice(ctyVal cty.Value) (interface{}, error) {
+	var val []interface{}
+	err := gocty.FromCtyValue(ctyVal, &val)
+	return val, err
+}
+
+// ctyToMap tries to converts the incoming cty.Value into map[string]cty.Value
+// then for every key value of this map, tries to convert the cty.Value into
+// native golang value and create a new map[string]interface{}
+func ctyToMap(ctyVal cty.Value) (interface{}, error) {
+
+	var (
+		ctyValMap = ctyVal.AsValueMap() // map[string]cty.Value
+		val       = make(map[string]interface{})
+		allErrs   error
+	)
+
+	// cannot process an empty ctValMap
+	if len(ctyValMap) < 1 {
+		return nil, fmt.Errorf("empty ctyValMap")
+	}
+
+	// iterate over every key cty.Value pair, try to convert cty.Value into
+	// golang value
+	for k, v := range ctyValMap {
+		// convert cty.Value to native golang type based on cty.Type
+		for _, converter := range ctyNativeConverterFuncs {
+			resolved, err := converter(v)
+			if err == nil {
+				val[k] = resolved
+				break
+			}
+			allErrs = errors.Wrap(allErrs, err.Error())
+		}
+	}
+	if allErrs != nil {
+		return nil, allErrs
+	}
+
+	// hopefully successful!
+	return val, nil
+}

pkg/iac-providers/terraform/v12/load-dir.go

@@ -37,6 +37,15 @@ var (
 	errBuildTFConfigDir = fmt.Errorf("failed to build terraform allResourcesConfig")
 )
 
+// ModuleConfig contains the *hclConfigs.Config for every module in the
+// unified config tree along with *hclConfig.ModuleCall made by the parent
+// module. The ParentModuleCall helps in resolving references for variables
+// initilaized in the parent ModuleCall
+type ModuleConfig struct {
+	Config           *hclConfigs.Config
+	ParentModuleCall *hclConfigs.ModuleCall
+}
+
 // LoadIacDir starts traversing from the given rootDir and traverses through
 // all the descendant modules present to create an output list of all the
 // resources present in rootDir and descendant modules
@@ -113,7 +122,8 @@ func (*TfV12) LoadIacDir(absRootDir string) (allResourcesConfig output.AllResour
 	*/
 
 	// queue of for BFS, add root module config to it
-	configsQ := []*hclConfigs.Config{unified.Root}
+	root := &ModuleConfig{Config: unified.Root}
+	configsQ := []*ModuleConfig{root}
 
 	// initialize normalized output
 	allResourcesConfig = make(map[string][]output.ResourceConfig)
@@ -126,15 +136,22 @@ func (*TfV12) LoadIacDir(absRootDir string) (allResourcesConfig output.AllResour
 		current := configsQ[0]
 		configsQ = configsQ[1:]
 
+		// reference resolver
+		r := NewRefResolver(current.Config, current.ParentModuleCall)
+
 		// traverse through all current's resources
-		for _, managedResource := range current.Module.ManagedResources {
+		for _, managedResource := range current.Config.Module.ManagedResources {
 
 			// create output.ResourceConfig from hclConfigs.Resource
 			resourceConfig, err := CreateResourceConfig(managedResource)
 			if err != nil {
 				return allResourcesConfig, fmt.Errorf("failed to create ResourceConfig")
 			}
 
+			// resolve references
+			resourceConfig.Config = r.ResolveRefs(resourceConfig.Config.(jsonObj))
+
+			// source file path
 			resourceConfig.Source, err = filepath.Rel(absRootDir, resourceConfig.Source)
 			if err != nil {
 				return allResourcesConfig, fmt.Errorf("failed to get resource: %s", err)
@@ -149,8 +166,12 @@ func (*TfV12) LoadIacDir(absRootDir string) (allResourcesConfig output.AllResour
 		}
 
 		// add all current's children to the queue
-		for _, childModule := range current.Children {
-			configsQ = append(configsQ, childModule)
+		for childName, childModule := range current.Config.Children {
+			childModuleConfig := &ModuleConfig{
+				Config:           childModule,
+				ParentModuleCall: current.Config.Module.ModuleCalls[childName],
+			}
+			configsQ = append(configsQ, childModuleConfig)
 		}
 	}
 

pkg/iac-providers/terraform/v12/local-references.go

@@ -0,0 +1,99 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+package tfv12
+
+import (
+	"io/ioutil"
+	"reflect"
+	"regexp"
+	"strings"
+
+	"github.com/hashicorp/hcl/v2/hclsyntax"
+	"go.uber.org/zap"
+)
+
+var (
+	// reference patterns
+	localRefPattern = regexp.MustCompile(`(\$\{)?local\.(?P<name>\w*)(\})?`)
+)
+
+// isLocalRef returns true if the given string is a local value reference
+func isLocalRef(attrVal string) bool {
+	return localRefPattern.MatchString(attrVal)
+}
+
+// getLocalName returns the actual local value name as configured in IaC. It
+// trims of "${local." prefix and "}" suffix and returns the local value name
+func getLocalName(localRef string) (string, string) {
+
+	// 1. extract the exact local value reference from the string
+	localExpr := localRefPattern.FindString(localRef)
+
+	// 2. extract local value name from local value reference
+	match := localRefPattern.FindStringSubmatch(localRef)
+	result := make(map[string]string)
+	for i, name := range localRefPattern.SubexpNames() {
+		if i != 0 && name != "" {
+			result[name] = match[i]
+		}
+	}
+	localName := result["name"]
+
+	zap.S().Debugf("extracted local value name %q from reference %q", localName, localRef)
+	return localName, localExpr
+}
+
+// ResolveLocalRef returns the local value as configured in IaC config in module
+func (r *RefResolver) ResolveLocalRef(localRef string) interface{} {
+
+	// get local name from localRef
+	localName, localExpr := getLocalName(localRef)
+
+	// check if local name exists in the map of locals read from IaC
+	localAttr, present := r.Config.Module.Locals[localName]
+	if !present {
+		zap.S().Debugf("local name: %q, ref: %q not present in locals", localName, localRef)
+		return localRef
+	}
+
+	// read source file
+	fileBytes, err := ioutil.ReadFile(localAttr.DeclRange.Filename)
+	if err != nil {
+		zap.S().Errorf("failed to read terrafrom IaC file '%s'. error: '%v'", localAttr.DeclRange.Filename, err)
+		return localRef
+	}
+
+	// extract values from attribute expressions as golang interface{}
+	c := converter{bytes: fileBytes}
+	val, err := c.convertExpression(localAttr.Expr.(hclsyntax.Expression))
+	if err != nil {
+		zap.S().Errorf("failed to convert expression '%v', ref: '%v'", localAttr.Expr, localRef)
+		return localRef
+	}
+
+	// replace the local value reference string with actual value
+	if reflect.TypeOf(val).Kind() == reflect.String {
+		valStr := val.(string)
+		resolvedVal := strings.Replace(localRef, localExpr, valStr, 1)
+		zap.S().Debugf("resolved str local value ref: '%v', value: '%v'", localRef, resolvedVal)
+		return r.ResolveStrRef(resolvedVal)
+	}
+
+	// return extracted value
+	zap.S().Debugf("resolved local value ref: '%v', value: '%v'", localRef, val)
+	return val
+}

pkg/iac-providers/terraform/v12/lookup-references.go

@@ -0,0 +1,91 @@
+/*
+    Copyright (C) 2020 Accurics, Inc.
+
+	Licensed under the Apache License, Version 2.0 (the "License");
+    you may not use this file except in compliance with the License.
+    You may obtain a copy of the License at
+
+		http://www.apache.org/licenses/LICENSE-2.0
+
+	Unless required by applicable law or agreed to in writing, software
+    distributed under the License is distributed on an "AS IS" BASIS,
+    WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+    See the License for the specific language governing permissions and
+    limitations under the License.
+*/
+
+package tfv12
+
+import (
+	"reflect"
+	"regexp"
+
+	"go.uber.org/zap"
+)
+
+var (
+	// reference patterns
+	lookupRefPattern = regexp.MustCompile(`(\$\{)?lookup\((?P<table>\S+)\,\s*?(?P<key>\S+)\)(\})?`)
+)
+
+// isLookupRef returns true if the given string is a lookup value reference
+func isLookupRef(attrVal string) bool {
+	return lookupRefPattern.MatchString(attrVal)
+}
+
+// getLookupName returns the actual lookup value name as configured in IaC. It
+// trims of "${lookup(." prefix and ")}" suffix and returns the lookup value name
+func getLookupName(lookupRef string) (string, string, string) {
+
+	// 1. extract the exact lookup value reference from the string
+	lookupExpr := lookupRefPattern.FindString(lookupRef)
+
+	// 2. extract lookup value name from lookup value reference
+	match := lookupRefPattern.FindStringSubmatch(lookupRef)
+	result := make(map[string]string)
+	for i, name := range lookupRefPattern.SubexpNames() {
+		if i != 0 && name != "" {
+			result[name] = match[i]
+		}
+	}
+	table := result["table"]
+	key := result["key"]
+
+	zap.S().Debugf("extracted lookup table %q key %q from reference %q", table, key, lookupRef)
+	return table, key, lookupExpr
+}
+
+// ResolveLookupRef returns the lookup value as configured in IaC config in module
+func (r *RefResolver) ResolveLookupRef(lookupRef string) interface{} {
+
+	// get lookup name from lookupRef
+	table, key, _ := getLookupName(lookupRef)
+
+	// resolve key, if it is a reference
+	resolvedKey := r.ResolveStrRef(key)
+
+	// check if key is still an unresolved reference
+	if reflect.TypeOf(resolvedKey).Kind() == reflect.String && isRef(resolvedKey.(string)) {
+		zap.S().Debugf("failed to resolve key ref: '%v'", key)
+		return lookupRef
+	}
+
+	// resolve table, if it is a ref
+	lookup := r.ResolveStrRef(table)
+
+	// check if lookup is a map
+	if reflect.TypeOf(lookup).String() != "map[string]interface {}" {
+		zap.S().Debugf("failed to resolve lookup ref %q, table name %q into a map, received %v", lookupRef, table, reflect.TypeOf(lookup).String())
+		return lookupRef
+	}
+
+	// check if key is present in lookup table
+	resolved, ok := lookup.(map[string]interface{})[resolvedKey.(string)]
+	if !ok {
+		zap.S().Debugf("key %q not present in lookup table %v", key, lookup)
+		return lookupRef
+	}
+
+	zap.S().Debugf("resolved lookup ref %q to value %v", lookupRef, resolved)
+	return resolved
+}