Add support for CFT nested stacks (tenable#949)

* Add support for CFT nested stacks

* Review Changes

go.mod

@@ -9,8 +9,12 @@ replace (
 )
 
 require (
-	github.com/BurntSushi/toml v0.4.0 // indirect
+	github.com/BurntSushi/toml v0.4.1 // indirect
 	github.com/VerbalExpressions/GoVerbalExpressions v0.0.0-20200410162751-4d76a1099a6e
+	github.com/aws/aws-sdk-go-v2/config v1.5.0
+	github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.2
+	github.com/aws/aws-sdk-go-v2/service/s3 v1.11.1
+	github.com/aws/smithy-go v1.6.0
 	github.com/awslabs/goformation/v4 v4.19.1
 	github.com/ghodss/yaml v1.0.0
 	github.com/go-errors/errors v1.0.1

go.sum

@@ -122,18 +122,9 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt
 github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU=
 github.com/Azure/go-ntlmssp v0.0.0-20180810175552-4a21cbd618b4/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
 github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU=
-github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ=
 github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU=
-github.com/BurntSushi/toml v0.3.2-0.20210614224209-34d990aa228d/go.mod h1:2QZjSXA5e+XyFeCAxxtL8Z4StYUsTquL8ODGPR3C3MA=
-github.com/BurntSushi/toml v0.3.2-0.20210621044154-20a94d639b8e/go.mod h1:t4zg8TkHfP16Vb3x4WKIw7zVYMit5QFtPEO8lOWxzTg=
-github.com/BurntSushi/toml v0.3.2-0.20210624061728-01bfc69d1057/go.mod h1:NMj2lD5LfMqcE0w8tnqOsH6944oaqpI1974lrIwerfE=
-github.com/BurntSushi/toml v0.3.2-0.20210704081116-ccff24ee4463/go.mod h1:EkRrMiQQmfxK6kIldz3QbPlhmVkrjW1RDJUnbDqGYvc=
-github.com/BurntSushi/toml v0.4.0 h1:qD/r9AL67srjW6O3fcSKZDsXqzBNX6ieSRywr2hRrdE=
-github.com/BurntSushi/toml v0.4.0/go.mod h1:wtejDu7Q0FhCWAo2aXkywSJyYFg01EDTKozLNCz2JBA=
-github.com/BurntSushi/toml-test v0.1.1-0.20210620192437-de01089bbf76/go.mod h1:P/PrhmZ37t5llHfDuiouWXtFgqOoQ12SAh9j6EjrBR4=
-github.com/BurntSushi/toml-test v0.1.1-0.20210624055653-1f6389604dc6/go.mod h1:UAIt+Eo8itMZAAgImXkPGDMYsT1SsJkVdB5TuONl86A=
-github.com/BurntSushi/toml-test v0.1.1-0.20210704062846-269931e74e3f/go.mod h1:fnFWrIwqgHsEjVsW3RYCJmDo86oq9eiJ9u6bnqhtm2g=
-github.com/BurntSushi/toml-test v0.1.1-0.20210723065233-facb9eccd4da/go.mod h1:ve9Q/RRu2vHi42LocPLNvagxuUJh993/95b18bw/Nws=
+github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw=
+github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ=
 github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo=
 github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4=
 github.com/ChrisTrenkamp/goxpath v0.0.0-20190607011252-c5096ec8773d/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4=
@@ -260,6 +251,32 @@ github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU
 github.com/aws/aws-sdk-go v1.37.0 h1:GzFnhOIsrGyQ69s7VgqtrG2BG8v7X7vwB3Xpbd/DBBk=
 github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro=
 github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g=
+github.com/aws/aws-sdk-go-v2 v1.7.1 h1:TswSc7KNqZ/K1Ijt3IkpXk/2+62vi3Q82Yrr5wSbRBQ=
+github.com/aws/aws-sdk-go-v2 v1.7.1/go.mod h1:L5LuPC1ZgDr2xQS7AmIec/Jlc7O/Y1u2KxJyNVab250=
+github.com/aws/aws-sdk-go-v2/config v1.5.0 h1:tRQcWXVmO7wC+ApwYc2LiYKfIBoIrdzcJ+7HIh6AlR0=
+github.com/aws/aws-sdk-go-v2/config v1.5.0/go.mod h1:RWlPOAW3E3tbtNAqTwvSW54Of/yP3oiZXMI0xfUdjyA=
+github.com/aws/aws-sdk-go-v2/credentials v1.3.1 h1:fFeqL5+9kwFKsCb2oci5yAIDsWYqn/Nga8oQ5bIasI8=
+github.com/aws/aws-sdk-go-v2/credentials v1.3.1/go.mod h1:r0n73xwsIVagq8RsxmZbGSRQFj9As3je72C2WzUIToc=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0 h1:s4vtv3Mv1CisI3qm2HGHi1Ls9ZtbCOEqeQn6oz7fTyU=
+github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0/go.mod h1:2LAuqPx1I6jNfaGDucWfA2zqQCYCOMCDHiCOciALyNw=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.2 h1:fzEMxnHQWh+bUV0ZzfhMbgUG8zjIPnAgApjtdHtC9Yg=
+github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.2/go.mod h1:qaqQiHSrOUVOfKe6fhgQ6UzhxjwqVW8aHNegd6Ws4w4=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1 h1:SDLwr1NKyowP7uqxuLNdvFZhjnoVWxNv456zAp+ZFjU=
+github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1/go.mod h1:Zy8smImhTdOETZqfyn01iNOe0CNggVbPjCajyaz6Gvg=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.1 h1:s/uV8UyMB4UcO0ERHxG9BJhYJAD9MiY0QeYvJmlC7PE=
+github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.1/go.mod h1:v33JQ57i2nekYTA70Mb+O18KeH4KqhdqxTJZNK1zdRE=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1 h1:VJe/XEhrfyfBLupcGg1BfUSK2VMZNdbDcZQ49jnp+h0=
+github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1/go.mod h1:zceowr5Z1Nh2WVP8bf/3ikB41IZW59E4yIYbg+pC6mw=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.1 h1:1ds3HkMQEBx9XvOkqsPuqBmNFn0w8XEDuB4LOi6KepU=
+github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.1/go.mod h1:6EQZIwNNvHpq/2/QSJnp4+ECvqIy55w95Ofs0ze+nGQ=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.11.1 h1:HiXhafnqG0AkVJIZA/BHhFvuc/8xFdUO1uaeqF2Artc=
+github.com/aws/aws-sdk-go-v2/service/s3 v1.11.1/go.mod h1:XLAGFrEjbvMCLvAtWLLP32yTv8GpBquCApZEycDLunI=
+github.com/aws/aws-sdk-go-v2/service/sso v1.3.1 h1:H2ZLWHUbbeYtghuqCY5s/7tbBM99PAwCioRJF8QvV/U=
+github.com/aws/aws-sdk-go-v2/service/sso v1.3.1/go.mod h1:J3A3RGUvuCZjvSuZEcOpHDnzZP/sKbhDWV2T1EOzFIM=
+github.com/aws/aws-sdk-go-v2/service/sts v1.6.0 h1:Y9r6mrzOyAYz4qKaluSH19zqH1236il/nGbsPKOUT0s=
+github.com/aws/aws-sdk-go-v2/service/sts v1.6.0/go.mod h1:q7o0j7d7HrJk/vr9uUt3BVRASvcU7gYZB9PUgPiByXg=
+github.com/aws/smithy-go v1.6.0 h1:T6puApfBcYiTIsaI+SYWqanjMt5pc3aoyyDrI+0YH54=
+github.com/aws/smithy-go v1.6.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E=
 github.com/awslabs/goformation/v4 v4.19.1 h1:xqCDM4+gtkUNmxe1xP3LyH0X7EDMBR4HR1bqHUiMB7o=
 github.com/awslabs/goformation/v4 v4.19.1/go.mod h1:ygNqNsr904Q/Jan2A6ZKw9ewZWDTL9zlclZx2JzZhlM=
 github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I=
@@ -664,8 +681,10 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/
 github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
-github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo=
 github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
+github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ=
+github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE=
 github.com/google/go-containerregistry v0.0.0-20191010200024-a3d713f9b7f8/go.mod h1:KyKXa9ciM8+lgMXwOVsXi7UxGrsf9mM61Mzs+xKUrKE=
 github.com/google/go-containerregistry v0.1.2/go.mod h1:GPivBPgdAyd2SU+vf6EpsgOtWDuPqjW0hJZt4rNdTZ4=
 github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ=

pkg/iac-providers/cft/v1/load-file.go

@@ -25,6 +25,8 @@ import (
 
 	"github.com/accurics/terrascan/pkg/iac-providers/output"
 	"github.com/accurics/terrascan/pkg/mapper"
+	cftRes "github.com/accurics/terrascan/pkg/mapper/iac-providers/cft/config"
+	"github.com/accurics/terrascan/pkg/mapper/iac-providers/cft/store"
 	"github.com/awslabs/goformation/v4"
 	"github.com/awslabs/goformation/v4/cloudformation"
 	"go.uber.org/zap"
@@ -40,44 +42,102 @@ func (a *CFTV1) LoadIacFile(absFilePath string, options map[string]interface{})
 	}
 
 	// parse the file as cloudformation.Template
-	fileExt := a.getFileType(absFilePath, &fileData)
-	var template *cloudformation.Template
+	configs, err := a.getConfig(absFilePath, &fileData, nil)
+	if err != nil {
+		zap.S().Debug("unable to normalize data", zap.Error(err), zap.String("file", absFilePath))
+		return allResourcesConfig, err
+	}
+
+	// fill AllResourceConfigs
+	allResourcesConfig = make(map[string][]output.ResourceConfig)
+	var config *output.ResourceConfig
+	for _, resource := range configs {
+		config = &resource
+		config.Line = 1
+		if config.Source == "" {
+			config.Source = a.getSourceRelativePath(absFilePath)
+		}
+		allResourcesConfig[config.Type] = append(allResourcesConfig[config.Type], *config)
+	}
+	return allResourcesConfig, nil
+}
+
+func (a *CFTV1) getConfig(absFilePath string, fileData *[]byte, parameters *map[string]string) ([]output.ResourceConfig, error) {
+	// parse the file as cloudformation.Template
+	template, err := a.extractTemplate(absFilePath, fileData)
+	if err != nil {
+		return nil, err
+	}
+
+	// replace template parameter values
+	if parameters != nil {
+		for key, value := range *parameters {
+			if parameter, ok := template.Parameters[key]; ok {
+				parameter.Default = value
+				template.Parameters[key] = parameter
+			}
+		}
+	}
+
+	// map resource to a terrascan type
+	configs, err := a.translateResources(template, absFilePath)
+	if err != nil {
+		zap.S().Debug("unable to normalize data", zap.Error(err), zap.String("file", absFilePath))
+		return nil, err
+	}
+	return configs, nil
+}
+
+func (a *CFTV1) extractTemplate(file string, data *[]byte) (*cloudformation.Template, error) {
+	fileExt := a.getFileType(file, data)
+
 	switch fileExt {
 	case YAMLExtension, YAMLExtension2:
-		template, err = goformation.ParseYAML(fileData)
+		template, err := goformation.ParseYAML(*data)
 		if err != nil {
-			zap.S().Debug("failed to parse file", zap.String("file", absFilePath))
-			return allResourcesConfig, err
+			zap.S().Debug("failed to parse file", zap.String("file", file))
+			return nil, err
 		}
+		return template, nil
 	case JSONExtension:
-		template, err = goformation.ParseJSON(fileData)
+		template, err := goformation.ParseJSON(*data)
 		if err != nil {
-			zap.S().Debug("failed to parse file", zap.String("file", absFilePath))
-			return allResourcesConfig, err
+			zap.S().Debug("failed to parse file", zap.String("file", file))
+			return nil, err
 		}
+		return template, nil
 	default:
 		zap.S().Debug("unknown extension found", zap.String("extension", fileExt))
-		return allResourcesConfig, fmt.Errorf("unsupported extension for file %s", absFilePath)
+		return nil, fmt.Errorf("unsupported extension for file %s", file)
 	}
+}
 
-	// map resource to a terrascan type
+func (a *CFTV1) translateResources(template *cloudformation.Template, absFilePath string) ([]output.ResourceConfig, error) {
 	m := mapper.NewMapper("cft")
 	configs, err := m.Map(template)
 	if err != nil {
 		zap.S().Debug("unable to normalize data", zap.Error(err), zap.String("file", absFilePath))
-		return allResourcesConfig, err
+		return nil, err
 	}
 
-	// fill AllResourceConfigs
-	allResourcesConfig = make(map[string][]output.ResourceConfig)
-	var config *output.ResourceConfig
-	for _, resource := range configs {
-		config = &resource
-		config.Line = 1
-		config.Source = a.getSourceRelativePath(absFilePath)
-		allResourcesConfig[config.Type] = append(allResourcesConfig[config.Type], *config)
+	for _, config := range configs {
+		if config.Type == store.AwsCloudFormationStack {
+			if stackConfig, ok := config.Config.(cftRes.CloudFormationStackConfig); ok {
+				if stackConfig.TemplateData != nil {
+					stackResourceConfigs, err := a.getConfig(stackConfig.TemplateURL, &stackConfig.TemplateData, &stackConfig.Parameters)
+					if err == nil {
+						for i := range stackResourceConfigs {
+							// Add template url as source for the nested resources
+							stackResourceConfigs[i].Source = stackConfig.TemplateURL
+						}
+						configs = append(configs, stackResourceConfigs...)
+					}
+				}
+			}
+		}
 	}
-	return allResourcesConfig, nil
+
+	return configs, nil
 }
 
 func (*CFTV1) getFileType(file string, data *[]byte) string {

pkg/iac-providers/cft/v1/load-file_test.go

@@ -31,6 +31,7 @@ func TestLoadIacFile(t *testing.T) {
 	testFile, _ := filepath.Abs(path.Join(testDataDir, "testfile"))
 	invalidFile, _ := filepath.Abs(path.Join(testDataDir, "deploy.yaml"))
 	validFile, _ := filepath.Abs(path.Join(testDataDir, "templates", "s3", "deploy.template"))
+	nestedFile, _ := filepath.Abs(path.Join(testDataDir, "templates", "s3", "nested.template"))
 
 	testErrString1 := fmt.Sprintf("unsupported extension for file %s", testFile)
 	testErrString2 := "unable to read file nonexistent.txt"
@@ -73,6 +74,13 @@ func TestLoadIacFile(t *testing.T) {
 			name:     "invalid file",
 			filePath: validFile,
 			typeOnly: false,
+		}, {
+			wantErr:  nil,
+			want:     output.AllResourceConfigs{},
+			cftv1:    CFTV1{},
+			name:     "nested file",
+			filePath: nestedFile,
+			typeOnly: false,
 		},
 	}
 

pkg/iac-providers/cft/v1/testdata/templates/s3/nested.template

@@ -0,0 +1,17 @@
+{
+  "AWSTemplateFormatVersion" : "2010-09-09",
+  "Resources" : {
+    "myStackWithParams" : {
+      "Type" : "AWS::CloudFormation::Stack",
+      "Properties" : {
+        "TemplateURL" : "https://s3.amazonaws.com/cloudformation-templates-us-east-1/S3_Bucket.template",
+        "Parameters" : {
+          "InstanceType" : "t1.micro",
+          "KeyName" : "mykey"
+        },
+        "NotificationARNs":[]
+      }
+    }
+  }
+}
+

pkg/mapper/iac-providers/cft/config/cloudformation-stack.go

@@ -17,29 +17,47 @@
 package config
 
 import (
+	fn "github.com/accurics/terrascan/pkg/mapper/iac-providers/cft/functions"
 	"github.com/awslabs/goformation/v4/cloudformation/cloudformation"
 )
 
 // CloudFormationStackConfig holds config for aws_cloudformation_stack
 type CloudFormationStackConfig struct {
 	Config
-	TemplateURL      interface{} `json:"template_url"`
-	NotificationARNs interface{} `json:"notification_arns"`
+	TemplateURL      string            `json:"template_url"`
+	NotificationARNs interface{}       `json:"notification_arns"`
+	Parameters       map[string]string `json:"-"`
+	TemplateData     []byte            `json:"-"`
 }
 
 // GetCloudFormationStackConfig returns config for aws_cloudformation_stack
 func GetCloudFormationStackConfig(s *cloudformation.Stack) []AWSResourceConfig {
 	cf := CloudFormationStackConfig{
-		Config: Config{
-			Tags: s.Tags,
-		},
+		Config:           Config{Tags: s.Tags},
+		TemplateURL:      "",
+		NotificationARNs: nil,
+		TemplateData:     []byte{},
 	}
+
 	if len(s.NotificationARNs) > 0 {
 		cf.NotificationARNs = s.NotificationARNs
 	}
+
+	// Add and resolve template URL
 	if len(s.TemplateURL) > 0 {
 		cf.TemplateURL = s.TemplateURL
+
+		templateData, err := fn.DownloadBucketObj(s.TemplateURL)
+		if err == nil {
+			cf.TemplateData = templateData
+		}
 	}
+
+	// Add Parameters for propogation to the nested Stack
+	if s.Parameters != nil {
+		cf.Parameters = s.Parameters
+	}
+
 	return []AWSResourceConfig{{
 		Resource: cf,
 		Metadata: s.AWSCloudFormationMetadata,