From c1b2d57c7cb4b796dd7ee1bcd4c6e505738eb3ee Mon Sep 17 00:00:00 2001 From: Shrey Sonar Date: Thu, 5 Aug 2021 22:57:45 +0530 Subject: [PATCH] Add support for CFT nested stacks (#949) * Add support for CFT nested stacks * Review Changes --- go.mod | 6 +- go.sum | 43 +++-- pkg/iac-providers/cft/v1/load-file.go | 100 ++++++++--- pkg/iac-providers/cft/v1/load-file_test.go | 8 + .../v1/testdata/templates/s3/nested.template | 17 ++ .../cft/config/cloudformation-stack.go | 28 ++- .../cft/functions/s3-download.go | 161 ++++++++++++++++++ .../cft/functions/s3-download_test.go | 120 +++++++++++++ .../iac-providers/cft/functions/s3-uri.go | 123 +++++++++++++ .../cft/functions/s3-uri_test.go | 90 ++++++++++ 10 files changed, 658 insertions(+), 38 deletions(-) create mode 100644 pkg/iac-providers/cft/v1/testdata/templates/s3/nested.template create mode 100644 pkg/mapper/iac-providers/cft/functions/s3-download.go create mode 100644 pkg/mapper/iac-providers/cft/functions/s3-download_test.go create mode 100644 pkg/mapper/iac-providers/cft/functions/s3-uri.go create mode 100644 pkg/mapper/iac-providers/cft/functions/s3-uri_test.go diff --git a/go.mod b/go.mod index 2c9cef344..b2cd51b77 100644 --- a/go.mod +++ b/go.mod @@ -9,8 +9,12 @@ replace ( ) require ( - github.com/BurntSushi/toml v0.4.0 // indirect + github.com/BurntSushi/toml v0.4.1 // indirect github.com/VerbalExpressions/GoVerbalExpressions v0.0.0-20200410162751-4d76a1099a6e + github.com/aws/aws-sdk-go-v2/config v1.5.0 + github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.2 + github.com/aws/aws-sdk-go-v2/service/s3 v1.11.1 + github.com/aws/smithy-go v1.6.0 github.com/awslabs/goformation/v4 v4.19.1 github.com/ghodss/yaml v1.0.0 github.com/go-errors/errors v1.0.1 diff --git a/go.sum b/go.sum index 3853c6422..e0cc368a9 100644 --- a/go.sum +++ b/go.sum @@ -122,18 +122,9 @@ github.com/Azure/go-autorest/tracing v0.5.0/go.mod h1:r/s2XiOKccPW3HrqB+W0TQzfbt github.com/Azure/go-autorest/tracing v0.6.0/go.mod h1:+vhtPC754Xsa23ID7GlGsrdKBpUA79WCAKPPZVC2DeU= github.com/Azure/go-ntlmssp v0.0.0-20180810175552-4a21cbd618b4/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= github.com/Azure/go-ntlmssp v0.0.0-20200615164410-66371956d46c/go.mod h1:chxPXzSsl7ZWRAuOIE23GDNzjWuZquvFlgA8xmpunjU= -github.com/BurntSushi/toml v0.3.1 h1:WXkYYl6Yr3qBf1K79EBnL4mak0OimBfB0XUf9Vl28OQ= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= -github.com/BurntSushi/toml v0.3.2-0.20210614224209-34d990aa228d/go.mod h1:2QZjSXA5e+XyFeCAxxtL8Z4StYUsTquL8ODGPR3C3MA= -github.com/BurntSushi/toml v0.3.2-0.20210621044154-20a94d639b8e/go.mod h1:t4zg8TkHfP16Vb3x4WKIw7zVYMit5QFtPEO8lOWxzTg= -github.com/BurntSushi/toml v0.3.2-0.20210624061728-01bfc69d1057/go.mod h1:NMj2lD5LfMqcE0w8tnqOsH6944oaqpI1974lrIwerfE= -github.com/BurntSushi/toml v0.3.2-0.20210704081116-ccff24ee4463/go.mod h1:EkRrMiQQmfxK6kIldz3QbPlhmVkrjW1RDJUnbDqGYvc= -github.com/BurntSushi/toml v0.4.0 h1:qD/r9AL67srjW6O3fcSKZDsXqzBNX6ieSRywr2hRrdE= -github.com/BurntSushi/toml v0.4.0/go.mod h1:wtejDu7Q0FhCWAo2aXkywSJyYFg01EDTKozLNCz2JBA= -github.com/BurntSushi/toml-test v0.1.1-0.20210620192437-de01089bbf76/go.mod h1:P/PrhmZ37t5llHfDuiouWXtFgqOoQ12SAh9j6EjrBR4= -github.com/BurntSushi/toml-test v0.1.1-0.20210624055653-1f6389604dc6/go.mod h1:UAIt+Eo8itMZAAgImXkPGDMYsT1SsJkVdB5TuONl86A= -github.com/BurntSushi/toml-test v0.1.1-0.20210704062846-269931e74e3f/go.mod h1:fnFWrIwqgHsEjVsW3RYCJmDo86oq9eiJ9u6bnqhtm2g= -github.com/BurntSushi/toml-test v0.1.1-0.20210723065233-facb9eccd4da/go.mod h1:ve9Q/RRu2vHi42LocPLNvagxuUJh993/95b18bw/Nws= +github.com/BurntSushi/toml v0.4.1 h1:GaI7EiDXDRfa8VshkTj7Fym7ha+y8/XxIgD2okUIjLw= +github.com/BurntSushi/toml v0.4.1/go.mod h1:CxXYINrC8qIiEnFrOxCa7Jy5BFHlXnUU2pbicEuybxQ= github.com/BurntSushi/xgb v0.0.0-20160522181843-27f122750802/go.mod h1:IVnqGOEym/WlBOVXweHU+Q+/VP0lqqI8lqeDx9IjBqo= github.com/ChrisTrenkamp/goxpath v0.0.0-20170922090931-c385f95c6022/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4= github.com/ChrisTrenkamp/goxpath v0.0.0-20190607011252-c5096ec8773d/go.mod h1:nuWgzSkT5PnyOd+272uUmV0dnAnAn42Mk7PiQC5VzN4= @@ -260,6 +251,32 @@ github.com/aws/aws-sdk-go v1.31.9/go.mod h1:5zCpMtNQVjRREroY7sYe8lOMRSxkhG6MZveU github.com/aws/aws-sdk-go v1.37.0 h1:GzFnhOIsrGyQ69s7VgqtrG2BG8v7X7vwB3Xpbd/DBBk= github.com/aws/aws-sdk-go v1.37.0/go.mod h1:hcU610XS61/+aQV88ixoOzUoG7v3b31pl2zKMmprdro= github.com/aws/aws-sdk-go-v2 v0.18.0/go.mod h1:JWVYvqSMppoMJC0x5wdwiImzgXTI9FuZwxzkQq9wy+g= +github.com/aws/aws-sdk-go-v2 v1.7.1 h1:TswSc7KNqZ/K1Ijt3IkpXk/2+62vi3Q82Yrr5wSbRBQ= +github.com/aws/aws-sdk-go-v2 v1.7.1/go.mod h1:L5LuPC1ZgDr2xQS7AmIec/Jlc7O/Y1u2KxJyNVab250= +github.com/aws/aws-sdk-go-v2/config v1.5.0 h1:tRQcWXVmO7wC+ApwYc2LiYKfIBoIrdzcJ+7HIh6AlR0= +github.com/aws/aws-sdk-go-v2/config v1.5.0/go.mod h1:RWlPOAW3E3tbtNAqTwvSW54Of/yP3oiZXMI0xfUdjyA= +github.com/aws/aws-sdk-go-v2/credentials v1.3.1 h1:fFeqL5+9kwFKsCb2oci5yAIDsWYqn/Nga8oQ5bIasI8= +github.com/aws/aws-sdk-go-v2/credentials v1.3.1/go.mod h1:r0n73xwsIVagq8RsxmZbGSRQFj9As3je72C2WzUIToc= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0 h1:s4vtv3Mv1CisI3qm2HGHi1Ls9ZtbCOEqeQn6oz7fTyU= +github.com/aws/aws-sdk-go-v2/feature/ec2/imds v1.3.0/go.mod h1:2LAuqPx1I6jNfaGDucWfA2zqQCYCOMCDHiCOciALyNw= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.2 h1:fzEMxnHQWh+bUV0ZzfhMbgUG8zjIPnAgApjtdHtC9Yg= +github.com/aws/aws-sdk-go-v2/feature/s3/manager v1.3.2/go.mod h1:qaqQiHSrOUVOfKe6fhgQ6UzhxjwqVW8aHNegd6Ws4w4= +github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1 h1:SDLwr1NKyowP7uqxuLNdvFZhjnoVWxNv456zAp+ZFjU= +github.com/aws/aws-sdk-go-v2/internal/ini v1.1.1/go.mod h1:Zy8smImhTdOETZqfyn01iNOe0CNggVbPjCajyaz6Gvg= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.1 h1:s/uV8UyMB4UcO0ERHxG9BJhYJAD9MiY0QeYvJmlC7PE= +github.com/aws/aws-sdk-go-v2/service/internal/accept-encoding v1.2.1/go.mod h1:v33JQ57i2nekYTA70Mb+O18KeH4KqhdqxTJZNK1zdRE= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1 h1:VJe/XEhrfyfBLupcGg1BfUSK2VMZNdbDcZQ49jnp+h0= +github.com/aws/aws-sdk-go-v2/service/internal/presigned-url v1.2.1/go.mod h1:zceowr5Z1Nh2WVP8bf/3ikB41IZW59E4yIYbg+pC6mw= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.1 h1:1ds3HkMQEBx9XvOkqsPuqBmNFn0w8XEDuB4LOi6KepU= +github.com/aws/aws-sdk-go-v2/service/internal/s3shared v1.5.1/go.mod h1:6EQZIwNNvHpq/2/QSJnp4+ECvqIy55w95Ofs0ze+nGQ= +github.com/aws/aws-sdk-go-v2/service/s3 v1.11.1 h1:HiXhafnqG0AkVJIZA/BHhFvuc/8xFdUO1uaeqF2Artc= +github.com/aws/aws-sdk-go-v2/service/s3 v1.11.1/go.mod h1:XLAGFrEjbvMCLvAtWLLP32yTv8GpBquCApZEycDLunI= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.1 h1:H2ZLWHUbbeYtghuqCY5s/7tbBM99PAwCioRJF8QvV/U= +github.com/aws/aws-sdk-go-v2/service/sso v1.3.1/go.mod h1:J3A3RGUvuCZjvSuZEcOpHDnzZP/sKbhDWV2T1EOzFIM= +github.com/aws/aws-sdk-go-v2/service/sts v1.6.0 h1:Y9r6mrzOyAYz4qKaluSH19zqH1236il/nGbsPKOUT0s= +github.com/aws/aws-sdk-go-v2/service/sts v1.6.0/go.mod h1:q7o0j7d7HrJk/vr9uUt3BVRASvcU7gYZB9PUgPiByXg= +github.com/aws/smithy-go v1.6.0 h1:T6puApfBcYiTIsaI+SYWqanjMt5pc3aoyyDrI+0YH54= +github.com/aws/smithy-go v1.6.0/go.mod h1:SObp3lf9smib00L/v3U2eAKG8FyQ7iLrJnQiAmR5n+E= github.com/awslabs/goformation/v4 v4.19.1 h1:xqCDM4+gtkUNmxe1xP3LyH0X7EDMBR4HR1bqHUiMB7o= github.com/awslabs/goformation/v4 v4.19.1/go.mod h1:ygNqNsr904Q/Jan2A6ZKw9ewZWDTL9zlclZx2JzZhlM= github.com/aybabtme/rgbterm v0.0.0-20170906152045-cc83f3b3ce59/go.mod h1:q/89r3U2H7sSsE2t6Kca0lfwTK8JdoNGS/yzM/4iH5I= @@ -664,8 +681,10 @@ github.com/google/go-cmp v0.4.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/ github.com/google/go-cmp v0.5.0/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.1/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-cmp v0.5.2/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= -github.com/google/go-cmp v0.5.3 h1:x95R7cp+rSeeqAMI2knLtQ0DKlaBhv2NrtrOvafPHRo= github.com/google/go-cmp v0.5.3/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.4/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= +github.com/google/go-cmp v0.5.6 h1:BKbKCqvP6I+rmFHt06ZmyQtvB8xAkWdhFyr0ZUNZcxQ= +github.com/google/go-cmp v0.5.6/go.mod h1:v8dTdLbMG2kIc/vJvl+f65V22dbkXbowE6jgT/gNBxE= github.com/google/go-containerregistry v0.0.0-20191010200024-a3d713f9b7f8/go.mod h1:KyKXa9ciM8+lgMXwOVsXi7UxGrsf9mM61Mzs+xKUrKE= github.com/google/go-containerregistry v0.1.2/go.mod h1:GPivBPgdAyd2SU+vf6EpsgOtWDuPqjW0hJZt4rNdTZ4= github.com/google/go-github v17.0.0+incompatible/go.mod h1:zLgOLi98H3fifZn+44m+umXrS52loVEgC2AApnigrVQ= diff --git a/pkg/iac-providers/cft/v1/load-file.go b/pkg/iac-providers/cft/v1/load-file.go index 4700968a0..621307272 100644 --- a/pkg/iac-providers/cft/v1/load-file.go +++ b/pkg/iac-providers/cft/v1/load-file.go @@ -25,6 +25,8 @@ import ( "github.com/accurics/terrascan/pkg/iac-providers/output" "github.com/accurics/terrascan/pkg/mapper" + cftRes "github.com/accurics/terrascan/pkg/mapper/iac-providers/cft/config" + "github.com/accurics/terrascan/pkg/mapper/iac-providers/cft/store" "github.com/awslabs/goformation/v4" "github.com/awslabs/goformation/v4/cloudformation" "go.uber.org/zap" @@ -40,44 +42,102 @@ func (a *CFTV1) LoadIacFile(absFilePath string, options map[string]interface{}) } // parse the file as cloudformation.Template - fileExt := a.getFileType(absFilePath, &fileData) - var template *cloudformation.Template + configs, err := a.getConfig(absFilePath, &fileData, nil) + if err != nil { + zap.S().Debug("unable to normalize data", zap.Error(err), zap.String("file", absFilePath)) + return allResourcesConfig, err + } + + // fill AllResourceConfigs + allResourcesConfig = make(map[string][]output.ResourceConfig) + var config *output.ResourceConfig + for _, resource := range configs { + config = &resource + config.Line = 1 + if config.Source == "" { + config.Source = a.getSourceRelativePath(absFilePath) + } + allResourcesConfig[config.Type] = append(allResourcesConfig[config.Type], *config) + } + return allResourcesConfig, nil +} + +func (a *CFTV1) getConfig(absFilePath string, fileData *[]byte, parameters *map[string]string) ([]output.ResourceConfig, error) { + // parse the file as cloudformation.Template + template, err := a.extractTemplate(absFilePath, fileData) + if err != nil { + return nil, err + } + + // replace template parameter values + if parameters != nil { + for key, value := range *parameters { + if parameter, ok := template.Parameters[key]; ok { + parameter.Default = value + template.Parameters[key] = parameter + } + } + } + + // map resource to a terrascan type + configs, err := a.translateResources(template, absFilePath) + if err != nil { + zap.S().Debug("unable to normalize data", zap.Error(err), zap.String("file", absFilePath)) + return nil, err + } + return configs, nil +} + +func (a *CFTV1) extractTemplate(file string, data *[]byte) (*cloudformation.Template, error) { + fileExt := a.getFileType(file, data) + switch fileExt { case YAMLExtension, YAMLExtension2: - template, err = goformation.ParseYAML(fileData) + template, err := goformation.ParseYAML(*data) if err != nil { - zap.S().Debug("failed to parse file", zap.String("file", absFilePath)) - return allResourcesConfig, err + zap.S().Debug("failed to parse file", zap.String("file", file)) + return nil, err } + return template, nil case JSONExtension: - template, err = goformation.ParseJSON(fileData) + template, err := goformation.ParseJSON(*data) if err != nil { - zap.S().Debug("failed to parse file", zap.String("file", absFilePath)) - return allResourcesConfig, err + zap.S().Debug("failed to parse file", zap.String("file", file)) + return nil, err } + return template, nil default: zap.S().Debug("unknown extension found", zap.String("extension", fileExt)) - return allResourcesConfig, fmt.Errorf("unsupported extension for file %s", absFilePath) + return nil, fmt.Errorf("unsupported extension for file %s", file) } +} - // map resource to a terrascan type +func (a *CFTV1) translateResources(template *cloudformation.Template, absFilePath string) ([]output.ResourceConfig, error) { m := mapper.NewMapper("cft") configs, err := m.Map(template) if err != nil { zap.S().Debug("unable to normalize data", zap.Error(err), zap.String("file", absFilePath)) - return allResourcesConfig, err + return nil, err } - // fill AllResourceConfigs - allResourcesConfig = make(map[string][]output.ResourceConfig) - var config *output.ResourceConfig - for _, resource := range configs { - config = &resource - config.Line = 1 - config.Source = a.getSourceRelativePath(absFilePath) - allResourcesConfig[config.Type] = append(allResourcesConfig[config.Type], *config) + for _, config := range configs { + if config.Type == store.AwsCloudFormationStack { + if stackConfig, ok := config.Config.(cftRes.CloudFormationStackConfig); ok { + if stackConfig.TemplateData != nil { + stackResourceConfigs, err := a.getConfig(stackConfig.TemplateURL, &stackConfig.TemplateData, &stackConfig.Parameters) + if err == nil { + for i := range stackResourceConfigs { + // Add template url as source for the nested resources + stackResourceConfigs[i].Source = stackConfig.TemplateURL + } + configs = append(configs, stackResourceConfigs...) + } + } + } + } } - return allResourcesConfig, nil + + return configs, nil } func (*CFTV1) getFileType(file string, data *[]byte) string { diff --git a/pkg/iac-providers/cft/v1/load-file_test.go b/pkg/iac-providers/cft/v1/load-file_test.go index 2913e996d..86068153f 100644 --- a/pkg/iac-providers/cft/v1/load-file_test.go +++ b/pkg/iac-providers/cft/v1/load-file_test.go @@ -31,6 +31,7 @@ func TestLoadIacFile(t *testing.T) { testFile, _ := filepath.Abs(path.Join(testDataDir, "testfile")) invalidFile, _ := filepath.Abs(path.Join(testDataDir, "deploy.yaml")) validFile, _ := filepath.Abs(path.Join(testDataDir, "templates", "s3", "deploy.template")) + nestedFile, _ := filepath.Abs(path.Join(testDataDir, "templates", "s3", "nested.template")) testErrString1 := fmt.Sprintf("unsupported extension for file %s", testFile) testErrString2 := "unable to read file nonexistent.txt" @@ -73,6 +74,13 @@ func TestLoadIacFile(t *testing.T) { name: "invalid file", filePath: validFile, typeOnly: false, + }, { + wantErr: nil, + want: output.AllResourceConfigs{}, + cftv1: CFTV1{}, + name: "nested file", + filePath: nestedFile, + typeOnly: false, }, } diff --git a/pkg/iac-providers/cft/v1/testdata/templates/s3/nested.template b/pkg/iac-providers/cft/v1/testdata/templates/s3/nested.template new file mode 100644 index 000000000..92ec52d8c --- /dev/null +++ b/pkg/iac-providers/cft/v1/testdata/templates/s3/nested.template @@ -0,0 +1,17 @@ +{ + "AWSTemplateFormatVersion" : "2010-09-09", + "Resources" : { + "myStackWithParams" : { + "Type" : "AWS::CloudFormation::Stack", + "Properties" : { + "TemplateURL" : "https://s3.amazonaws.com/cloudformation-templates-us-east-1/S3_Bucket.template", + "Parameters" : { + "InstanceType" : "t1.micro", + "KeyName" : "mykey" + }, + "NotificationARNs":[] + } + } + } +} + diff --git a/pkg/mapper/iac-providers/cft/config/cloudformation-stack.go b/pkg/mapper/iac-providers/cft/config/cloudformation-stack.go index 75375ea8f..9ef941b85 100644 --- a/pkg/mapper/iac-providers/cft/config/cloudformation-stack.go +++ b/pkg/mapper/iac-providers/cft/config/cloudformation-stack.go @@ -17,29 +17,47 @@ package config import ( + fn "github.com/accurics/terrascan/pkg/mapper/iac-providers/cft/functions" "github.com/awslabs/goformation/v4/cloudformation/cloudformation" ) // CloudFormationStackConfig holds config for aws_cloudformation_stack type CloudFormationStackConfig struct { Config - TemplateURL interface{} `json:"template_url"` - NotificationARNs interface{} `json:"notification_arns"` + TemplateURL string `json:"template_url"` + NotificationARNs interface{} `json:"notification_arns"` + Parameters map[string]string `json:"-"` + TemplateData []byte `json:"-"` } // GetCloudFormationStackConfig returns config for aws_cloudformation_stack func GetCloudFormationStackConfig(s *cloudformation.Stack) []AWSResourceConfig { cf := CloudFormationStackConfig{ - Config: Config{ - Tags: s.Tags, - }, + Config: Config{Tags: s.Tags}, + TemplateURL: "", + NotificationARNs: nil, + TemplateData: []byte{}, } + if len(s.NotificationARNs) > 0 { cf.NotificationARNs = s.NotificationARNs } + + // Add and resolve template URL if len(s.TemplateURL) > 0 { cf.TemplateURL = s.TemplateURL + + templateData, err := fn.DownloadBucketObj(s.TemplateURL) + if err == nil { + cf.TemplateData = templateData + } } + + // Add Parameters for propogation to the nested Stack + if s.Parameters != nil { + cf.Parameters = s.Parameters + } + return []AWSResourceConfig{{ Resource: cf, Metadata: s.AWSCloudFormationMetadata, diff --git a/pkg/mapper/iac-providers/cft/functions/s3-download.go b/pkg/mapper/iac-providers/cft/functions/s3-download.go new file mode 100644 index 000000000..479b179cd --- /dev/null +++ b/pkg/mapper/iac-providers/cft/functions/s3-download.go @@ -0,0 +1,161 @@ +/* + Copyright (C) 2021 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package functions + +import ( + "context" + "fmt" + "io" + "io/ioutil" + "net/url" + "os" + "path/filepath" + "strings" + + "github.com/accurics/terrascan/pkg/utils" + "github.com/aws/aws-sdk-go-v2/config" + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + getter "github.com/hashicorp/go-getter" + "go.uber.org/zap" +) + +// HeadBucketAPIClient is an S3 API client that can invoke the HeadBucket operation. +type HeadBucketAPIClient interface { + HeadObject(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error) +} + +// S3DownloadManager is an S3 manager that can invoke the Download operation. +type S3DownloadManager interface { + Download(ctx context.Context, w io.WriterAt, input *s3.GetObjectInput, options ...func(*manager.Downloader)) (n int64, err error) +} + +// S3Client struct is used to hold s3.Client, manager.Downloader corresponding interfaces +type S3Client struct { + client HeadBucketAPIClient + downloader S3DownloadManager +} + +// NewS3Client returns S3Client initialized with AWS credentials +func NewS3Client() (*S3Client, error) { + cfg, err := config.LoadDefaultConfig(context.TODO()) + if err != nil { + zap.S().Debug("error loading AWS credentials for bucket", err) + return nil, err + } + client := s3.NewFromConfig(cfg) + return &S3Client{ + client: client, + downloader: manager.NewDownloader(client), + }, nil +} + +// DownloadBucketObj returns the content for S3 bucket object +func DownloadBucketObj(templateURL string) ([]byte, error) { + s3URL, err := url.Parse(templateURL) + if err != nil { + return nil, fmt.Errorf("unable to parse given S3 endpoint URL: %w", err) + } + + // assuming url points to public object + switch s3URL.Scheme { + case "http", "https": + buf, err := downloadPublicTemplate(templateURL) + if err != nil { + zap.S().Debug("the s3 url for nested stack is not a public object", zap.String("url", templateURL), err) + } else { + return buf, nil + } + } + + // if not public get bucket name and key + s3c, err := NewS3Client() + if err != nil { + zap.S().Debug("error loading AWS credentials for bucket", zap.String("url", templateURL)) + return nil, err + } + + return downloadPrivateTemplate(s3URL, s3c) +} + +func downloadPrivateTemplate(url *url.URL, s3c *S3Client) ([]byte, error) { + s3URI, err := ParseS3URI(url) + if err != nil { + zap.S().Debug("error parsing S3 uri", s3URI, err) + return nil, err + } + + // get size and check access + headInput := &s3.HeadObjectInput{ + Bucket: s3URI.Bucket, + Key: s3URI.Key, + } + if s3URI.VersionID != nil { + headInput.VersionId = s3URI.VersionID + } + + headObject, err := s3c.client.HeadObject(context.TODO(), headInput) + if err != nil { + zap.S().Debug("error in HEAD operation for bucket object", err) + return nil, err + } + buf := make([]byte, int(headObject.ContentLength)) + w := manager.NewWriteAtBuffer(buf) + + // get the object + downloaderInput := &s3.GetObjectInput{ + Bucket: s3URI.Bucket, + Key: s3URI.Key, + } + if s3URI.VersionID != nil { + downloaderInput.VersionId = s3URI.VersionID + } + + _, err = s3c.downloader.Download(context.TODO(), w, downloaderInput) + if err != nil { + zap.S().Debug("error downloading bucket object for uri", s3URI) + return nil, err + } + + return buf, nil + +} + +func downloadPublicTemplate(uri string) ([]byte, error) { + dst := filepath.Join(os.TempDir(), utils.GenRandomString(6)) + defer os.RemoveAll(dst) + parts := strings.Split(uri, "/") + path := filepath.Join(dst, parts[len(parts)-1]) + + client := getter.Client{ + Src: uri, + Dst: path, + Mode: getter.ClientModeFile, + } + err := client.Get() + if err != nil { + zap.S().Debug("unable to parse linked template", zap.Error(err), zap.String("file", path)) + return nil, err + } + + fileData, err := ioutil.ReadFile(path) + if err != nil { + return nil, err + } + return fileData, nil + +} diff --git a/pkg/mapper/iac-providers/cft/functions/s3-download_test.go b/pkg/mapper/iac-providers/cft/functions/s3-download_test.go new file mode 100644 index 000000000..c6ebf8df9 --- /dev/null +++ b/pkg/mapper/iac-providers/cft/functions/s3-download_test.go @@ -0,0 +1,120 @@ +/* + Copyright (C) 2021 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ +package functions + +import ( + "context" + "errors" + "fmt" + "io" + "net/url" + "reflect" + "testing" + + "github.com/aws/aws-sdk-go-v2/feature/s3/manager" + "github.com/aws/aws-sdk-go-v2/service/s3" + "github.com/aws/smithy-go" +) + +func TestDownloadBucketObj(t *testing.T) { + table := []struct { + errorOperation string + name string + templateURL string + }{ + { + name: "public template", + templateURL: "https://s3.amazonaws.com/cloudformation-templates-us-east-1/S3_Bucket.template", + }, { + errorOperation: "HeadObject", + name: "private template head object error", + templateURL: "https://s3.amazonaws.com/cloudformation-templates-us-east-1/S3_Bucket_Not_There.template", + }, + } + + for _, tt := range table { + t.Run(tt.name, func(t *testing.T) { + _, err := DownloadBucketObj(tt.templateURL) + if tt.errorOperation != "" { + var oe *smithy.OperationError + if errors.As(err, &oe) { + if oe.Operation() != tt.errorOperation { + t.Errorf("unexpected operation; got: '%+v'", oe.Operation()) + } + } else { + t.Errorf("unexpected error; got: '%+v'", reflect.TypeOf(err)) + } + } + }) + } +} + +type mockHeadObjectAPI func(context.Context, *s3.HeadObjectInput, ...func(*s3.Options)) (*s3.HeadObjectOutput, error) + +func (m mockHeadObjectAPI) HeadObject(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + return m(ctx, params, optFns...) +} + +type mockS3DownloadManager func(ctx context.Context, w io.WriterAt, input *s3.GetObjectInput, options ...func(*manager.Downloader)) (n int64, err error) + +func (m mockS3DownloadManager) Download(ctx context.Context, w io.WriterAt, input *s3.GetObjectInput, options ...func(*manager.Downloader)) (n int64, err error) { + return m(ctx, w, input, options...) +} + +func TestDownloadPrivateBucketObj(t *testing.T) { + + table := []struct { + client func(t *testing.T) HeadBucketAPIClient + manager func(t *testing.T) S3DownloadManager + errorOperation string + name string + templateURL string + }{ + { + client: func(t *testing.T) HeadBucketAPIClient { + return mockHeadObjectAPI(func(ctx context.Context, params *s3.HeadObjectInput, optFns ...func(*s3.Options)) (*s3.HeadObjectOutput, error) { + t.Helper() + return &s3.HeadObjectOutput{ContentLength: 64}, nil + }) + }, + manager: func(t *testing.T) S3DownloadManager { + return mockS3DownloadManager(func(ctx context.Context, w io.WriterAt, input *s3.GetObjectInput, options ...func(*manager.Downloader)) (n int64, err error) { + t.Helper() + return 0, fmt.Errorf("error in download operation") + }) + }, + errorOperation: "error in download operation", + name: "read obj error", + templateURL: "https://s3.amazonaws.com/cloudformation-templates-us-east-1/S3_Bucket_Not_There.template", + }, + } + + for _, tt := range table { + t.Run(tt.name, func(t *testing.T) { + u, _ := url.Parse(tt.templateURL) + s3c := S3Client{ + client: tt.client(t), + downloader: tt.manager(t), + } + _, err := downloadPrivateTemplate(u, &s3c) + if tt.errorOperation != "" { + if err.Error() != tt.errorOperation { + t.Errorf("unexpected error; got: '%+v'", reflect.TypeOf(err)) + } + } + }) + } +} diff --git a/pkg/mapper/iac-providers/cft/functions/s3-uri.go b/pkg/mapper/iac-providers/cft/functions/s3-uri.go new file mode 100644 index 000000000..96404357e --- /dev/null +++ b/pkg/mapper/iac-providers/cft/functions/s3-uri.go @@ -0,0 +1,123 @@ +/* + Copyright (C) 2021 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package functions + +import ( + "errors" + "fmt" + "net/url" + "regexp" + "strings" +) + +// borrowed from https://gist.github.com/kwilczynski/f6e626990d6d2395b42a12721b165b86 +// modified per need + +// S3URI holds the metadata for s3 url +type S3URI struct { + uri *url.URL + VersionID *string + Scheme *string + Bucket *string + Key *string +} + +// String returns pointer to the string object +func String(s string) *string { + return &s +} + +var ( + errBucketNotFound = errors.New("bucket name could not be found") + errHostnameNotFound = errors.New("hostname could not be found") + errInvalidS3Endpoint = errors.New("an invalid S3 endpoint URL") + + // Pattern used to parse multiple path and host style S3 endpoint URLs. + s3URLPattern = regexp.MustCompile(`^(.+\.)?s3[.-](?:(accelerated|dualstack|website)[.-])?([a-z0-9-]+)\.`) +) + +// ParseS3URI returns s3 metadata given url +func ParseS3URI(u *url.URL) (*S3URI, error) { + s3u := &S3URI{ + uri: u, + } + + switch u.Scheme { + case "s3", "http", "https": + s3u.Scheme = String(u.Scheme) + default: + return nil, fmt.Errorf("unable to parse schema type: %s", u.Scheme) + } + + // Handle S3 endpoint URL with the schema s3:// that is neither + // the host style nor the path style. + if u.Scheme == "s3" { + if u.Host == "" { + return nil, errBucketNotFound + } + s3u.Bucket = String(u.Host) + + if u.Path != "" && u.Path != "/" { + s3u.Key = String(strings.TrimLeft(u.Path, "/")) + } + + return s3u, nil + } + + if u.Host == "" { + return nil, errHostnameNotFound + } + + matches := s3URLPattern.FindStringSubmatch(u.Host) + if matches == nil || len(matches) < 1 { + return nil, errInvalidS3Endpoint + } + + prefix := matches[1] + + if prefix == "" { + if u.Path != "" && u.Path != "/" { + u.Path = u.Path[1:len(u.Path)] + + index := strings.Index(u.Path, "/") + switch { + case index == -1: + s3u.Bucket = String(u.Path) + case index == len(u.Path)-1: + s3u.Bucket = String(u.Path[:index]) + default: + s3u.Bucket = String(u.Path[:index]) + s3u.Key = String(u.Path[index+1:]) + } + } + } else { + s3u.Bucket = String(prefix[:len(prefix)-1]) + + if u.Path != "" && u.Path != "/" { + s3u.Key = String(u.Path[1:len(u.Path)]) + } + } + + // Query string used when requesting a particular version of a given + // S3 object (key). + const versionID = "versionID" + if s := u.Query().Get(versionID); s != "" { + s3u.VersionID = String(s) + } + + return s3u, nil +} diff --git a/pkg/mapper/iac-providers/cft/functions/s3-uri_test.go b/pkg/mapper/iac-providers/cft/functions/s3-uri_test.go new file mode 100644 index 000000000..9e3df0c08 --- /dev/null +++ b/pkg/mapper/iac-providers/cft/functions/s3-uri_test.go @@ -0,0 +1,90 @@ +/* + Copyright (C) 2021 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package functions + +import ( + "net/url" + "reflect" + "testing" +) + +const ( + bucket = "bucket" + key = "key" +) + +func TestParseS3URI(t *testing.T) { + table := []struct { + wantErr error + name string + templateURL string + }{ + { + name: "https host style", + templateURL: "http://bucket.s3-region.amazonaws.com/key", + wantErr: nil, + }, { + name: "https path style", + templateURL: "http://s3-region.amazonaws.com/bucket/key", + wantErr: nil, + }, { + name: "dualstack 1", + templateURL: "https://s3.dualstack.region.amazonaws.com/bucket/key", + wantErr: nil, + }, { + name: "dualstack 2", + templateURL: "http://bucket.s3.dualstack.region.amazonaws.com/key", + wantErr: nil, + }, { + name: "static 1", + templateURL: "http://bucket.s3-website.region.amazonaws.com/key", + wantErr: nil, + }, { + name: "static 2", + templateURL: "http://bucket.s3-website-region.amazonaws.com/key", + wantErr: nil, + }, { + name: "s3 1", + templateURL: "https://s3.region.amazonaws.com/bucket/key", + wantErr: nil, + }, { + name: "s3 2", + templateURL: "http://s3-region.amazonaws.com/bucket/key", + wantErr: nil, + }, { + name: "s3 3", + templateURL: "https://s3.amazonaws.com/bucket/key", + wantErr: nil, + }, + } + + for _, tt := range table { + t.Run(tt.name, func(t *testing.T) { + u, _ := url.Parse(tt.templateURL) + s3u, err := ParseS3URI(u) + if err != nil { + if err != tt.wantErr { + t.Errorf("unexpected error; got: '%+v'", reflect.TypeOf(err)) + } + } else { + if *s3u.Bucket != bucket || *s3u.Key != key { + t.Errorf("unexpected metadata; got '%+v'", s3u) + } + } + }) + } +}