From 390eed3601279aecb02017e41a7162a88223a0f7 Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Wed, 20 Jan 2021 13:44:07 +0530 Subject: [PATCH 01/15] 1. initial changes for registry module support 2. fix issue of remote module containing local modules --- go.sum | 4 + pkg/cli/register.go | 7 + .../terraform/commons/load-dir.go | 35 ++++- .../terraform/commons/module-download.go | 11 ++ .../commons/remote-module-download.go | 137 ++++++++++++++++++ 5 files changed, 193 insertions(+), 1 deletion(-) create mode 100644 pkg/iac-providers/terraform/commons/remote-module-download.go diff --git a/go.sum b/go.sum index ebc4f35b3..695ad5a28 100644 --- a/go.sum +++ b/go.sum @@ -553,6 +553,7 @@ github.com/hashicorp/go-version v1.0.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09 github.com/hashicorp/go-version v1.1.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go-version v1.2.0 h1:3vNe/fWF5CBgRIguda1meWhsZHy3m8gCJ5wx+dIzX/E= github.com/hashicorp/go-version v1.2.0/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= +github.com/hashicorp/go-version v1.2.1 h1:zEfKbn2+PDgroKdiOzqiE8rsmLqU2uwi5PB5pBJ3TkI= github.com/hashicorp/go-version v1.2.1/go.mod h1:fltr4n8CU8Ke44wwGCBoEymUuxUHl09ZGVZPK5anwXA= github.com/hashicorp/go.net v0.0.1/go.mod h1:hjKkEWcCURg++eb33jQU7oqQcI9XDCnUzHA0oac0k90= github.com/hashicorp/golang-lru v0.5.0/go.mod h1:/m3WP610KZHVQ1SGc6re/UDhFvYD7pJ4Ao+sR/qLZy8= @@ -683,6 +684,7 @@ github.com/mattn/go-isatty v0.0.4/go.mod h1:M+lRXTBqGeGNdLjl/ufCoiOlB5xdOkqRJdNx github.com/mattn/go-isatty v0.0.5/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= github.com/mattn/go-isatty v0.0.8 h1:HLtExJ+uU2HOZ+wI0Tt5DtUDrx8yhUqDcp7fYERX4CE= github.com/mattn/go-isatty v0.0.8/go.mod h1:Iq45c/XA43vh69/j3iqttzPXn0bhXyGjM0Hdxcsrc5s= +github.com/mattn/go-isatty v0.0.12 h1:wuysRhFDzyxgEmMf5xjvJ2M9dZoWAXNNr5LSBS7uHXY= github.com/mattn/go-isatty v0.0.12/go.mod h1:cbi8OIDigv2wuxKPP5vlRcQ1OAZbq2CE4Kysco4FUpU= github.com/mattn/go-oci8 v0.0.7/go.mod h1:wjDx6Xm9q7dFtHJvIlrI99JytznLw5wQ4R+9mNXJwGI= github.com/mattn/go-runewidth v0.0.0-20181025052659-b20a3daf6a39/go.mod h1:LwmH8dsx7+W8Uxz3IHJYH5QSwggIsqBzpuz5H//U1FU= @@ -1498,9 +1500,11 @@ sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbL sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/kustomize/api v0.7.1 h1:/cjDi4Pk/hqRSeCCj/Xum66rYrEtc7osM2/O+lvYKkM= sigs.k8s.io/kustomize/api v0.7.1/go.mod h1:XOt24UrCkv0x63eT5JVaph4Kqf5EVU2UBAXo6SPBaAY= +sigs.k8s.io/kustomize/api v0.7.2 h1:ItTD/2XaKO8CosOMFZdaGFdUGTCHdQriW7zQ7AR98rs= sigs.k8s.io/kustomize/api v0.7.2/go.mod h1:50/vLATrjhRmMr3spZsI1GcpoZJ8IARy9QstPbA9lGE= sigs.k8s.io/kustomize/kyaml v0.10.5 h1:PbJcsZsEM7O3hHtUWTR+4WkHVbQRW9crSy75or1gRbI= sigs.k8s.io/kustomize/kyaml v0.10.5/go.mod h1:P6Oy/ah/GZMKzJMIJA2a3/bc8YrBkuL5kJji13PSIzY= +sigs.k8s.io/kustomize/kyaml v0.10.6 h1:xUJxc/k8JoWqHUahaB8DTqY0KwEPxTbTGStvW8TOcDc= sigs.k8s.io/kustomize/kyaml v0.10.6/go.mod h1:K9yg1k/HB/6xNOf5VH3LhTo1DK9/5ykSZO5uIv+Y/1k= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e/go.mod h1:wWxsB5ozmmv/SG7nM11ayaAW51xMvak/t1r0CSlcokI= diff --git a/pkg/cli/register.go b/pkg/cli/register.go index 98da7e4b9..c4dd9c953 100644 --- a/pkg/cli/register.go +++ b/pkg/cli/register.go @@ -18,6 +18,8 @@ package cli import ( "flag" + "io/ioutil" + "log" "os" "github.com/accurics/terrascan/pkg/config" @@ -59,6 +61,11 @@ func Execute() { } } + // disable terraform logs when TF_LOG env variable is not set + if os.Getenv("TF_LOG") == "" { + log.SetOutput(ioutil.Discard) + } + if err := rootCmd.Execute(); err != nil { os.Exit(1) } diff --git a/pkg/iac-providers/terraform/commons/load-dir.go b/pkg/iac-providers/terraform/commons/load-dir.go index 9aae1aa2a..4869ea526 100644 --- a/pkg/iac-providers/terraform/commons/load-dir.go +++ b/pkg/iac-providers/terraform/commons/load-dir.go @@ -20,12 +20,14 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/accurics/terrascan/pkg/iac-providers/output" "github.com/accurics/terrascan/pkg/utils" version "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2" hclConfigs "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry/regsrc" "github.com/spf13/afero" "go.uber.org/zap" ) @@ -53,6 +55,9 @@ type ModuleConfig struct { // resources present in rootDir and descendant modules func LoadIacDir(absRootDir string) (allResourcesConfig output.AllResourceConfigs, err error) { + // map to hold the download paths of remote modules + remoteModPaths := make(map[string]string) + // create a new config parser parser := hclConfigs.NewParser(afero.NewOsFs()) @@ -92,8 +97,36 @@ func LoadIacDir(absRootDir string) (allResourcesConfig output.AllResourceConfigs for p := req.Parent; p != nil; p = p.Parent { pathToModule = filepath.Join(p.SourceAddr, pathToModule) } - pathToModule = filepath.Join(absRootDir, pathToModule) + + // check if pathToModule consists of a remote module downloaded + // if yes, then update the module path, with the remote module + // download path + keyFound := false + for remoteSourceAddr, downloadPath := range remoteModPaths { + if strings.Contains(pathToModule, remoteSourceAddr) { + pathToModule = strings.Replace(pathToModule, remoteSourceAddr, "", 1) + pathToModule = downloadPath + pathToModule + keyFound = true + break + } + } + if !keyFound { + pathToModule = filepath.Join(absRootDir, pathToModule) + } zap.S().Debugf("processing local module %q", pathToModule) + } else if isRegistrySourceAddr(req.SourceAddr) { + // regsrc.ParseModuleSource func returns a terraform registry module source + // error check is not required as the source address is already validated above + module, _ := regsrc.ParseModuleSource(req.SourceAddr) + + tempDir := filepath.Join(os.TempDir(), utils.GenRandomString(6)) + pathToModule, err = r.DownloadRemoteModule(req.VersionConstraint, tempDir, module) + if err != nil { + zap.S().Errorf("failed to download remote module %q. error: '%v'", req.SourceAddr, err) + } else { + // add the entry of remote module's source address to the map + remoteModPaths[req.SourceAddr] = pathToModule + } } else { // temp dir to download the remote repo tempDir := filepath.Join(os.TempDir(), utils.GenRandomString(6)) diff --git a/pkg/iac-providers/terraform/commons/module-download.go b/pkg/iac-providers/terraform/commons/module-download.go index 40fb9b364..f24f667de 100644 --- a/pkg/iac-providers/terraform/commons/module-download.go +++ b/pkg/iac-providers/terraform/commons/module-download.go @@ -23,6 +23,8 @@ import ( "github.com/accurics/terrascan/pkg/downloader" "go.uber.org/zap" + + "github.com/hashicorp/terraform/registry/regsrc" ) var localSourcePrefixes = []string{ @@ -41,6 +43,15 @@ func isLocalSourceAddr(addr string) bool { return false } +// isRegistrySourceAddr will validate if the source address is a valid registry +// module or not. +// a valid source address is of the form /NAMESPACE>// +// regsrc.ParseModuleSource func returns a terraform registry module source. +func isRegistrySourceAddr(addr string) bool { + _, err := regsrc.ParseModuleSource(addr) + return err == nil +} + // RemoteModuleInstaller helps in downloading remote modules, it also maintains a // cache of all the installed modules and their respective resolved addresses // (URL) diff --git a/pkg/iac-providers/terraform/commons/remote-module-download.go b/pkg/iac-providers/terraform/commons/remote-module-download.go new file mode 100644 index 000000000..3dd6b585a --- /dev/null +++ b/pkg/iac-providers/terraform/commons/remote-module-download.go @@ -0,0 +1,137 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package commons + +import ( + "fmt" + "path/filepath" + + version "github.com/hashicorp/go-version" + hclConfigs "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry" + "github.com/hashicorp/terraform/registry/regsrc" + "go.uber.org/zap" +) + +// DownloadRemoteModule will download remote modules from public and private terraform registries +// this function takes similar approach taken by terraform init for downloading terraform registry modules +func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs.VersionConstraint, destPath string, module *regsrc.Module) (string, error) { + // Terraform doesn't allow the hostname to contain Punycode + // module.SvcHost returns an error for such case + _, err := module.SvcHost() + if err != nil { + zap.S().Errorf("hostname for the module %s is invalid", module.String()) + return "", err + } + + // get terraform registry client. + // terraform registry client provides methods for querying the terraform module registry + regClient := registry.NewClient(nil, nil) + + // get all the available module versions from the terraform registry + moduleVersions, err := regClient.ModuleVersions(module) + if err != nil { + if registry.IsModuleNotFound(err) { + zap.S().Errorf("module: %s, not be found at registry: %s", module.String(), module.Host().Display()) + } else { + zap.S().Errorf("error while fetching available modules for module: %s, at registry: %s", module.String(), module.Host().Display()) + } + return "", err + } + + // terraform init command pulls all the available versions of a module, + // and downloads the latest non pre-release (unless a pre-release version was + // specified in tf file) version, if a version constraint is not provided in the tf file. + // we are following what terraform does + allModules := moduleVersions.Modules[0] + + var latestMatch *version.Version + var latestVersion *version.Version + var versionToDownload *version.Version + for _, moduleVersion := range allModules.Versions { + currentVersion, err := version.NewVersion(moduleVersion.Version) + if err != nil { + // if error is received for a version, then skip the current version + zap.S().Errorf("invalid version: %s, for module: %s, at registry: %s", moduleVersion.Version, module.String(), module.Host().Display()) + continue + } + + if requiredVersion.Required == nil { + // skip the pre release version + if currentVersion.Prerelease() != "" { + continue + } + + // update the latest version + latestVersion = getGreaterVersion(latestVersion, currentVersion) + } else { + // skip the pre-release version, unless specified in the tf file + if currentVersion.Prerelease() != "" && requiredVersion.Required.String() != currentVersion.String() { + continue + } + + // update the latest version + latestVersion = getGreaterVersion(latestVersion, currentVersion) + + // update latest match + if requiredVersion.Required.Check(currentVersion) { + latestMatch = getGreaterVersion(latestMatch, currentVersion) + } + } + } + + if latestVersion == nil { + return "", fmt.Errorf("no versions for module: %s, found at registry: %s", module.String(), module.Host().Display()) + } + + if requiredVersion.Required != nil && latestMatch == nil { + return "", fmt.Errorf("no versions matching: %s, for module: %s, found at registry: %s, latest version found: %s", requiredVersion.Required.String(), module.String(), module.Host().Display(), latestVersion.String()) + } + + versionToDownload = latestVersion + if latestMatch != nil { + versionToDownload = latestMatch + } + + // get the source location for the matched version + sourceLocation, err := regClient.ModuleLocation(module, versionToDownload.String()) + if err != nil { + zap.S().Errorf("error while getting the source location for module: %s, at registry: %s", module.String(), module.Host().Display()) + return "", err + } + + downloadLocation, err := r.DownloadModule(sourceLocation, destPath) + if err != nil { + zap.S().Errorf("error while downloading module: %s, with source location: %s", module.String(), sourceLocation) + return "", nil + } + + if module.RawSubmodule != "" { + // Append the user's requested subdirectory to any subdirectory that + // was implied by any of the nested layers we expanded within go-getter. + downloadLocation = filepath.Join(downloadLocation, module.RawSubmodule) + } + + return downloadLocation, nil +} + +func getGreaterVersion(latestVersion *version.Version, currentVersion *version.Version) *version.Version { + if latestVersion == nil || currentVersion.GreaterThan(latestVersion) { + latestVersion = currentVersion + } + return latestVersion +} From 32c11963db078574e0062674073d81192896944d Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Wed, 20 Jan 2021 16:59:08 +0530 Subject: [PATCH 02/15] tests for remote module --- pkg/cli/register.go | 7 - pkg/cli/run_test.go | 11 ++ pkg/cli/testdata/run-test/remote-modules.tf | 18 +++ .../terraform/commons/load-dir.go | 7 + .../commons/remote-module-download_test.go | 120 ++++++++++++++++++ 5 files changed, 156 insertions(+), 7 deletions(-) create mode 100644 pkg/cli/testdata/run-test/remote-modules.tf create mode 100644 pkg/iac-providers/terraform/commons/remote-module-download_test.go diff --git a/pkg/cli/register.go b/pkg/cli/register.go index c4dd9c953..98da7e4b9 100644 --- a/pkg/cli/register.go +++ b/pkg/cli/register.go @@ -18,8 +18,6 @@ package cli import ( "flag" - "io/ioutil" - "log" "os" "github.com/accurics/terrascan/pkg/config" @@ -61,11 +59,6 @@ func Execute() { } } - // disable terraform logs when TF_LOG env variable is not set - if os.Getenv("TF_LOG") == "" { - log.SetOutput(ioutil.Discard) - } - if err := rootCmd.Execute(); err != nil { os.Exit(1) } diff --git a/pkg/cli/run_test.go b/pkg/cli/run_test.go index 6cee107ee..ca88eda42 100644 --- a/pkg/cli/run_test.go +++ b/pkg/cli/run_test.go @@ -50,6 +50,8 @@ func TestRun(t *testing.T) { testDirPath := "testdata/run-test" kustomizeTestDirPath := testDirPath + "/kustomize-test" testTerraformFilePath := testDirPath + "/config-only.tf" + testRemoteModuleFilePath := testDirPath + "/remote-modules.tf" + ruleSlice := []string{"AWS.ECR.DataSecurity.High.0579", "AWS.SecurityGroup.NetworkPortsSecurity.Low.0561"} table := []struct { @@ -171,6 +173,15 @@ func TestRun(t *testing.T) { configFile: "testdata/configFile.toml", }, }, + { + name: "config file with remote module", + scanOptions: &ScanOptions{ + policyType: []string{"all"}, + iacFilePath: testRemoteModuleFilePath, + outputType: "human", + configFile: "testdata/configFile.toml", + }, + }, } for _, tt := range table { diff --git a/pkg/cli/testdata/run-test/remote-modules.tf b/pkg/cli/testdata/run-test/remote-modules.tf new file mode 100644 index 000000000..09c2ca082 --- /dev/null +++ b/pkg/cli/testdata/run-test/remote-modules.tf @@ -0,0 +1,18 @@ +#file added for fix for #418 + +# source https://registry.terraform.io/ + +module "network" { + source = "Azure/network/azurerm" + version = "3.2.1" +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" +} + +## contains local modules +module "rds" { + source = "terraform-aws-modules/rds/aws" + version = "2.20.0" +} \ No newline at end of file diff --git a/pkg/iac-providers/terraform/commons/load-dir.go b/pkg/iac-providers/terraform/commons/load-dir.go index 4869ea526..5197fed74 100644 --- a/pkg/iac-providers/terraform/commons/load-dir.go +++ b/pkg/iac-providers/terraform/commons/load-dir.go @@ -18,6 +18,8 @@ package commons import ( "fmt" + "io/ioutil" + "log" "os" "path/filepath" "strings" @@ -55,6 +57,11 @@ type ModuleConfig struct { // resources present in rootDir and descendant modules func LoadIacDir(absRootDir string) (allResourcesConfig output.AllResourceConfigs, err error) { + // disable terraform logs when TF_LOG env variable is not set + if os.Getenv("TF_LOG") == "" { + log.SetOutput(ioutil.Discard) + } + // map to hold the download paths of remote modules remoteModPaths := make(map[string]string) diff --git a/pkg/iac-providers/terraform/commons/remote-module-download_test.go b/pkg/iac-providers/terraform/commons/remote-module-download_test.go new file mode 100644 index 000000000..b47b4bb07 --- /dev/null +++ b/pkg/iac-providers/terraform/commons/remote-module-download_test.go @@ -0,0 +1,120 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package commons + +import ( + "io/ioutil" + "log" + "os" + "path/filepath" + "testing" + + "github.com/accurics/terrascan/pkg/downloader" + "github.com/accurics/terrascan/pkg/utils" + version "github.com/hashicorp/go-version" + hclConfigs "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry/regsrc" +) + +func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { + + // disable terraform logs when TF_LOG env variable is not set + if os.Getenv("TF_LOG") == "" { + log.SetOutput(ioutil.Discard) + } + + testConstraintsSecurityGroup, _ := version.NewConstraint("3.17.0") + testModuleSecurityGroup, _ := regsrc.ParseModuleSource("terraform-aws-modules/security-group/aws") + testModuleInvalidProvider, _ := regsrc.ParseModuleSource("terraform-aws-modules/testgroup/testprovider") + + type fields struct { + cache InstalledCache + downloader downloader.Downloader + } + type args struct { + requiredVersion hclConfigs.VersionConstraint + destPath string + module *regsrc.Module + } + tests := []struct { + name string + fields fields + args args + want string + wantErr bool + }{ + { + name: "remote module download with valid module and version", + fields: fields{ + cache: make(map[string]string), + downloader: downloader.NewDownloader(), + }, + args: args{ + requiredVersion: hclConfigs.VersionConstraint{ + Required: testConstraintsSecurityGroup, + }, + module: testModuleSecurityGroup, + }, + }, + { + name: "remote module download with valid module, without version", + fields: fields{ + cache: make(map[string]string), + downloader: downloader.NewDownloader(), + }, + args: args{ + requiredVersion: hclConfigs.VersionConstraint{}, + module: testModuleSecurityGroup, + }, + }, + { + name: "remote module download with invalid module source", + fields: fields{ + cache: make(map[string]string), + downloader: downloader.NewDownloader(), + }, + args: args{ + requiredVersion: hclConfigs.VersionConstraint{}, + module: testModuleInvalidProvider, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &RemoteModuleInstaller{ + cache: tt.fields.cache, + downloader: tt.fields.downloader, + } + testDir := filepath.Join(os.TempDir(), utils.GenRandomString(6)) + tt.want = testDir + if tt.wantErr { + tt.want = "" + } + + defer os.RemoveAll(testDir) + got, err := r.DownloadRemoteModule(tt.args.requiredVersion, testDir, tt.args.module) + if (err != nil) != tt.wantErr { + t.Errorf("RemoteModuleInstaller.DownloadRemoteModule() got error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("RemoteModuleInstaller.DownloadRemoteModule() got = %v, want %v", got, tt.want) + } + }) + } +} From 8cb68f181180e7aef0667822f8f6308bff1d502d Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Wed, 20 Jan 2021 17:11:54 +0530 Subject: [PATCH 03/15] fix static check failure --- .../terraform/commons/remote-module-download_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/iac-providers/terraform/commons/remote-module-download_test.go b/pkg/iac-providers/terraform/commons/remote-module-download_test.go index b47b4bb07..00642473e 100644 --- a/pkg/iac-providers/terraform/commons/remote-module-download_test.go +++ b/pkg/iac-providers/terraform/commons/remote-module-download_test.go @@ -47,7 +47,6 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { } type args struct { requiredVersion hclConfigs.VersionConstraint - destPath string module *regsrc.Module } tests := []struct { From e64db7fe23edf0bd4cfcd2025de51bfa9c4db3ec Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Thu, 21 Jan 2021 10:31:14 +0530 Subject: [PATCH 04/15] refactor DownloadRemoteModule func and more tests --- .../commons/remote-module-download.go | 71 +++++---- .../commons/remote-module-download_test.go | 148 +++++++++++++++++- 2 files changed, 185 insertions(+), 34 deletions(-) diff --git a/pkg/iac-providers/terraform/commons/remote-module-download.go b/pkg/iac-providers/terraform/commons/remote-module-download.go index 3dd6b585a..b6120eee0 100644 --- a/pkg/iac-providers/terraform/commons/remote-module-download.go +++ b/pkg/iac-providers/terraform/commons/remote-module-download.go @@ -24,6 +24,7 @@ import ( hclConfigs "github.com/hashicorp/terraform/configs" "github.com/hashicorp/terraform/registry" "github.com/hashicorp/terraform/registry/regsrc" + "github.com/hashicorp/terraform/registry/response" "go.uber.org/zap" ) @@ -53,6 +54,44 @@ func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs. return "", err } + // get the version to download + versionToDownload, err := getVersionToDownload(moduleVersions, requiredVersion, module) + if err != nil { + zap.S().Error("error while fetching the version to download,", zap.Error(err)) + return "", err + } + + // get the source location for the matched version + sourceLocation, err := regClient.ModuleLocation(module, versionToDownload.String()) + if err != nil { + zap.S().Errorf("error while getting the source location for module: %s, at registry: %s", module.String(), module.Host().Display()) + return "", err + } + + downloadLocation, err := r.DownloadModule(sourceLocation, destPath) + if err != nil { + zap.S().Errorf("error while downloading module: %s, with source location: %s", module.String(), sourceLocation) + return "", nil + } + + if module.RawSubmodule != "" { + // Append the user's requested subdirectory + downloadLocation = filepath.Join(downloadLocation, module.RawSubmodule) + } + + return downloadLocation, nil +} + +// helper func to compare and update the version +func getGreaterVersion(latestVersion *version.Version, currentVersion *version.Version) *version.Version { + if latestVersion == nil || currentVersion.GreaterThan(latestVersion) { + latestVersion = currentVersion + } + return latestVersion +} + +// helper func to get the module version to download +func getVersionToDownload(moduleVersions *response.ModuleVersions, requiredVersion hclConfigs.VersionConstraint, module *regsrc.Module) (*version.Version, error) { // terraform init command pulls all the available versions of a module, // and downloads the latest non pre-release (unless a pre-release version was // specified in tf file) version, if a version constraint is not provided in the tf file. @@ -95,11 +134,11 @@ func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs. } if latestVersion == nil { - return "", fmt.Errorf("no versions for module: %s, found at registry: %s", module.String(), module.Host().Display()) + return nil, fmt.Errorf("no versions for module: %s, found at registry: %s", module.String(), module.Host().Display()) } if requiredVersion.Required != nil && latestMatch == nil { - return "", fmt.Errorf("no versions matching: %s, for module: %s, found at registry: %s, latest version found: %s", requiredVersion.Required.String(), module.String(), module.Host().Display(), latestVersion.String()) + return nil, fmt.Errorf("no versions matching: %s, for module: %s, found at registry: %s, latest version found: %s", requiredVersion.Required.String(), module.String(), module.Host().Display(), latestVersion.String()) } versionToDownload = latestVersion @@ -107,31 +146,5 @@ func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs. versionToDownload = latestMatch } - // get the source location for the matched version - sourceLocation, err := regClient.ModuleLocation(module, versionToDownload.String()) - if err != nil { - zap.S().Errorf("error while getting the source location for module: %s, at registry: %s", module.String(), module.Host().Display()) - return "", err - } - - downloadLocation, err := r.DownloadModule(sourceLocation, destPath) - if err != nil { - zap.S().Errorf("error while downloading module: %s, with source location: %s", module.String(), sourceLocation) - return "", nil - } - - if module.RawSubmodule != "" { - // Append the user's requested subdirectory to any subdirectory that - // was implied by any of the nested layers we expanded within go-getter. - downloadLocation = filepath.Join(downloadLocation, module.RawSubmodule) - } - - return downloadLocation, nil -} - -func getGreaterVersion(latestVersion *version.Version, currentVersion *version.Version) *version.Version { - if latestVersion == nil || currentVersion.GreaterThan(latestVersion) { - latestVersion = currentVersion - } - return latestVersion + return versionToDownload, nil } diff --git a/pkg/iac-providers/terraform/commons/remote-module-download_test.go b/pkg/iac-providers/terraform/commons/remote-module-download_test.go index 00642473e..12b8144c6 100644 --- a/pkg/iac-providers/terraform/commons/remote-module-download_test.go +++ b/pkg/iac-providers/terraform/commons/remote-module-download_test.go @@ -21,6 +21,7 @@ import ( "log" "os" "path/filepath" + "reflect" "testing" "github.com/accurics/terrascan/pkg/downloader" @@ -28,6 +29,7 @@ import ( version "github.com/hashicorp/go-version" hclConfigs "github.com/hashicorp/terraform/configs" "github.com/hashicorp/terraform/registry/regsrc" + "github.com/hashicorp/terraform/registry/response" ) func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { @@ -40,6 +42,8 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { testConstraintsSecurityGroup, _ := version.NewConstraint("3.17.0") testModuleSecurityGroup, _ := regsrc.ParseModuleSource("terraform-aws-modules/security-group/aws") testModuleInvalidProvider, _ := regsrc.ParseModuleSource("terraform-aws-modules/testgroup/testprovider") + testModuleRdsWithRawSubModule, _ := regsrc.ParseModuleSource("terraform-aws-modules/security-group/aws//db_subnet_group") + testRawSubModuleName := "db_subnet_group" type fields struct { cache InstalledCache @@ -50,11 +54,13 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { module *regsrc.Module } tests := []struct { - name string - fields fields - args args - want string - wantErr bool + name string + fields fields + args args + want string + wantErr bool + hasRawSubModule bool + rawSubModule string }{ { name: "remote module download with valid module and version", @@ -92,6 +98,19 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { }, wantErr: true, }, + { + name: "remote module download with raw sub module", + fields: fields{ + cache: make(map[string]string), + downloader: downloader.NewDownloader(), + }, + args: args{ + requiredVersion: hclConfigs.VersionConstraint{}, + module: testModuleRdsWithRawSubModule, + }, + hasRawSubModule: true, + rawSubModule: testRawSubModuleName, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -105,6 +124,10 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { tt.want = "" } + if tt.hasRawSubModule { + tt.want = tt.want + string(os.PathSeparator) + tt.rawSubModule + } + defer os.RemoveAll(testDir) got, err := r.DownloadRemoteModule(tt.args.requiredVersion, testDir, tt.args.module) if (err != nil) != tt.wantErr { @@ -117,3 +140,118 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { }) } } + +func TestGetVersionToDownload(t *testing.T) { + source := "terraform-aws-modules/security-group/aws" + testModule, _ := regsrc.ParseModuleSource(source) + testVersionConstraint, _ := version.NewConstraint("3.17.0") + testVersionConstraintMatching, _ := version.NewConstraint("1.2.0") + testVersion, _ := version.NewVersion("1.2.0") + testVersions := []*response.ModuleVersion{ + { + Version: "1.0.0", + }, + { + Version: "1.1.0", + }, + { + Version: "1.2.0", + }, + } + + type args struct { + moduleVersions *response.ModuleVersions + requiredVersion hclConfigs.VersionConstraint + module *regsrc.Module + } + tests := []struct { + name string + args args + want *version.Version + wantErr bool + }{ + { + name: "invalid version", + args: args{ + moduleVersions: &response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{ + { + Source: "", + Versions: []*response.ModuleVersion{ + {}, + }, + }, + }, + }, + module: testModule, + requiredVersion: hclConfigs.VersionConstraint{ + Required: testVersionConstraint, + }, + }, + wantErr: true, + }, + { + name: "no matching versions", + args: args{ + moduleVersions: &response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{ + { + Source: "source", + Versions: testVersions, + }, + }, + }, + module: testModule, + requiredVersion: hclConfigs.VersionConstraint{ + Required: testVersionConstraint, + }, + }, + wantErr: true, + }, + { + name: "no required version specified", + args: args{ + moduleVersions: &response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{ + { + Source: "source", + Versions: testVersions, + }, + }, + }, + module: testModule, + }, + want: testVersion, + }, + { + name: "required version specified", + args: args{ + moduleVersions: &response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{ + { + Source: "source", + Versions: testVersions, + }, + }, + }, + module: testModule, + requiredVersion: hclConfigs.VersionConstraint{ + Required: testVersionConstraintMatching, + }, + }, + want: testVersion, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getVersionToDownload(tt.args.moduleVersions, tt.args.requiredVersion, tt.args.module) + if (err != nil) != tt.wantErr { + t.Errorf("getVersionToDownload() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getVersionToDownload() = %v, want %v", got, tt.want) + } + }) + } +} From bfbd608ee26582ddc0289eac86b08b989de8a993 Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Wed, 20 Jan 2021 13:44:07 +0530 Subject: [PATCH 05/15] 1. initial changes for registry module support 2. fix issue of remote module containing local modules --- go.sum | 6 + pkg/cli/register.go | 7 + .../terraform/commons/load-dir.go | 35 ++++- .../terraform/commons/module-download.go | 11 ++ .../commons/remote-module-download.go | 137 ++++++++++++++++++ 5 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 pkg/iac-providers/terraform/commons/remote-module-download.go diff --git a/go.sum b/go.sum index 85da99b21..e4d732c13 100644 --- a/go.sum +++ b/go.sum @@ -1503,6 +1503,12 @@ sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbL sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/kustomize/api v0.7.2 h1:ItTD/2XaKO8CosOMFZdaGFdUGTCHdQriW7zQ7AR98rs= sigs.k8s.io/kustomize/api v0.7.2/go.mod h1:50/vLATrjhRmMr3spZsI1GcpoZJ8IARy9QstPbA9lGE= +sigs.k8s.io/kustomize/api v0.7.1 h1:/cjDi4Pk/hqRSeCCj/Xum66rYrEtc7osM2/O+lvYKkM= +sigs.k8s.io/kustomize/api v0.7.1/go.mod h1:XOt24UrCkv0x63eT5JVaph4Kqf5EVU2UBAXo6SPBaAY= +sigs.k8s.io/kustomize/api v0.7.2 h1:ItTD/2XaKO8CosOMFZdaGFdUGTCHdQriW7zQ7AR98rs= +sigs.k8s.io/kustomize/api v0.7.2/go.mod h1:50/vLATrjhRmMr3spZsI1GcpoZJ8IARy9QstPbA9lGE= +sigs.k8s.io/kustomize/kyaml v0.10.5 h1:PbJcsZsEM7O3hHtUWTR+4WkHVbQRW9crSy75or1gRbI= +sigs.k8s.io/kustomize/kyaml v0.10.5/go.mod h1:P6Oy/ah/GZMKzJMIJA2a3/bc8YrBkuL5kJji13PSIzY= sigs.k8s.io/kustomize/kyaml v0.10.6 h1:xUJxc/k8JoWqHUahaB8DTqY0KwEPxTbTGStvW8TOcDc= sigs.k8s.io/kustomize/kyaml v0.10.6/go.mod h1:K9yg1k/HB/6xNOf5VH3LhTo1DK9/5ykSZO5uIv+Y/1k= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= diff --git a/pkg/cli/register.go b/pkg/cli/register.go index 98da7e4b9..c4dd9c953 100644 --- a/pkg/cli/register.go +++ b/pkg/cli/register.go @@ -18,6 +18,8 @@ package cli import ( "flag" + "io/ioutil" + "log" "os" "github.com/accurics/terrascan/pkg/config" @@ -59,6 +61,11 @@ func Execute() { } } + // disable terraform logs when TF_LOG env variable is not set + if os.Getenv("TF_LOG") == "" { + log.SetOutput(ioutil.Discard) + } + if err := rootCmd.Execute(); err != nil { os.Exit(1) } diff --git a/pkg/iac-providers/terraform/commons/load-dir.go b/pkg/iac-providers/terraform/commons/load-dir.go index 9aae1aa2a..4869ea526 100644 --- a/pkg/iac-providers/terraform/commons/load-dir.go +++ b/pkg/iac-providers/terraform/commons/load-dir.go @@ -20,12 +20,14 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/accurics/terrascan/pkg/iac-providers/output" "github.com/accurics/terrascan/pkg/utils" version "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2" hclConfigs "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry/regsrc" "github.com/spf13/afero" "go.uber.org/zap" ) @@ -53,6 +55,9 @@ type ModuleConfig struct { // resources present in rootDir and descendant modules func LoadIacDir(absRootDir string) (allResourcesConfig output.AllResourceConfigs, err error) { + // map to hold the download paths of remote modules + remoteModPaths := make(map[string]string) + // create a new config parser parser := hclConfigs.NewParser(afero.NewOsFs()) @@ -92,8 +97,36 @@ func LoadIacDir(absRootDir string) (allResourcesConfig output.AllResourceConfigs for p := req.Parent; p != nil; p = p.Parent { pathToModule = filepath.Join(p.SourceAddr, pathToModule) } - pathToModule = filepath.Join(absRootDir, pathToModule) + + // check if pathToModule consists of a remote module downloaded + // if yes, then update the module path, with the remote module + // download path + keyFound := false + for remoteSourceAddr, downloadPath := range remoteModPaths { + if strings.Contains(pathToModule, remoteSourceAddr) { + pathToModule = strings.Replace(pathToModule, remoteSourceAddr, "", 1) + pathToModule = downloadPath + pathToModule + keyFound = true + break + } + } + if !keyFound { + pathToModule = filepath.Join(absRootDir, pathToModule) + } zap.S().Debugf("processing local module %q", pathToModule) + } else if isRegistrySourceAddr(req.SourceAddr) { + // regsrc.ParseModuleSource func returns a terraform registry module source + // error check is not required as the source address is already validated above + module, _ := regsrc.ParseModuleSource(req.SourceAddr) + + tempDir := filepath.Join(os.TempDir(), utils.GenRandomString(6)) + pathToModule, err = r.DownloadRemoteModule(req.VersionConstraint, tempDir, module) + if err != nil { + zap.S().Errorf("failed to download remote module %q. error: '%v'", req.SourceAddr, err) + } else { + // add the entry of remote module's source address to the map + remoteModPaths[req.SourceAddr] = pathToModule + } } else { // temp dir to download the remote repo tempDir := filepath.Join(os.TempDir(), utils.GenRandomString(6)) diff --git a/pkg/iac-providers/terraform/commons/module-download.go b/pkg/iac-providers/terraform/commons/module-download.go index 40fb9b364..f24f667de 100644 --- a/pkg/iac-providers/terraform/commons/module-download.go +++ b/pkg/iac-providers/terraform/commons/module-download.go @@ -23,6 +23,8 @@ import ( "github.com/accurics/terrascan/pkg/downloader" "go.uber.org/zap" + + "github.com/hashicorp/terraform/registry/regsrc" ) var localSourcePrefixes = []string{ @@ -41,6 +43,15 @@ func isLocalSourceAddr(addr string) bool { return false } +// isRegistrySourceAddr will validate if the source address is a valid registry +// module or not. +// a valid source address is of the form /NAMESPACE>// +// regsrc.ParseModuleSource func returns a terraform registry module source. +func isRegistrySourceAddr(addr string) bool { + _, err := regsrc.ParseModuleSource(addr) + return err == nil +} + // RemoteModuleInstaller helps in downloading remote modules, it also maintains a // cache of all the installed modules and their respective resolved addresses // (URL) diff --git a/pkg/iac-providers/terraform/commons/remote-module-download.go b/pkg/iac-providers/terraform/commons/remote-module-download.go new file mode 100644 index 000000000..3dd6b585a --- /dev/null +++ b/pkg/iac-providers/terraform/commons/remote-module-download.go @@ -0,0 +1,137 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package commons + +import ( + "fmt" + "path/filepath" + + version "github.com/hashicorp/go-version" + hclConfigs "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry" + "github.com/hashicorp/terraform/registry/regsrc" + "go.uber.org/zap" +) + +// DownloadRemoteModule will download remote modules from public and private terraform registries +// this function takes similar approach taken by terraform init for downloading terraform registry modules +func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs.VersionConstraint, destPath string, module *regsrc.Module) (string, error) { + // Terraform doesn't allow the hostname to contain Punycode + // module.SvcHost returns an error for such case + _, err := module.SvcHost() + if err != nil { + zap.S().Errorf("hostname for the module %s is invalid", module.String()) + return "", err + } + + // get terraform registry client. + // terraform registry client provides methods for querying the terraform module registry + regClient := registry.NewClient(nil, nil) + + // get all the available module versions from the terraform registry + moduleVersions, err := regClient.ModuleVersions(module) + if err != nil { + if registry.IsModuleNotFound(err) { + zap.S().Errorf("module: %s, not be found at registry: %s", module.String(), module.Host().Display()) + } else { + zap.S().Errorf("error while fetching available modules for module: %s, at registry: %s", module.String(), module.Host().Display()) + } + return "", err + } + + // terraform init command pulls all the available versions of a module, + // and downloads the latest non pre-release (unless a pre-release version was + // specified in tf file) version, if a version constraint is not provided in the tf file. + // we are following what terraform does + allModules := moduleVersions.Modules[0] + + var latestMatch *version.Version + var latestVersion *version.Version + var versionToDownload *version.Version + for _, moduleVersion := range allModules.Versions { + currentVersion, err := version.NewVersion(moduleVersion.Version) + if err != nil { + // if error is received for a version, then skip the current version + zap.S().Errorf("invalid version: %s, for module: %s, at registry: %s", moduleVersion.Version, module.String(), module.Host().Display()) + continue + } + + if requiredVersion.Required == nil { + // skip the pre release version + if currentVersion.Prerelease() != "" { + continue + } + + // update the latest version + latestVersion = getGreaterVersion(latestVersion, currentVersion) + } else { + // skip the pre-release version, unless specified in the tf file + if currentVersion.Prerelease() != "" && requiredVersion.Required.String() != currentVersion.String() { + continue + } + + // update the latest version + latestVersion = getGreaterVersion(latestVersion, currentVersion) + + // update latest match + if requiredVersion.Required.Check(currentVersion) { + latestMatch = getGreaterVersion(latestMatch, currentVersion) + } + } + } + + if latestVersion == nil { + return "", fmt.Errorf("no versions for module: %s, found at registry: %s", module.String(), module.Host().Display()) + } + + if requiredVersion.Required != nil && latestMatch == nil { + return "", fmt.Errorf("no versions matching: %s, for module: %s, found at registry: %s, latest version found: %s", requiredVersion.Required.String(), module.String(), module.Host().Display(), latestVersion.String()) + } + + versionToDownload = latestVersion + if latestMatch != nil { + versionToDownload = latestMatch + } + + // get the source location for the matched version + sourceLocation, err := regClient.ModuleLocation(module, versionToDownload.String()) + if err != nil { + zap.S().Errorf("error while getting the source location for module: %s, at registry: %s", module.String(), module.Host().Display()) + return "", err + } + + downloadLocation, err := r.DownloadModule(sourceLocation, destPath) + if err != nil { + zap.S().Errorf("error while downloading module: %s, with source location: %s", module.String(), sourceLocation) + return "", nil + } + + if module.RawSubmodule != "" { + // Append the user's requested subdirectory to any subdirectory that + // was implied by any of the nested layers we expanded within go-getter. + downloadLocation = filepath.Join(downloadLocation, module.RawSubmodule) + } + + return downloadLocation, nil +} + +func getGreaterVersion(latestVersion *version.Version, currentVersion *version.Version) *version.Version { + if latestVersion == nil || currentVersion.GreaterThan(latestVersion) { + latestVersion = currentVersion + } + return latestVersion +} From 48ae5d985f338f877875dc8e27996254541b80f9 Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Wed, 20 Jan 2021 16:59:08 +0530 Subject: [PATCH 06/15] tests for remote module --- pkg/cli/register.go | 7 - pkg/cli/run_test.go | 11 ++ pkg/cli/testdata/run-test/remote-modules.tf | 18 +++ .../terraform/commons/load-dir.go | 7 + .../commons/remote-module-download_test.go | 120 ++++++++++++++++++ 5 files changed, 156 insertions(+), 7 deletions(-) create mode 100644 pkg/cli/testdata/run-test/remote-modules.tf create mode 100644 pkg/iac-providers/terraform/commons/remote-module-download_test.go diff --git a/pkg/cli/register.go b/pkg/cli/register.go index c4dd9c953..98da7e4b9 100644 --- a/pkg/cli/register.go +++ b/pkg/cli/register.go @@ -18,8 +18,6 @@ package cli import ( "flag" - "io/ioutil" - "log" "os" "github.com/accurics/terrascan/pkg/config" @@ -61,11 +59,6 @@ func Execute() { } } - // disable terraform logs when TF_LOG env variable is not set - if os.Getenv("TF_LOG") == "" { - log.SetOutput(ioutil.Discard) - } - if err := rootCmd.Execute(); err != nil { os.Exit(1) } diff --git a/pkg/cli/run_test.go b/pkg/cli/run_test.go index 6cee107ee..ca88eda42 100644 --- a/pkg/cli/run_test.go +++ b/pkg/cli/run_test.go @@ -50,6 +50,8 @@ func TestRun(t *testing.T) { testDirPath := "testdata/run-test" kustomizeTestDirPath := testDirPath + "/kustomize-test" testTerraformFilePath := testDirPath + "/config-only.tf" + testRemoteModuleFilePath := testDirPath + "/remote-modules.tf" + ruleSlice := []string{"AWS.ECR.DataSecurity.High.0579", "AWS.SecurityGroup.NetworkPortsSecurity.Low.0561"} table := []struct { @@ -171,6 +173,15 @@ func TestRun(t *testing.T) { configFile: "testdata/configFile.toml", }, }, + { + name: "config file with remote module", + scanOptions: &ScanOptions{ + policyType: []string{"all"}, + iacFilePath: testRemoteModuleFilePath, + outputType: "human", + configFile: "testdata/configFile.toml", + }, + }, } for _, tt := range table { diff --git a/pkg/cli/testdata/run-test/remote-modules.tf b/pkg/cli/testdata/run-test/remote-modules.tf new file mode 100644 index 000000000..09c2ca082 --- /dev/null +++ b/pkg/cli/testdata/run-test/remote-modules.tf @@ -0,0 +1,18 @@ +#file added for fix for #418 + +# source https://registry.terraform.io/ + +module "network" { + source = "Azure/network/azurerm" + version = "3.2.1" +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" +} + +## contains local modules +module "rds" { + source = "terraform-aws-modules/rds/aws" + version = "2.20.0" +} \ No newline at end of file diff --git a/pkg/iac-providers/terraform/commons/load-dir.go b/pkg/iac-providers/terraform/commons/load-dir.go index 4869ea526..5197fed74 100644 --- a/pkg/iac-providers/terraform/commons/load-dir.go +++ b/pkg/iac-providers/terraform/commons/load-dir.go @@ -18,6 +18,8 @@ package commons import ( "fmt" + "io/ioutil" + "log" "os" "path/filepath" "strings" @@ -55,6 +57,11 @@ type ModuleConfig struct { // resources present in rootDir and descendant modules func LoadIacDir(absRootDir string) (allResourcesConfig output.AllResourceConfigs, err error) { + // disable terraform logs when TF_LOG env variable is not set + if os.Getenv("TF_LOG") == "" { + log.SetOutput(ioutil.Discard) + } + // map to hold the download paths of remote modules remoteModPaths := make(map[string]string) diff --git a/pkg/iac-providers/terraform/commons/remote-module-download_test.go b/pkg/iac-providers/terraform/commons/remote-module-download_test.go new file mode 100644 index 000000000..b47b4bb07 --- /dev/null +++ b/pkg/iac-providers/terraform/commons/remote-module-download_test.go @@ -0,0 +1,120 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package commons + +import ( + "io/ioutil" + "log" + "os" + "path/filepath" + "testing" + + "github.com/accurics/terrascan/pkg/downloader" + "github.com/accurics/terrascan/pkg/utils" + version "github.com/hashicorp/go-version" + hclConfigs "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry/regsrc" +) + +func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { + + // disable terraform logs when TF_LOG env variable is not set + if os.Getenv("TF_LOG") == "" { + log.SetOutput(ioutil.Discard) + } + + testConstraintsSecurityGroup, _ := version.NewConstraint("3.17.0") + testModuleSecurityGroup, _ := regsrc.ParseModuleSource("terraform-aws-modules/security-group/aws") + testModuleInvalidProvider, _ := regsrc.ParseModuleSource("terraform-aws-modules/testgroup/testprovider") + + type fields struct { + cache InstalledCache + downloader downloader.Downloader + } + type args struct { + requiredVersion hclConfigs.VersionConstraint + destPath string + module *regsrc.Module + } + tests := []struct { + name string + fields fields + args args + want string + wantErr bool + }{ + { + name: "remote module download with valid module and version", + fields: fields{ + cache: make(map[string]string), + downloader: downloader.NewDownloader(), + }, + args: args{ + requiredVersion: hclConfigs.VersionConstraint{ + Required: testConstraintsSecurityGroup, + }, + module: testModuleSecurityGroup, + }, + }, + { + name: "remote module download with valid module, without version", + fields: fields{ + cache: make(map[string]string), + downloader: downloader.NewDownloader(), + }, + args: args{ + requiredVersion: hclConfigs.VersionConstraint{}, + module: testModuleSecurityGroup, + }, + }, + { + name: "remote module download with invalid module source", + fields: fields{ + cache: make(map[string]string), + downloader: downloader.NewDownloader(), + }, + args: args{ + requiredVersion: hclConfigs.VersionConstraint{}, + module: testModuleInvalidProvider, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &RemoteModuleInstaller{ + cache: tt.fields.cache, + downloader: tt.fields.downloader, + } + testDir := filepath.Join(os.TempDir(), utils.GenRandomString(6)) + tt.want = testDir + if tt.wantErr { + tt.want = "" + } + + defer os.RemoveAll(testDir) + got, err := r.DownloadRemoteModule(tt.args.requiredVersion, testDir, tt.args.module) + if (err != nil) != tt.wantErr { + t.Errorf("RemoteModuleInstaller.DownloadRemoteModule() got error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("RemoteModuleInstaller.DownloadRemoteModule() got = %v, want %v", got, tt.want) + } + }) + } +} From f784302dd62adeda3d5700432eb6eb53e98a5dbf Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Wed, 20 Jan 2021 17:11:54 +0530 Subject: [PATCH 07/15] fix static check failure --- .../terraform/commons/remote-module-download_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/iac-providers/terraform/commons/remote-module-download_test.go b/pkg/iac-providers/terraform/commons/remote-module-download_test.go index b47b4bb07..00642473e 100644 --- a/pkg/iac-providers/terraform/commons/remote-module-download_test.go +++ b/pkg/iac-providers/terraform/commons/remote-module-download_test.go @@ -47,7 +47,6 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { } type args struct { requiredVersion hclConfigs.VersionConstraint - destPath string module *regsrc.Module } tests := []struct { From 19128c962228acfa8cff3c60641033eea5ade92f Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Thu, 21 Jan 2021 10:31:14 +0530 Subject: [PATCH 08/15] refactor DownloadRemoteModule func and more tests --- .../commons/remote-module-download.go | 71 +++++---- .../commons/remote-module-download_test.go | 148 +++++++++++++++++- 2 files changed, 185 insertions(+), 34 deletions(-) diff --git a/pkg/iac-providers/terraform/commons/remote-module-download.go b/pkg/iac-providers/terraform/commons/remote-module-download.go index 3dd6b585a..b6120eee0 100644 --- a/pkg/iac-providers/terraform/commons/remote-module-download.go +++ b/pkg/iac-providers/terraform/commons/remote-module-download.go @@ -24,6 +24,7 @@ import ( hclConfigs "github.com/hashicorp/terraform/configs" "github.com/hashicorp/terraform/registry" "github.com/hashicorp/terraform/registry/regsrc" + "github.com/hashicorp/terraform/registry/response" "go.uber.org/zap" ) @@ -53,6 +54,44 @@ func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs. return "", err } + // get the version to download + versionToDownload, err := getVersionToDownload(moduleVersions, requiredVersion, module) + if err != nil { + zap.S().Error("error while fetching the version to download,", zap.Error(err)) + return "", err + } + + // get the source location for the matched version + sourceLocation, err := regClient.ModuleLocation(module, versionToDownload.String()) + if err != nil { + zap.S().Errorf("error while getting the source location for module: %s, at registry: %s", module.String(), module.Host().Display()) + return "", err + } + + downloadLocation, err := r.DownloadModule(sourceLocation, destPath) + if err != nil { + zap.S().Errorf("error while downloading module: %s, with source location: %s", module.String(), sourceLocation) + return "", nil + } + + if module.RawSubmodule != "" { + // Append the user's requested subdirectory + downloadLocation = filepath.Join(downloadLocation, module.RawSubmodule) + } + + return downloadLocation, nil +} + +// helper func to compare and update the version +func getGreaterVersion(latestVersion *version.Version, currentVersion *version.Version) *version.Version { + if latestVersion == nil || currentVersion.GreaterThan(latestVersion) { + latestVersion = currentVersion + } + return latestVersion +} + +// helper func to get the module version to download +func getVersionToDownload(moduleVersions *response.ModuleVersions, requiredVersion hclConfigs.VersionConstraint, module *regsrc.Module) (*version.Version, error) { // terraform init command pulls all the available versions of a module, // and downloads the latest non pre-release (unless a pre-release version was // specified in tf file) version, if a version constraint is not provided in the tf file. @@ -95,11 +134,11 @@ func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs. } if latestVersion == nil { - return "", fmt.Errorf("no versions for module: %s, found at registry: %s", module.String(), module.Host().Display()) + return nil, fmt.Errorf("no versions for module: %s, found at registry: %s", module.String(), module.Host().Display()) } if requiredVersion.Required != nil && latestMatch == nil { - return "", fmt.Errorf("no versions matching: %s, for module: %s, found at registry: %s, latest version found: %s", requiredVersion.Required.String(), module.String(), module.Host().Display(), latestVersion.String()) + return nil, fmt.Errorf("no versions matching: %s, for module: %s, found at registry: %s, latest version found: %s", requiredVersion.Required.String(), module.String(), module.Host().Display(), latestVersion.String()) } versionToDownload = latestVersion @@ -107,31 +146,5 @@ func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs. versionToDownload = latestMatch } - // get the source location for the matched version - sourceLocation, err := regClient.ModuleLocation(module, versionToDownload.String()) - if err != nil { - zap.S().Errorf("error while getting the source location for module: %s, at registry: %s", module.String(), module.Host().Display()) - return "", err - } - - downloadLocation, err := r.DownloadModule(sourceLocation, destPath) - if err != nil { - zap.S().Errorf("error while downloading module: %s, with source location: %s", module.String(), sourceLocation) - return "", nil - } - - if module.RawSubmodule != "" { - // Append the user's requested subdirectory to any subdirectory that - // was implied by any of the nested layers we expanded within go-getter. - downloadLocation = filepath.Join(downloadLocation, module.RawSubmodule) - } - - return downloadLocation, nil -} - -func getGreaterVersion(latestVersion *version.Version, currentVersion *version.Version) *version.Version { - if latestVersion == nil || currentVersion.GreaterThan(latestVersion) { - latestVersion = currentVersion - } - return latestVersion + return versionToDownload, nil } diff --git a/pkg/iac-providers/terraform/commons/remote-module-download_test.go b/pkg/iac-providers/terraform/commons/remote-module-download_test.go index 00642473e..12b8144c6 100644 --- a/pkg/iac-providers/terraform/commons/remote-module-download_test.go +++ b/pkg/iac-providers/terraform/commons/remote-module-download_test.go @@ -21,6 +21,7 @@ import ( "log" "os" "path/filepath" + "reflect" "testing" "github.com/accurics/terrascan/pkg/downloader" @@ -28,6 +29,7 @@ import ( version "github.com/hashicorp/go-version" hclConfigs "github.com/hashicorp/terraform/configs" "github.com/hashicorp/terraform/registry/regsrc" + "github.com/hashicorp/terraform/registry/response" ) func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { @@ -40,6 +42,8 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { testConstraintsSecurityGroup, _ := version.NewConstraint("3.17.0") testModuleSecurityGroup, _ := regsrc.ParseModuleSource("terraform-aws-modules/security-group/aws") testModuleInvalidProvider, _ := regsrc.ParseModuleSource("terraform-aws-modules/testgroup/testprovider") + testModuleRdsWithRawSubModule, _ := regsrc.ParseModuleSource("terraform-aws-modules/security-group/aws//db_subnet_group") + testRawSubModuleName := "db_subnet_group" type fields struct { cache InstalledCache @@ -50,11 +54,13 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { module *regsrc.Module } tests := []struct { - name string - fields fields - args args - want string - wantErr bool + name string + fields fields + args args + want string + wantErr bool + hasRawSubModule bool + rawSubModule string }{ { name: "remote module download with valid module and version", @@ -92,6 +98,19 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { }, wantErr: true, }, + { + name: "remote module download with raw sub module", + fields: fields{ + cache: make(map[string]string), + downloader: downloader.NewDownloader(), + }, + args: args{ + requiredVersion: hclConfigs.VersionConstraint{}, + module: testModuleRdsWithRawSubModule, + }, + hasRawSubModule: true, + rawSubModule: testRawSubModuleName, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -105,6 +124,10 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { tt.want = "" } + if tt.hasRawSubModule { + tt.want = tt.want + string(os.PathSeparator) + tt.rawSubModule + } + defer os.RemoveAll(testDir) got, err := r.DownloadRemoteModule(tt.args.requiredVersion, testDir, tt.args.module) if (err != nil) != tt.wantErr { @@ -117,3 +140,118 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { }) } } + +func TestGetVersionToDownload(t *testing.T) { + source := "terraform-aws-modules/security-group/aws" + testModule, _ := regsrc.ParseModuleSource(source) + testVersionConstraint, _ := version.NewConstraint("3.17.0") + testVersionConstraintMatching, _ := version.NewConstraint("1.2.0") + testVersion, _ := version.NewVersion("1.2.0") + testVersions := []*response.ModuleVersion{ + { + Version: "1.0.0", + }, + { + Version: "1.1.0", + }, + { + Version: "1.2.0", + }, + } + + type args struct { + moduleVersions *response.ModuleVersions + requiredVersion hclConfigs.VersionConstraint + module *regsrc.Module + } + tests := []struct { + name string + args args + want *version.Version + wantErr bool + }{ + { + name: "invalid version", + args: args{ + moduleVersions: &response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{ + { + Source: "", + Versions: []*response.ModuleVersion{ + {}, + }, + }, + }, + }, + module: testModule, + requiredVersion: hclConfigs.VersionConstraint{ + Required: testVersionConstraint, + }, + }, + wantErr: true, + }, + { + name: "no matching versions", + args: args{ + moduleVersions: &response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{ + { + Source: "source", + Versions: testVersions, + }, + }, + }, + module: testModule, + requiredVersion: hclConfigs.VersionConstraint{ + Required: testVersionConstraint, + }, + }, + wantErr: true, + }, + { + name: "no required version specified", + args: args{ + moduleVersions: &response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{ + { + Source: "source", + Versions: testVersions, + }, + }, + }, + module: testModule, + }, + want: testVersion, + }, + { + name: "required version specified", + args: args{ + moduleVersions: &response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{ + { + Source: "source", + Versions: testVersions, + }, + }, + }, + module: testModule, + requiredVersion: hclConfigs.VersionConstraint{ + Required: testVersionConstraintMatching, + }, + }, + want: testVersion, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getVersionToDownload(tt.args.moduleVersions, tt.args.requiredVersion, tt.args.module) + if (err != nil) != tt.wantErr { + t.Errorf("getVersionToDownload() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getVersionToDownload() = %v, want %v", got, tt.want) + } + }) + } +} From 78fd1d2941c5d5743dbddd8bb1ab1011cca1d421 Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Wed, 20 Jan 2021 13:44:07 +0530 Subject: [PATCH 09/15] 1. initial changes for registry module support 2. fix issue of remote module containing local modules --- go.sum | 6 + pkg/cli/register.go | 7 + .../terraform/commons/load-dir.go | 35 ++++- .../terraform/commons/module-download.go | 11 ++ .../commons/remote-module-download.go | 137 ++++++++++++++++++ 5 files changed, 195 insertions(+), 1 deletion(-) create mode 100644 pkg/iac-providers/terraform/commons/remote-module-download.go diff --git a/go.sum b/go.sum index 56c59cb6d..f1d2e6441 100644 --- a/go.sum +++ b/go.sum @@ -1498,6 +1498,12 @@ sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbL sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/kustomize/api v0.7.2 h1:ItTD/2XaKO8CosOMFZdaGFdUGTCHdQriW7zQ7AR98rs= sigs.k8s.io/kustomize/api v0.7.2/go.mod h1:50/vLATrjhRmMr3spZsI1GcpoZJ8IARy9QstPbA9lGE= +sigs.k8s.io/kustomize/api v0.7.1 h1:/cjDi4Pk/hqRSeCCj/Xum66rYrEtc7osM2/O+lvYKkM= +sigs.k8s.io/kustomize/api v0.7.1/go.mod h1:XOt24UrCkv0x63eT5JVaph4Kqf5EVU2UBAXo6SPBaAY= +sigs.k8s.io/kustomize/api v0.7.2 h1:ItTD/2XaKO8CosOMFZdaGFdUGTCHdQriW7zQ7AR98rs= +sigs.k8s.io/kustomize/api v0.7.2/go.mod h1:50/vLATrjhRmMr3spZsI1GcpoZJ8IARy9QstPbA9lGE= +sigs.k8s.io/kustomize/kyaml v0.10.5 h1:PbJcsZsEM7O3hHtUWTR+4WkHVbQRW9crSy75or1gRbI= +sigs.k8s.io/kustomize/kyaml v0.10.5/go.mod h1:P6Oy/ah/GZMKzJMIJA2a3/bc8YrBkuL5kJji13PSIzY= sigs.k8s.io/kustomize/kyaml v0.10.6 h1:xUJxc/k8JoWqHUahaB8DTqY0KwEPxTbTGStvW8TOcDc= sigs.k8s.io/kustomize/kyaml v0.10.6/go.mod h1:K9yg1k/HB/6xNOf5VH3LhTo1DK9/5ykSZO5uIv+Y/1k= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ= diff --git a/pkg/cli/register.go b/pkg/cli/register.go index 98da7e4b9..c4dd9c953 100644 --- a/pkg/cli/register.go +++ b/pkg/cli/register.go @@ -18,6 +18,8 @@ package cli import ( "flag" + "io/ioutil" + "log" "os" "github.com/accurics/terrascan/pkg/config" @@ -59,6 +61,11 @@ func Execute() { } } + // disable terraform logs when TF_LOG env variable is not set + if os.Getenv("TF_LOG") == "" { + log.SetOutput(ioutil.Discard) + } + if err := rootCmd.Execute(); err != nil { os.Exit(1) } diff --git a/pkg/iac-providers/terraform/commons/load-dir.go b/pkg/iac-providers/terraform/commons/load-dir.go index 9aae1aa2a..4869ea526 100644 --- a/pkg/iac-providers/terraform/commons/load-dir.go +++ b/pkg/iac-providers/terraform/commons/load-dir.go @@ -20,12 +20,14 @@ import ( "fmt" "os" "path/filepath" + "strings" "github.com/accurics/terrascan/pkg/iac-providers/output" "github.com/accurics/terrascan/pkg/utils" version "github.com/hashicorp/go-version" "github.com/hashicorp/hcl/v2" hclConfigs "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry/regsrc" "github.com/spf13/afero" "go.uber.org/zap" ) @@ -53,6 +55,9 @@ type ModuleConfig struct { // resources present in rootDir and descendant modules func LoadIacDir(absRootDir string) (allResourcesConfig output.AllResourceConfigs, err error) { + // map to hold the download paths of remote modules + remoteModPaths := make(map[string]string) + // create a new config parser parser := hclConfigs.NewParser(afero.NewOsFs()) @@ -92,8 +97,36 @@ func LoadIacDir(absRootDir string) (allResourcesConfig output.AllResourceConfigs for p := req.Parent; p != nil; p = p.Parent { pathToModule = filepath.Join(p.SourceAddr, pathToModule) } - pathToModule = filepath.Join(absRootDir, pathToModule) + + // check if pathToModule consists of a remote module downloaded + // if yes, then update the module path, with the remote module + // download path + keyFound := false + for remoteSourceAddr, downloadPath := range remoteModPaths { + if strings.Contains(pathToModule, remoteSourceAddr) { + pathToModule = strings.Replace(pathToModule, remoteSourceAddr, "", 1) + pathToModule = downloadPath + pathToModule + keyFound = true + break + } + } + if !keyFound { + pathToModule = filepath.Join(absRootDir, pathToModule) + } zap.S().Debugf("processing local module %q", pathToModule) + } else if isRegistrySourceAddr(req.SourceAddr) { + // regsrc.ParseModuleSource func returns a terraform registry module source + // error check is not required as the source address is already validated above + module, _ := regsrc.ParseModuleSource(req.SourceAddr) + + tempDir := filepath.Join(os.TempDir(), utils.GenRandomString(6)) + pathToModule, err = r.DownloadRemoteModule(req.VersionConstraint, tempDir, module) + if err != nil { + zap.S().Errorf("failed to download remote module %q. error: '%v'", req.SourceAddr, err) + } else { + // add the entry of remote module's source address to the map + remoteModPaths[req.SourceAddr] = pathToModule + } } else { // temp dir to download the remote repo tempDir := filepath.Join(os.TempDir(), utils.GenRandomString(6)) diff --git a/pkg/iac-providers/terraform/commons/module-download.go b/pkg/iac-providers/terraform/commons/module-download.go index 40fb9b364..f24f667de 100644 --- a/pkg/iac-providers/terraform/commons/module-download.go +++ b/pkg/iac-providers/terraform/commons/module-download.go @@ -23,6 +23,8 @@ import ( "github.com/accurics/terrascan/pkg/downloader" "go.uber.org/zap" + + "github.com/hashicorp/terraform/registry/regsrc" ) var localSourcePrefixes = []string{ @@ -41,6 +43,15 @@ func isLocalSourceAddr(addr string) bool { return false } +// isRegistrySourceAddr will validate if the source address is a valid registry +// module or not. +// a valid source address is of the form /NAMESPACE>// +// regsrc.ParseModuleSource func returns a terraform registry module source. +func isRegistrySourceAddr(addr string) bool { + _, err := regsrc.ParseModuleSource(addr) + return err == nil +} + // RemoteModuleInstaller helps in downloading remote modules, it also maintains a // cache of all the installed modules and their respective resolved addresses // (URL) diff --git a/pkg/iac-providers/terraform/commons/remote-module-download.go b/pkg/iac-providers/terraform/commons/remote-module-download.go new file mode 100644 index 000000000..3dd6b585a --- /dev/null +++ b/pkg/iac-providers/terraform/commons/remote-module-download.go @@ -0,0 +1,137 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package commons + +import ( + "fmt" + "path/filepath" + + version "github.com/hashicorp/go-version" + hclConfigs "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry" + "github.com/hashicorp/terraform/registry/regsrc" + "go.uber.org/zap" +) + +// DownloadRemoteModule will download remote modules from public and private terraform registries +// this function takes similar approach taken by terraform init for downloading terraform registry modules +func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs.VersionConstraint, destPath string, module *regsrc.Module) (string, error) { + // Terraform doesn't allow the hostname to contain Punycode + // module.SvcHost returns an error for such case + _, err := module.SvcHost() + if err != nil { + zap.S().Errorf("hostname for the module %s is invalid", module.String()) + return "", err + } + + // get terraform registry client. + // terraform registry client provides methods for querying the terraform module registry + regClient := registry.NewClient(nil, nil) + + // get all the available module versions from the terraform registry + moduleVersions, err := regClient.ModuleVersions(module) + if err != nil { + if registry.IsModuleNotFound(err) { + zap.S().Errorf("module: %s, not be found at registry: %s", module.String(), module.Host().Display()) + } else { + zap.S().Errorf("error while fetching available modules for module: %s, at registry: %s", module.String(), module.Host().Display()) + } + return "", err + } + + // terraform init command pulls all the available versions of a module, + // and downloads the latest non pre-release (unless a pre-release version was + // specified in tf file) version, if a version constraint is not provided in the tf file. + // we are following what terraform does + allModules := moduleVersions.Modules[0] + + var latestMatch *version.Version + var latestVersion *version.Version + var versionToDownload *version.Version + for _, moduleVersion := range allModules.Versions { + currentVersion, err := version.NewVersion(moduleVersion.Version) + if err != nil { + // if error is received for a version, then skip the current version + zap.S().Errorf("invalid version: %s, for module: %s, at registry: %s", moduleVersion.Version, module.String(), module.Host().Display()) + continue + } + + if requiredVersion.Required == nil { + // skip the pre release version + if currentVersion.Prerelease() != "" { + continue + } + + // update the latest version + latestVersion = getGreaterVersion(latestVersion, currentVersion) + } else { + // skip the pre-release version, unless specified in the tf file + if currentVersion.Prerelease() != "" && requiredVersion.Required.String() != currentVersion.String() { + continue + } + + // update the latest version + latestVersion = getGreaterVersion(latestVersion, currentVersion) + + // update latest match + if requiredVersion.Required.Check(currentVersion) { + latestMatch = getGreaterVersion(latestMatch, currentVersion) + } + } + } + + if latestVersion == nil { + return "", fmt.Errorf("no versions for module: %s, found at registry: %s", module.String(), module.Host().Display()) + } + + if requiredVersion.Required != nil && latestMatch == nil { + return "", fmt.Errorf("no versions matching: %s, for module: %s, found at registry: %s, latest version found: %s", requiredVersion.Required.String(), module.String(), module.Host().Display(), latestVersion.String()) + } + + versionToDownload = latestVersion + if latestMatch != nil { + versionToDownload = latestMatch + } + + // get the source location for the matched version + sourceLocation, err := regClient.ModuleLocation(module, versionToDownload.String()) + if err != nil { + zap.S().Errorf("error while getting the source location for module: %s, at registry: %s", module.String(), module.Host().Display()) + return "", err + } + + downloadLocation, err := r.DownloadModule(sourceLocation, destPath) + if err != nil { + zap.S().Errorf("error while downloading module: %s, with source location: %s", module.String(), sourceLocation) + return "", nil + } + + if module.RawSubmodule != "" { + // Append the user's requested subdirectory to any subdirectory that + // was implied by any of the nested layers we expanded within go-getter. + downloadLocation = filepath.Join(downloadLocation, module.RawSubmodule) + } + + return downloadLocation, nil +} + +func getGreaterVersion(latestVersion *version.Version, currentVersion *version.Version) *version.Version { + if latestVersion == nil || currentVersion.GreaterThan(latestVersion) { + latestVersion = currentVersion + } + return latestVersion +} From 3b70d8859d23ee89246fd38617b97a266d1a7d22 Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Wed, 20 Jan 2021 16:59:08 +0530 Subject: [PATCH 10/15] tests for remote module --- pkg/cli/register.go | 7 - pkg/cli/run_test.go | 11 ++ pkg/cli/testdata/run-test/remote-modules.tf | 18 +++ .../terraform/commons/load-dir.go | 7 + .../commons/remote-module-download_test.go | 120 ++++++++++++++++++ 5 files changed, 156 insertions(+), 7 deletions(-) create mode 100644 pkg/cli/testdata/run-test/remote-modules.tf create mode 100644 pkg/iac-providers/terraform/commons/remote-module-download_test.go diff --git a/pkg/cli/register.go b/pkg/cli/register.go index c4dd9c953..98da7e4b9 100644 --- a/pkg/cli/register.go +++ b/pkg/cli/register.go @@ -18,8 +18,6 @@ package cli import ( "flag" - "io/ioutil" - "log" "os" "github.com/accurics/terrascan/pkg/config" @@ -61,11 +59,6 @@ func Execute() { } } - // disable terraform logs when TF_LOG env variable is not set - if os.Getenv("TF_LOG") == "" { - log.SetOutput(ioutil.Discard) - } - if err := rootCmd.Execute(); err != nil { os.Exit(1) } diff --git a/pkg/cli/run_test.go b/pkg/cli/run_test.go index 6cee107ee..ca88eda42 100644 --- a/pkg/cli/run_test.go +++ b/pkg/cli/run_test.go @@ -50,6 +50,8 @@ func TestRun(t *testing.T) { testDirPath := "testdata/run-test" kustomizeTestDirPath := testDirPath + "/kustomize-test" testTerraformFilePath := testDirPath + "/config-only.tf" + testRemoteModuleFilePath := testDirPath + "/remote-modules.tf" + ruleSlice := []string{"AWS.ECR.DataSecurity.High.0579", "AWS.SecurityGroup.NetworkPortsSecurity.Low.0561"} table := []struct { @@ -171,6 +173,15 @@ func TestRun(t *testing.T) { configFile: "testdata/configFile.toml", }, }, + { + name: "config file with remote module", + scanOptions: &ScanOptions{ + policyType: []string{"all"}, + iacFilePath: testRemoteModuleFilePath, + outputType: "human", + configFile: "testdata/configFile.toml", + }, + }, } for _, tt := range table { diff --git a/pkg/cli/testdata/run-test/remote-modules.tf b/pkg/cli/testdata/run-test/remote-modules.tf new file mode 100644 index 000000000..09c2ca082 --- /dev/null +++ b/pkg/cli/testdata/run-test/remote-modules.tf @@ -0,0 +1,18 @@ +#file added for fix for #418 + +# source https://registry.terraform.io/ + +module "network" { + source = "Azure/network/azurerm" + version = "3.2.1" +} + +module "eks" { + source = "terraform-aws-modules/eks/aws" +} + +## contains local modules +module "rds" { + source = "terraform-aws-modules/rds/aws" + version = "2.20.0" +} \ No newline at end of file diff --git a/pkg/iac-providers/terraform/commons/load-dir.go b/pkg/iac-providers/terraform/commons/load-dir.go index 4869ea526..5197fed74 100644 --- a/pkg/iac-providers/terraform/commons/load-dir.go +++ b/pkg/iac-providers/terraform/commons/load-dir.go @@ -18,6 +18,8 @@ package commons import ( "fmt" + "io/ioutil" + "log" "os" "path/filepath" "strings" @@ -55,6 +57,11 @@ type ModuleConfig struct { // resources present in rootDir and descendant modules func LoadIacDir(absRootDir string) (allResourcesConfig output.AllResourceConfigs, err error) { + // disable terraform logs when TF_LOG env variable is not set + if os.Getenv("TF_LOG") == "" { + log.SetOutput(ioutil.Discard) + } + // map to hold the download paths of remote modules remoteModPaths := make(map[string]string) diff --git a/pkg/iac-providers/terraform/commons/remote-module-download_test.go b/pkg/iac-providers/terraform/commons/remote-module-download_test.go new file mode 100644 index 000000000..b47b4bb07 --- /dev/null +++ b/pkg/iac-providers/terraform/commons/remote-module-download_test.go @@ -0,0 +1,120 @@ +/* + Copyright (C) 2020 Accurics, Inc. + + Licensed under the Apache License, Version 2.0 (the "License"); + you may not use this file except in compliance with the License. + You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + + Unless required by applicable law or agreed to in writing, software + distributed under the License is distributed on an "AS IS" BASIS, + WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + See the License for the specific language governing permissions and + limitations under the License. +*/ + +package commons + +import ( + "io/ioutil" + "log" + "os" + "path/filepath" + "testing" + + "github.com/accurics/terrascan/pkg/downloader" + "github.com/accurics/terrascan/pkg/utils" + version "github.com/hashicorp/go-version" + hclConfigs "github.com/hashicorp/terraform/configs" + "github.com/hashicorp/terraform/registry/regsrc" +) + +func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { + + // disable terraform logs when TF_LOG env variable is not set + if os.Getenv("TF_LOG") == "" { + log.SetOutput(ioutil.Discard) + } + + testConstraintsSecurityGroup, _ := version.NewConstraint("3.17.0") + testModuleSecurityGroup, _ := regsrc.ParseModuleSource("terraform-aws-modules/security-group/aws") + testModuleInvalidProvider, _ := regsrc.ParseModuleSource("terraform-aws-modules/testgroup/testprovider") + + type fields struct { + cache InstalledCache + downloader downloader.Downloader + } + type args struct { + requiredVersion hclConfigs.VersionConstraint + destPath string + module *regsrc.Module + } + tests := []struct { + name string + fields fields + args args + want string + wantErr bool + }{ + { + name: "remote module download with valid module and version", + fields: fields{ + cache: make(map[string]string), + downloader: downloader.NewDownloader(), + }, + args: args{ + requiredVersion: hclConfigs.VersionConstraint{ + Required: testConstraintsSecurityGroup, + }, + module: testModuleSecurityGroup, + }, + }, + { + name: "remote module download with valid module, without version", + fields: fields{ + cache: make(map[string]string), + downloader: downloader.NewDownloader(), + }, + args: args{ + requiredVersion: hclConfigs.VersionConstraint{}, + module: testModuleSecurityGroup, + }, + }, + { + name: "remote module download with invalid module source", + fields: fields{ + cache: make(map[string]string), + downloader: downloader.NewDownloader(), + }, + args: args{ + requiredVersion: hclConfigs.VersionConstraint{}, + module: testModuleInvalidProvider, + }, + wantErr: true, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + r := &RemoteModuleInstaller{ + cache: tt.fields.cache, + downloader: tt.fields.downloader, + } + testDir := filepath.Join(os.TempDir(), utils.GenRandomString(6)) + tt.want = testDir + if tt.wantErr { + tt.want = "" + } + + defer os.RemoveAll(testDir) + got, err := r.DownloadRemoteModule(tt.args.requiredVersion, testDir, tt.args.module) + if (err != nil) != tt.wantErr { + t.Errorf("RemoteModuleInstaller.DownloadRemoteModule() got error = %v, wantErr %v", err, tt.wantErr) + return + } + if got != tt.want { + t.Errorf("RemoteModuleInstaller.DownloadRemoteModule() got = %v, want %v", got, tt.want) + } + }) + } +} From bc015ac39c849bccd0fb5a0a223e59d498e503d8 Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Wed, 20 Jan 2021 17:11:54 +0530 Subject: [PATCH 11/15] fix static check failure --- .../terraform/commons/remote-module-download_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/pkg/iac-providers/terraform/commons/remote-module-download_test.go b/pkg/iac-providers/terraform/commons/remote-module-download_test.go index b47b4bb07..00642473e 100644 --- a/pkg/iac-providers/terraform/commons/remote-module-download_test.go +++ b/pkg/iac-providers/terraform/commons/remote-module-download_test.go @@ -47,7 +47,6 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { } type args struct { requiredVersion hclConfigs.VersionConstraint - destPath string module *regsrc.Module } tests := []struct { From 1fe039dd9060aabae2fc24cd8579b203fa733076 Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Thu, 21 Jan 2021 10:31:14 +0530 Subject: [PATCH 12/15] refactor DownloadRemoteModule func and more tests --- .../commons/remote-module-download.go | 71 +++++---- .../commons/remote-module-download_test.go | 148 +++++++++++++++++- 2 files changed, 185 insertions(+), 34 deletions(-) diff --git a/pkg/iac-providers/terraform/commons/remote-module-download.go b/pkg/iac-providers/terraform/commons/remote-module-download.go index 3dd6b585a..b6120eee0 100644 --- a/pkg/iac-providers/terraform/commons/remote-module-download.go +++ b/pkg/iac-providers/terraform/commons/remote-module-download.go @@ -24,6 +24,7 @@ import ( hclConfigs "github.com/hashicorp/terraform/configs" "github.com/hashicorp/terraform/registry" "github.com/hashicorp/terraform/registry/regsrc" + "github.com/hashicorp/terraform/registry/response" "go.uber.org/zap" ) @@ -53,6 +54,44 @@ func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs. return "", err } + // get the version to download + versionToDownload, err := getVersionToDownload(moduleVersions, requiredVersion, module) + if err != nil { + zap.S().Error("error while fetching the version to download,", zap.Error(err)) + return "", err + } + + // get the source location for the matched version + sourceLocation, err := regClient.ModuleLocation(module, versionToDownload.String()) + if err != nil { + zap.S().Errorf("error while getting the source location for module: %s, at registry: %s", module.String(), module.Host().Display()) + return "", err + } + + downloadLocation, err := r.DownloadModule(sourceLocation, destPath) + if err != nil { + zap.S().Errorf("error while downloading module: %s, with source location: %s", module.String(), sourceLocation) + return "", nil + } + + if module.RawSubmodule != "" { + // Append the user's requested subdirectory + downloadLocation = filepath.Join(downloadLocation, module.RawSubmodule) + } + + return downloadLocation, nil +} + +// helper func to compare and update the version +func getGreaterVersion(latestVersion *version.Version, currentVersion *version.Version) *version.Version { + if latestVersion == nil || currentVersion.GreaterThan(latestVersion) { + latestVersion = currentVersion + } + return latestVersion +} + +// helper func to get the module version to download +func getVersionToDownload(moduleVersions *response.ModuleVersions, requiredVersion hclConfigs.VersionConstraint, module *regsrc.Module) (*version.Version, error) { // terraform init command pulls all the available versions of a module, // and downloads the latest non pre-release (unless a pre-release version was // specified in tf file) version, if a version constraint is not provided in the tf file. @@ -95,11 +134,11 @@ func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs. } if latestVersion == nil { - return "", fmt.Errorf("no versions for module: %s, found at registry: %s", module.String(), module.Host().Display()) + return nil, fmt.Errorf("no versions for module: %s, found at registry: %s", module.String(), module.Host().Display()) } if requiredVersion.Required != nil && latestMatch == nil { - return "", fmt.Errorf("no versions matching: %s, for module: %s, found at registry: %s, latest version found: %s", requiredVersion.Required.String(), module.String(), module.Host().Display(), latestVersion.String()) + return nil, fmt.Errorf("no versions matching: %s, for module: %s, found at registry: %s, latest version found: %s", requiredVersion.Required.String(), module.String(), module.Host().Display(), latestVersion.String()) } versionToDownload = latestVersion @@ -107,31 +146,5 @@ func (r *RemoteModuleInstaller) DownloadRemoteModule(requiredVersion hclConfigs. versionToDownload = latestMatch } - // get the source location for the matched version - sourceLocation, err := regClient.ModuleLocation(module, versionToDownload.String()) - if err != nil { - zap.S().Errorf("error while getting the source location for module: %s, at registry: %s", module.String(), module.Host().Display()) - return "", err - } - - downloadLocation, err := r.DownloadModule(sourceLocation, destPath) - if err != nil { - zap.S().Errorf("error while downloading module: %s, with source location: %s", module.String(), sourceLocation) - return "", nil - } - - if module.RawSubmodule != "" { - // Append the user's requested subdirectory to any subdirectory that - // was implied by any of the nested layers we expanded within go-getter. - downloadLocation = filepath.Join(downloadLocation, module.RawSubmodule) - } - - return downloadLocation, nil -} - -func getGreaterVersion(latestVersion *version.Version, currentVersion *version.Version) *version.Version { - if latestVersion == nil || currentVersion.GreaterThan(latestVersion) { - latestVersion = currentVersion - } - return latestVersion + return versionToDownload, nil } diff --git a/pkg/iac-providers/terraform/commons/remote-module-download_test.go b/pkg/iac-providers/terraform/commons/remote-module-download_test.go index 00642473e..12b8144c6 100644 --- a/pkg/iac-providers/terraform/commons/remote-module-download_test.go +++ b/pkg/iac-providers/terraform/commons/remote-module-download_test.go @@ -21,6 +21,7 @@ import ( "log" "os" "path/filepath" + "reflect" "testing" "github.com/accurics/terrascan/pkg/downloader" @@ -28,6 +29,7 @@ import ( version "github.com/hashicorp/go-version" hclConfigs "github.com/hashicorp/terraform/configs" "github.com/hashicorp/terraform/registry/regsrc" + "github.com/hashicorp/terraform/registry/response" ) func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { @@ -40,6 +42,8 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { testConstraintsSecurityGroup, _ := version.NewConstraint("3.17.0") testModuleSecurityGroup, _ := regsrc.ParseModuleSource("terraform-aws-modules/security-group/aws") testModuleInvalidProvider, _ := regsrc.ParseModuleSource("terraform-aws-modules/testgroup/testprovider") + testModuleRdsWithRawSubModule, _ := regsrc.ParseModuleSource("terraform-aws-modules/security-group/aws//db_subnet_group") + testRawSubModuleName := "db_subnet_group" type fields struct { cache InstalledCache @@ -50,11 +54,13 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { module *regsrc.Module } tests := []struct { - name string - fields fields - args args - want string - wantErr bool + name string + fields fields + args args + want string + wantErr bool + hasRawSubModule bool + rawSubModule string }{ { name: "remote module download with valid module and version", @@ -92,6 +98,19 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { }, wantErr: true, }, + { + name: "remote module download with raw sub module", + fields: fields{ + cache: make(map[string]string), + downloader: downloader.NewDownloader(), + }, + args: args{ + requiredVersion: hclConfigs.VersionConstraint{}, + module: testModuleRdsWithRawSubModule, + }, + hasRawSubModule: true, + rawSubModule: testRawSubModuleName, + }, } for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { @@ -105,6 +124,10 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { tt.want = "" } + if tt.hasRawSubModule { + tt.want = tt.want + string(os.PathSeparator) + tt.rawSubModule + } + defer os.RemoveAll(testDir) got, err := r.DownloadRemoteModule(tt.args.requiredVersion, testDir, tt.args.module) if (err != nil) != tt.wantErr { @@ -117,3 +140,118 @@ func TestRemoteModuleInstallerDownloadRemoteModule(t *testing.T) { }) } } + +func TestGetVersionToDownload(t *testing.T) { + source := "terraform-aws-modules/security-group/aws" + testModule, _ := regsrc.ParseModuleSource(source) + testVersionConstraint, _ := version.NewConstraint("3.17.0") + testVersionConstraintMatching, _ := version.NewConstraint("1.2.0") + testVersion, _ := version.NewVersion("1.2.0") + testVersions := []*response.ModuleVersion{ + { + Version: "1.0.0", + }, + { + Version: "1.1.0", + }, + { + Version: "1.2.0", + }, + } + + type args struct { + moduleVersions *response.ModuleVersions + requiredVersion hclConfigs.VersionConstraint + module *regsrc.Module + } + tests := []struct { + name string + args args + want *version.Version + wantErr bool + }{ + { + name: "invalid version", + args: args{ + moduleVersions: &response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{ + { + Source: "", + Versions: []*response.ModuleVersion{ + {}, + }, + }, + }, + }, + module: testModule, + requiredVersion: hclConfigs.VersionConstraint{ + Required: testVersionConstraint, + }, + }, + wantErr: true, + }, + { + name: "no matching versions", + args: args{ + moduleVersions: &response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{ + { + Source: "source", + Versions: testVersions, + }, + }, + }, + module: testModule, + requiredVersion: hclConfigs.VersionConstraint{ + Required: testVersionConstraint, + }, + }, + wantErr: true, + }, + { + name: "no required version specified", + args: args{ + moduleVersions: &response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{ + { + Source: "source", + Versions: testVersions, + }, + }, + }, + module: testModule, + }, + want: testVersion, + }, + { + name: "required version specified", + args: args{ + moduleVersions: &response.ModuleVersions{ + Modules: []*response.ModuleProviderVersions{ + { + Source: "source", + Versions: testVersions, + }, + }, + }, + module: testModule, + requiredVersion: hclConfigs.VersionConstraint{ + Required: testVersionConstraintMatching, + }, + }, + want: testVersion, + }, + } + for _, tt := range tests { + t.Run(tt.name, func(t *testing.T) { + got, err := getVersionToDownload(tt.args.moduleVersions, tt.args.requiredVersion, tt.args.module) + if (err != nil) != tt.wantErr { + t.Errorf("getVersionToDownload() error = %v, wantErr %v", err, tt.wantErr) + return + } + if !reflect.DeepEqual(got, tt.want) { + t.Errorf("getVersionToDownload() = %v, want %v", got, tt.want) + } + }) + } +} From 3bc7f14b234a88f79e23db3de7c4c9aca649e730 Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Wed, 20 Jan 2021 13:44:07 +0530 Subject: [PATCH 13/15] 1. initial changes for registry module support 2. fix issue of remote module containing local modules --- pkg/cli/register.go | 7 +++++++ 1 file changed, 7 insertions(+) diff --git a/pkg/cli/register.go b/pkg/cli/register.go index 98da7e4b9..c4dd9c953 100644 --- a/pkg/cli/register.go +++ b/pkg/cli/register.go @@ -18,6 +18,8 @@ package cli import ( "flag" + "io/ioutil" + "log" "os" "github.com/accurics/terrascan/pkg/config" @@ -59,6 +61,11 @@ func Execute() { } } + // disable terraform logs when TF_LOG env variable is not set + if os.Getenv("TF_LOG") == "" { + log.SetOutput(ioutil.Discard) + } + if err := rootCmd.Execute(); err != nil { os.Exit(1) } From 7f02c6bd7049aa253d313f79844cb6e7774ae122 Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Wed, 20 Jan 2021 16:59:08 +0530 Subject: [PATCH 14/15] tests for remote module --- pkg/cli/register.go | 7 ------- 1 file changed, 7 deletions(-) diff --git a/pkg/cli/register.go b/pkg/cli/register.go index c4dd9c953..98da7e4b9 100644 --- a/pkg/cli/register.go +++ b/pkg/cli/register.go @@ -18,8 +18,6 @@ package cli import ( "flag" - "io/ioutil" - "log" "os" "github.com/accurics/terrascan/pkg/config" @@ -61,11 +59,6 @@ func Execute() { } } - // disable terraform logs when TF_LOG env variable is not set - if os.Getenv("TF_LOG") == "" { - log.SetOutput(ioutil.Discard) - } - if err := rootCmd.Execute(); err != nil { os.Exit(1) } From 4ccc1d038cdcfe12b6d8774403b4e3f44de6445b Mon Sep 17 00:00:00 2001 From: Pankaj Patil Date: Fri, 22 Jan 2021 20:25:49 +0530 Subject: [PATCH 15/15] go mod tidy --- go.sum | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/go.sum b/go.sum index f1d2e6441..d5a69398e 100644 --- a/go.sum +++ b/go.sum @@ -1497,13 +1497,9 @@ sigs.k8s.io/apiserver-network-proxy/konnectivity-client v0.0.9/go.mod h1:dzAXnQb sigs.k8s.io/kustomize v2.0.3+incompatible h1:JUufWFNlI44MdtnjUqVnvh29rR37PQFzPbLXqhyOyX0= sigs.k8s.io/kustomize v2.0.3+incompatible/go.mod h1:MkjgH3RdOWrievjo6c9T245dYlB5QeXV4WCbnt/PEpU= sigs.k8s.io/kustomize/api v0.7.2 h1:ItTD/2XaKO8CosOMFZdaGFdUGTCHdQriW7zQ7AR98rs= -sigs.k8s.io/kustomize/api v0.7.2/go.mod h1:50/vLATrjhRmMr3spZsI1GcpoZJ8IARy9QstPbA9lGE= -sigs.k8s.io/kustomize/api v0.7.1 h1:/cjDi4Pk/hqRSeCCj/Xum66rYrEtc7osM2/O+lvYKkM= -sigs.k8s.io/kustomize/api v0.7.1/go.mod h1:XOt24UrCkv0x63eT5JVaph4Kqf5EVU2UBAXo6SPBaAY= sigs.k8s.io/kustomize/api v0.7.2 h1:ItTD/2XaKO8CosOMFZdaGFdUGTCHdQriW7zQ7AR98rs= sigs.k8s.io/kustomize/api v0.7.2/go.mod h1:50/vLATrjhRmMr3spZsI1GcpoZJ8IARy9QstPbA9lGE= -sigs.k8s.io/kustomize/kyaml v0.10.5 h1:PbJcsZsEM7O3hHtUWTR+4WkHVbQRW9crSy75or1gRbI= -sigs.k8s.io/kustomize/kyaml v0.10.5/go.mod h1:P6Oy/ah/GZMKzJMIJA2a3/bc8YrBkuL5kJji13PSIzY= +sigs.k8s.io/kustomize/api v0.7.2/go.mod h1:50/vLATrjhRmMr3spZsI1GcpoZJ8IARy9QstPbA9lGE= sigs.k8s.io/kustomize/kyaml v0.10.6 h1:xUJxc/k8JoWqHUahaB8DTqY0KwEPxTbTGStvW8TOcDc= sigs.k8s.io/kustomize/kyaml v0.10.6/go.mod h1:K9yg1k/HB/6xNOf5VH3LhTo1DK9/5ykSZO5uIv+Y/1k= sigs.k8s.io/structured-merge-diff v0.0.0-20190525122527-15d366b2352e h1:4Z09Hglb792X0kfOBBJUPFEyvVfQWrYT/l8h5EKA6JQ=