Support one or more values.yaml file as an input to helm scan (#1501)

 Support one or more values.yaml file as an input to helm scan (#1501)
Co-authored-by: Nitin More <[email protected]>

pkg/cli/run.go

@@ -120,6 +120,9 @@ type ScanOptions struct {
 
 	// logOutputDir lets us specify the directory to write scan result and log files
 	logOutputDir string
+
+	// valuesFiles is the array of helm values file
+	valuesFiles []string
 }
 
 // NewScanOptions returns a new pointer to ScanOptions
@@ -205,7 +208,7 @@ func (s *ScanOptions) Run() error {
 	// create a new runtime executor for processing IaC
 	executor, err := runtime.NewExecutor(s.iacType, s.iacVersion, s.policyType, s.iacFilePath, s.iacDirPath,
 		s.policyPath, s.scanRules, s.skipRules, s.categories, s.severity, s.nonRecursive, s.useTerraformCache,
-		s.findVulnerabilities, s.notificationWebhookURL, s.notificationWebhookToken, s.repoURL, s.repoRef,
+		s.findVulnerabilities, s.notificationWebhookURL, s.notificationWebhookToken, s.repoURL, s.repoRef, s.valuesFiles,
 	)
 	if err != nil {
 		return err

pkg/cli/scan.go

@@ -81,5 +81,6 @@ func init() {
 	scanCmd.Flags().StringVarP(&scanOptions.notificationWebhookToken, "webhook-token", "", "", "optional token used when sending authenticated requests to the notification webhook")
 	scanCmd.Flags().StringVarP(&scanOptions.repoURL, "repo-url", "", "", "URL of the repo being scanned, will be reflected in scan summary")
 	scanCmd.Flags().StringVarP(&scanOptions.repoRef, "repo-ref", "", "", "branch of the repo being scanned")
+	scanCmd.Flags().StringSliceVarP(&scanOptions.valuesFiles, "values-files", "", []string{}, "one or more values files to scan(applicable for iactype=helm) (example: --values-files=\"file1-values.yaml,file2-values.yaml\")")
 	RegisterCommand(rootCmd, scanCmd)
 }

pkg/http-server/file-scan.go

@@ -178,10 +178,10 @@ func (g *APIHandler) scanFile(w http.ResponseWriter, r *http.Request) {
 	var executor *runtime.Executor
 	if g.test {
 		executor, err = runtime.NewExecutor(iacType, iacVersion, cloudType,
-			tempFile.Name(), "", []string{"./testdata/testpolicies"}, scanRules, skipRules, categories, severity, false, false, false, notificationWebhookURL, notificationWebhookToken, "", "")
+			tempFile.Name(), "", []string{"./testdata/testpolicies"}, scanRules, skipRules, categories, severity, false, false, false, notificationWebhookURL, notificationWebhookToken, "", "", []string{})
 	} else {
 		executor, err = runtime.NewExecutor(iacType, iacVersion, cloudType,
-			tempFile.Name(), "", getPolicyPathFromConfig(), scanRules, skipRules, categories, severity, false, false, findVulnerabilities, notificationWebhookURL, notificationWebhookToken, "", "")
+			tempFile.Name(), "", getPolicyPathFromConfig(), scanRules, skipRules, categories, severity, false, false, findVulnerabilities, notificationWebhookURL, notificationWebhookToken, "", "", []string{})
 	}
 	if err != nil {
 		zap.S().Error(err)

pkg/http-server/remote-repo.go

@@ -132,7 +132,7 @@ func (s *scanRemoteRepoReq) ScanRemoteRepo(iacType, iacVersion string, cloudType
 
 	// create a new runtime executor for scanning the remote repo
 	executor, err := runtime.NewExecutor(iacType, iacVersion, cloudType,
-		"", iacDirPath, policyPath, s.ScanRules, s.SkipRules, s.Categories, s.Severity, s.NonRecursive, false, s.FindVulnerabilities, s.NotificationWebhookURL, s.NotificationWebhookToken, s.RemoteURL, s.RepoRef)
+		"", iacDirPath, policyPath, s.ScanRules, s.SkipRules, s.Categories, s.Severity, s.NonRecursive, false, s.FindVulnerabilities, s.NotificationWebhookURL, s.NotificationWebhookToken, s.RemoteURL, s.RepoRef, []string{})
 	if err != nil {
 		zap.S().Error(err)
 		return output, isAdmissionDenied, err

pkg/iac-providers/helm/v3/load-dir.go

@@ -40,6 +40,8 @@ var (
 	errBadChartVersion = fmt.Errorf("invalid chart version in Chart.yaml")
 )
 
+const valuesFiles = "valuesFiles"
+
 // LoadIacDir loads all helm charts under the specified directory
 func (h *HelmV3) LoadIacDir(absRootDir string, options map[string]interface{}) (output.AllResourceConfigs, error) {
 
@@ -67,7 +69,7 @@ func (h *HelmV3) LoadIacDir(absRootDir string, options map[string]interface{}) (
 		// load helm charts into a map of IaC documents
 		var iacDocuments []*utils.IacDocument
 		var chartMap helmChartData
-		iacDocuments, chartMap, err = h.loadChart(chartPath)
+		iacDocuments, chartMap, err = h.loadChart(chartPath, options)
 		if err != nil && err != errSkipTestDir {
 			errMsg := fmt.Sprintf("error occurred while loading chart. err: %v", err)
 			logger.Debug("error occurred while loading chart", zap.Error(err))
@@ -239,11 +241,12 @@ func (h *HelmV3) renderChart(chartPath string, chartMap helmChartData, templateD
 }
 
 // loadChart renders and loads all templates within a chart path
-func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, helmChartData, error) {
+func (h *HelmV3) loadChart(chartPath string, options map[string]interface{}) ([]*utils.IacDocument, helmChartData, error) {
 	iacDocuments := make([]*utils.IacDocument, 0)
 	chartMap := make(helmChartData)
 	logger := zap.S().With("chart path", chartPath)
 
+	chartDir := filepath.Dir(chartPath)
 	// load the chart file and values file from the specified chart path
 	chartFileBytes, err := os.ReadFile(chartPath)
 	if err != nil {
@@ -256,32 +259,63 @@ func (h *HelmV3) loadChart(chartPath string) ([]*utils.IacDocument, helmChartDat
 		return iacDocuments, chartMap, err
 	}
 
-	var fileInfo os.FileInfo
-	chartDir := filepath.Dir(chartPath)
-	valuesFile := filepath.Join(chartDir, helmValuesFilename)
-	fileInfo, err = os.Stat(valuesFile)
-	if err != nil {
-		logger.Debug("unable to stat values.yaml", zap.Error(err))
-		return iacDocuments, chartMap, err
+	valuesFilePaths := []string{}
+	// check if custom values files are given as options
+	if valuesFiles, ok := options[valuesFiles]; ok {
+		valuesFilePaths = valuesFiles.([]string)
+		logger.Debug("found user defined values.yaml files list", zap.Any("values", valuesFilePaths))
+	}
+	if len(valuesFilePaths) == 0 { // if no values-files list provided then use default values.yaml file
+		valuesFilePaths = []string{helmValuesFilename}
+		logger.Debug("defaulting to values.yaml file present in the current directory", zap.Any("values", valuesFilePaths))
 	}
 
-	logger.With("file name", fileInfo.Name())
-	var valueFileBytes []byte
-	valueFileBytes, err = os.ReadFile(valuesFile)
-	if err != nil {
-		logger.Debug("unable to read values.yaml", zap.Error(err))
-		return iacDocuments, chartMap, err
+	allValuesFiles := make([]map[interface{}]interface{}, 0)
+
+	for _, valuesFile := range valuesFilePaths {
+		valuesFilePath := filepath.Join(chartDir, valuesFile)
+		valuesMap, err := h.readFileIntoInterface(valuesFilePath)
+		if err != nil {
+			return iacDocuments, chartMap, err
+		}
+		allValuesFiles = append(allValuesFiles, valuesMap)
 	}
 
-	var valueMap map[string]interface{}
-	if err = yaml.Unmarshal(valueFileBytes, &valueMap); err != nil {
-		logger.Debug("unable to unmarshal values.yaml", zap.Error(err))
-		return iacDocuments, chartMap, err
+	if len(allValuesFiles) > 0 {
+		resultValueMap := allValuesFiles[0]
+		for i := 1; i < len(valuesFilePaths); i++ {
+			resultValueMap = utils.MergeMaps(resultValueMap, allValuesFiles[i])
+		}
+
+		outValuesBytes, err := yaml.Marshal(resultValueMap)
+		if err != nil {
+			logger.Debug("unable to marshal merged values.yaml", zap.Error(err))
+			return iacDocuments, chartMap, err
+		}
+		var valueMap map[string]interface{}
+		// UnMarshal back to map[string]interface{}
+		if err = yaml.Unmarshal(outValuesBytes, &valueMap); err != nil {
+			logger.Debug("unable to unmarshal values.yaml", zap.Error(err))
+			return iacDocuments, chartMap, err
+		}
+		iacDocuments, chartMap, err = h.getIACDocumentsWithValues(chartPath, chartMap, valueMap)
+		if err != nil {
+			logger.Warn("error rendering chart with merged values file", zap.Error(err))
+			return iacDocuments, chartMap, err
+		}
 	}
+	return iacDocuments, chartMap, nil
+}
+
+// getIACDocumentsWithValues returns iacDocument given chart path and values map
+func (h *HelmV3) getIACDocumentsWithValues(chartPath string, chartMap helmChartData, valueMap map[string]interface{}) ([]*utils.IacDocument, helmChartData, error) {
+	iacDocuments := make([]*utils.IacDocument, 0)
+	logger := zap.S().With("chart path", chartPath)
 
+	chartDir := filepath.Dir(chartPath)
 	// for each template file found, render and save an iacDocument
 	var templateFileMap map[string][]*string
-	templateFileMap, err = utils.FindFilesBySuffix(filepath.Join(chartDir, helmTemplateDir), h.getHelmTemplateExtensions())
+	templateFileMap, err := utils.FindFilesBySuffix(filepath.Join(chartDir, helmTemplateDir), h.getHelmTemplateExtensions())
 	if err != nil {
 		logger.Warn("error while calling FindFilesBySuffix", zap.Error(err))
 		return iacDocuments, chartMap, err
@@ -312,3 +346,29 @@ func (h *HelmV3) getHelmChartFilenames() []string {
 func (h *HelmV3) Name() string {
 	return "helm"
 }
+
+// readFileIntoInterface reads and converts file into interface
+func (h *HelmV3) readFileIntoInterface(valuesFile string) (map[interface{}]interface{}, error) {
+	logger := zap.S().With("readFileIntoInterface", valuesFile)
+	fileInfo, err := os.Stat(valuesFile)
+	if err != nil {
+		logger.Debug("unable to stat values.yaml", zap.Error(err))
+		return nil, err
+	}
+
+	logger.With("file name", fileInfo.Name())
+	var valueFileBytes []byte
+	valueFileBytes, err = os.ReadFile(valuesFile)
+	if err != nil {
+		logger.Debug("unable to read values.yaml", zap.Error(err))
+		return nil, err
+	}
+
+	var valueMap map[interface{}]interface{}
+	if err = yaml.Unmarshal(valueFileBytes, &valueMap); err != nil {
+		logger.Debug("unable to unmarshal values.yaml", zap.Error(err))
+		return nil, err
+	}
+
+	return valueMap, nil
+}

pkg/iac-providers/helm/v3/load-dir_test.go

@@ -30,12 +30,17 @@ import (
 	"github.com/hashicorp/go-multierror"
 	"github.com/tenable/terrascan/pkg/iac-providers/output"
 	"github.com/tenable/terrascan/pkg/utils"
+
+	. "github.com/onsi/ginkgo"
+	. "github.com/onsi/gomega"
 )
 
 var testDataDir = "testdata"
 
 func TestLoadIacDir(t *testing.T) {
 
+	RegisterFailHandler(Fail)
+
 	invalidDirErr := &os.PathError{Err: syscall.ENOENT, Op: "lstat", Path: filepath.Join(testDataDir, "bad-dir")}
 	if utils.IsWindowsPlatform() {
 		invalidDirErr = &os.PathError{Err: syscall.ENOENT, Op: "CreateFile", Path: filepath.Join(testDataDir, "bad-dir")}
@@ -76,6 +81,24 @@ func TestLoadIacDir(t *testing.T) {
 			wantErr:       multierror.Append(fmt.Errorf("no helm charts found in directory %s", filepath.Join(testDataDir, "no-helm-charts"))),
 			resourceCount: 0,
 		},
+		{
+			name:          "happy path with multiple values files merged",
+			dirPath:       filepath.Join(testDataDir, "multiple-values-file"),
+			helmv3:        HelmV3{},
+			resourceCount: 3,
+			options: map[string]interface{}{
+				"valuesFiles": []string{"values1.yaml", "values2.yaml"},
+			},
+		},
+		{
+			name:          "happy path with multiple values files merged values1 override",
+			dirPath:       filepath.Join(testDataDir, "multiple-values-file"),
+			helmv3:        HelmV3{},
+			resourceCount: 3,
+			options: map[string]interface{}{
+				"valuesFiles": []string{"values2.yaml", "values1.yaml"},
+			},
+		},
 	}
 
 	for _, tt := range table {
@@ -97,6 +120,19 @@ func TestLoadIacDir(t *testing.T) {
 			if resCount != tt.resourceCount {
 				t.Errorf("resource count (%d) does not match expected (%d)", resCount, tt.resourceCount)
 			}
+			if tt.name == "happy path (credit to madhuakula/kubernetes-goat)" {
+				deploymentValue := resources["kubernetes_deployment"][0]
+				expectedInage := deploymentValue.ContainerImages[0].Image
+				Expect(expectedInage).To(Equal("madhuakula/k8s-goat-metadata-db:latest"))
+			} else if tt.name == "happy path with multiple values files merged" {
+				deploymentValue := resources["kubernetes_deployment"][0]
+				expectedInage := deploymentValue.ContainerImages[0].Image
+				Expect(expectedInage).To(Equal("madhuakula/k8s-goat-metadata-db-sample2:latest"))
+			} else if tt.name == "happy path with multiple values files merged values1 override" {
+				deploymentValue := resources["kubernetes_deployment"][0]
+				expectedInage := deploymentValue.ContainerImages[0].Image
+				Expect(expectedInage).To(Equal("madhuakula/k8s-goat-metadata-db:latest"))
+			}
 		})
 	}
 
@@ -109,13 +145,14 @@ func TestLoadChart(t *testing.T) {
 	unreadableChartFileErr := &os.PathError{Err: syscall.EISDIR, Op: "read", Path: filepath.Join(testDataDir, "bad-chart-file")}
 	chartPathUnreadableValuesErr := &os.PathError{Err: syscall.EISDIR, Op: "read", Path: filepath.Join(testDataDir, "chart-unreadable-values", "values.yaml")}
 	chartPathBadTemplateErr := &os.PathError{Err: syscall.EISDIR, Op: "read", Path: filepath.Join(testDataDir, "chart-bad-template-file", "templates", "service.yaml")}
-
+	chartPathNoCustomValuesYAMLErr := &os.PathError{Err: syscall.ENOENT, Op: "stat", Path: filepath.Join(testDataDir, "chart-no-values", "custom-values.yaml")}
 	if utils.IsWindowsPlatform() {
 		chartPathNoValuesYAMLErr = &os.PathError{Err: syscall.ENOENT, Op: "CreateFile", Path: filepath.Join(testDataDir, "chart-no-values", "values.yaml")}
 		chartPathNoTemplateDirErr = &os.PathError{Err: syscall.ENOENT, Op: "CreateFile", Path: filepath.Join(testDataDir, "chart-no-template-dir", "templates")}
 		unreadableChartFileErr = &os.PathError{Err: syscall.Errno(6), Op: "read", Path: filepath.Join(testDataDir, "bad-chart-file")}
 		chartPathUnreadableValuesErr = &os.PathError{Err: syscall.Errno(6), Op: "read", Path: filepath.Join(testDataDir, "chart-unreadable-values", "values.yaml")}
 		chartPathBadTemplateErr = &os.PathError{Err: syscall.Errno(6), Op: "read", Path: filepath.Join(testDataDir, "chart-bad-template-file", "templates", "service.yaml")}
+		chartPathNoCustomValuesYAMLErr = &os.PathError{Err: syscall.ENOENT, Op: "CreateFile", Path: filepath.Join(testDataDir, "chart-no-values", "custom-values.yaml")}
 	}
 
 	table := []struct {
@@ -124,6 +161,7 @@ func TestLoadChart(t *testing.T) {
 		helmv3    HelmV3
 		want      output.AllResourceConfigs
 		wantErr   error
+		options   map[string]interface{}
 	}{
 		{
 			name:      "happy path (credit to madhuakula/kubernetes-goat)",
@@ -159,7 +197,7 @@ func TestLoadChart(t *testing.T) {
 			name:      "chart path with unreadable values.yaml",
 			chartPath: filepath.Join(testDataDir, "chart-bad-values", "Chart.yaml"),
 			helmv3:    HelmV3{},
-			wantErr:   &yaml.TypeError{Errors: []string{"line 1: cannot unmarshal !!str `:bad <bad` into map[string]interface {}"}},
+			wantErr:   &yaml.TypeError{Errors: []string{"line 1: cannot unmarshal !!str `:bad <bad` into map[interface {}]interface {}"}},
 		},
 		{
 			name:      "chart path no template dir",
@@ -197,11 +235,20 @@ func TestLoadChart(t *testing.T) {
 			helmv3:    HelmV3{},
 			wantErr:   fmt.Errorf("parse error at (%s:40): unexpected {{end}}", path.Join("metadata-db", filepath.Join("templates", "ingress.yaml"))),
 		},
+		{
+			name:      "chart path with no values.yaml file in custom values.yaml",
+			chartPath: filepath.Join(testDataDir, "chart-no-values", "Chart.yaml"),
+			helmv3:    HelmV3{},
+			wantErr:   chartPathNoCustomValuesYAMLErr,
+			options: map[string]interface{}{
+				"valuesFiles": []string{"custom-values.yaml"},
+			},
+		},
 	}
 
 	for _, tt := range table {
 		t.Run(tt.name, func(t *testing.T) {
-			_, _, gotErr := tt.helmv3.loadChart(tt.chartPath)
+			_, _, gotErr := tt.helmv3.loadChart(tt.chartPath, tt.options)
 			if !reflect.DeepEqual(gotErr, tt.wantErr) {
 				t.Errorf("unexpected error; gotErr: '%v', wantErr: '%v'", gotErr, tt.wantErr)
 			}

pkg/iac-providers/helm/v3/testdata/multiple-values-file/Chart.yaml

@@ -0,0 +1,5 @@
+apiVersion: v1
+appVersion: "1.0"
+description: A Helm chart for Kubernetes metadata-db
+name: metadata-db
+version: 0.1.0

pkg/iac-providers/helm/v3/testdata/multiple-values-file/templates/_helpers.tpl

@@ -0,0 +1,32 @@
+{{/* vim: set filetype=mustache: */}}
+{{/*
+Expand the name of the chart.
+*/}}
+{{- define "metadata-db.name" -}}
+{{- default .Chart.Name .Values.nameOverride | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+
+{{/*
+Create a default fully qualified app name.
+We truncate at 63 chars because some Kubernetes name fields are limited to this (by the DNS naming spec).
+If release name contains chart name it will be used as a full name.
+*/}}
+{{- define "metadata-db.fullname" -}}
+{{- if .Values.fullnameOverride -}}
+{{- .Values.fullnameOverride | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- $name := default .Chart.Name .Values.nameOverride -}}
+{{- if contains $name .Release.Name -}}
+{{- .Release.Name | trunc 63 | trimSuffix "-" -}}
+{{- else -}}
+{{- printf "%s-%s" .Release.Name $name | trunc 63 | trimSuffix "-" -}}
+{{- end -}}
+{{- end -}}
+{{- end -}}
+
+{{/*
+Create chart name and version as used by the chart label.
+*/}}
+{{- define "metadata-db.chart" -}}
+{{- printf "%s-%s" .Chart.Name .Chart.Version | replace "+" "_" | trunc 63 | trimSuffix "-" -}}
+{{- end -}}