Merge pull request #6614 from Checkmarx/experimental-features

feat(engine): experimental features queries scan

Dockerfile

@@ -67,6 +67,7 @@ RUN wget https://github.com/GoogleCloudPlatform/terraformer/releases/download/0.
 COPY --from=build_env /app/bin/kics /app/bin/kics
 COPY --from=build_env /app/assets/queries /app/bin/assets/queries
 COPY --from=build_env /app/assets/libraries/* /app/bin/assets/libraries/
+COPY --from=build_env /app/assets/utils/* /app/bin/assets/utils/
 
 WORKDIR /app/bin
 

assets/utils/experimental-queries.json

@@ -0,0 +1,2 @@
+[
+]

docs/commands.md

@@ -67,6 +67,10 @@ Flags:
                                       can be provided multiple times or as a comma separated string
                                       example: 'info,low'
                                       possible values: 'high, medium, low, info, trace'
+      --experimental-queries strings include experimental queries (queries not yet thoroughly reviewed) by providing the path of the queries folder
+                                      can be provided multiple times or as a comma-separated string (platform/cloudProvider or platform)
+                                      example: 'terraform/databricks'
+                                      possible values found in: '/assets/utils/experimental-queries.json'
       --fail-on strings               which kind of results should return an exit code different from 0
                                       accepts: high, medium, low and info
                                       example: "high,low" (default [high,medium,low,info])

docs/dockerhub.md

@@ -104,6 +104,10 @@ Flags:
       --exclude-severities strings    exclude results by providing the severity of a result
                                       can be provided multiple times or as a comma separated string
                                       example: 'info,low'
+      --experimental-queries strings   include experimental queries (queries not yet thoroughly reviewed) by providing the path of the queries folder
+                                      can be provided multiple times or as a comma-separated string (platform/cloudProvider or platform)
+                                       example: 'terraform/databricks'
+                                      possible values found in: '/assets/utils/experimental-queries.json'
       --fail-on strings               which kind of results should return an exit code different from 0
                                       accepts: high, medium, low and info
                                       example: "high,low" (default [high,medium,low,info])

e2e/fixtures/assets/scan_help

@@ -2,62 +2,66 @@ Usage:
   kics scan [flags]
 
 Flags:
-  -m, --bom                           include bill of materials (BoM) in results output
-      --cloud-provider strings        list of cloud providers to scan (alicloud, aws, azure, gcp)
-      --config string                 path to configuration file
-      --disable-full-descriptions     disable request for full descriptions and use default vulnerability descriptions
-      --disable-secrets               disable secrets scanning
-      --exclude-categories strings    exclude categories by providing its name
-                                      cannot be provided with query inclusion flags
-                                      can be provided multiple times or as a comma separated string
-                                      example: 'Access control,Best practices'
-      --exclude-gitignore             disables the exclusion of paths specified within .gitignore file
-  -e, --exclude-paths strings         exclude paths from scan
-                                      supports glob and can be provided multiple times or as a quoted comma separated string
-                                      example: './shouldNotScan/*,somefile.txt'
-      --exclude-queries strings       exclude queries by providing the query ID
-                                      cannot be provided with query inclusion flags
-                                      can be provided multiple times or as a comma separated string
-                                      example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db'
-  -x, --exclude-results strings       exclude results by providing the similarity ID of a result
-                                      can be provided multiple times or as a comma separated string
-                                      example: 'fec62a97d569662093dbb9739360942f...,31263s5696620s93dbb973d9360942fc2a...'
-      --exclude-severities strings    exclude results by providing the severity of a result
-                                      can be provided multiple times or as a comma separated string
-                                      example: 'info,low'
-      --exclude-type strings          case insensitive list of platform types not to scan
-                                      (Ansible, AzureResourceManager, Buildah, CICD, CloudFormation, Crossplane, DockerCompose, Dockerfile, GRPC, GoogleDeploymentManager, Knative, Kubernetes, OpenAPI, Pulumi, ServerlessFW, Terraform)
-                                      cannot be provided with type inclusion flags
-      --fail-on strings               which kind of results should return an exit code different from 0
-                                      accepts: high, medium, low and info
-                                      example: "high,low" (default [high,medium,low,info])
-  -h, --help                          help for scan
-      --ignore-on-exit string         defines which kind of non-zero exits code should be ignored
-                                      accepts: all, results, errors, none
-                                      example: if 'results' is set, only engine errors will make KICS exit code different from 0 (default "none")
-  -i, --include-queries strings       include queries by providing the query ID
-                                      cannot be provided with query exclusion flags
-                                      can be provided multiple times or as a comma separated string
-                                      example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db'
-      --input-data string             path to query input data files
-  -b, --libraries-path string         path to directory with libraries (default "./assets/libraries")
-      --minimal-ui                    simplified version of CLI output
-      --no-progress                   hides the progress bar
-      --output-name string            name used on report creations (default "results")
-  -o, --output-path string            directory path to store reports
-  -p, --path strings                  paths or directories to scan
-                                      example: "./somepath,somefile.txt"
-      --payload-lines                 adds line information inside the payload when printing the payload file
-  -d, --payload-path string           path to store internal representation JSON file
-      --preview-lines int             number of lines to be display in CLI results (min: 1, max: 30) (default 3)
-  -q, --queries-path strings          paths to directory with queries (default [./assets/queries])
-      --report-formats strings        formats in which the results will be exported (all, asff, codeclimate, csv, cyclonedx, glsast, html, json, junit, pdf, sarif, sonarqube) (default [json])
-  -r, --secrets-regexes-path string   path to secrets regex rules configuration file
-      --terraform-vars-path string    path where terraform variables are present
-      --timeout int                   number of seconds the query has to execute before being canceled (default 60)
-  -t, --type strings                  case insensitive list of platform types to scan
-                                      (Ansible, AzureResourceManager, Buildah, CICD, CloudFormation, Crossplane, DockerCompose, Dockerfile, GRPC, GoogleDeploymentManager, Knative, Kubernetes, OpenAPI, Pulumi, ServerlessFW, Terraform)
-                                      cannot be provided with type exclusion flags
+  -m, --bom                            include bill of materials (BoM) in results output
+      --cloud-provider strings         list of cloud providers to scan (alicloud, aws, azure, gcp)
+      --config string                  path to configuration file
+      --disable-full-descriptions      disable request for full descriptions and use default vulnerability descriptions
+      --disable-secrets                disable secrets scanning
+      --exclude-categories strings     exclude categories by providing its name
+                                       cannot be provided with query inclusion flags
+                                       can be provided multiple times or as a comma separated string
+                                       example: 'Access control,Best practices'
+      --exclude-gitignore              disables the exclusion of paths specified within .gitignore file
+  -e, --exclude-paths strings          exclude paths from scan
+                                       supports glob and can be provided multiple times or as a quoted comma separated string
+                                       example: './shouldNotScan/*,somefile.txt'
+      --exclude-queries strings        exclude queries by providing the query ID
+                                       cannot be provided with query inclusion flags
+                                       can be provided multiple times or as a comma separated string
+                                       example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db'
+  -x, --exclude-results strings        exclude results by providing the similarity ID of a result
+                                       can be provided multiple times or as a comma separated string
+                                       example: 'fec62a97d569662093dbb9739360942f...,31263s5696620s93dbb973d9360942fc2a...'
+      --exclude-severities strings     exclude results by providing the severity of a result
+                                       can be provided multiple times or as a comma separated string
+                                       example: 'info,low'
+      --exclude-type strings           case insensitive list of platform types not to scan
+                                       (Ansible, AzureResourceManager, Buildah, CICD, CloudFormation, Crossplane, DockerCompose, Dockerfile, GRPC, GoogleDeploymentManager, Knative, Kubernetes, OpenAPI, Pulumi, ServerlessFW, Terraform)
+                                       cannot be provided with type inclusion flags
+      --experimental-queries strings   include experimental queries (queries not yet thoroughly reviewed) by providing the path of the queries folder
+                                       can be provided multiple times or as a comma-separated string (platform/cloudProvider or platform)
+                                       example: 'terraform/databricks'
+                                       possible values found in: '/assets/utils/experimental-queries.json'
+      --fail-on strings                which kind of results should return an exit code different from 0
+                                       accepts: high, medium, low and info
+                                       example: "high,low" (default [high,medium,low,info])
+  -h, --help                           help for scan
+      --ignore-on-exit string          defines which kind of non-zero exits code should be ignored
+                                       accepts: all, results, errors, none
+                                       example: if 'results' is set, only engine errors will make KICS exit code different from 0 (default "none")
+  -i, --include-queries strings        include queries by providing the query ID
+                                       cannot be provided with query exclusion flags
+                                       can be provided multiple times or as a comma separated string
+                                       example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db'
+      --input-data string              path to query input data files
+  -b, --libraries-path string          path to directory with libraries (default "./assets/libraries")
+      --minimal-ui                     simplified version of CLI output
+      --no-progress                    hides the progress bar
+      --output-name string             name used on report creations (default "results")
+  -o, --output-path string             directory path to store reports
+  -p, --path strings                   paths or directories to scan
+                                       example: "./somepath,somefile.txt"
+      --payload-lines                  adds line information inside the payload when printing the payload file
+  -d, --payload-path string            path to store internal representation JSON file
+      --preview-lines int              number of lines to be display in CLI results (min: 1, max: 30) (default 3)
+  -q, --queries-path strings           paths to directory with queries (default [./assets/queries])
+      --report-formats strings         formats in which the results will be exported (all, asff, codeclimate, csv, cyclonedx, glsast, html, json, junit, pdf, sarif, sonarqube) (default [json])
+  -r, --secrets-regexes-path string    path to secrets regex rules configuration file
+      --terraform-vars-path string     path where terraform variables are present
+      --timeout int                    number of seconds the query has to execute before being canceled (default 60)
+  -t, --type strings                   case insensitive list of platform types to scan
+                                       (Ansible, AzureResourceManager, Buildah, CICD, CloudFormation, Crossplane, DockerCompose, Dockerfile, GRPC, GoogleDeploymentManager, Knative, Kubernetes, OpenAPI, Pulumi, ServerlessFW, Terraform)
+                                       cannot be provided with type exclusion flags
 
 Global Flags:
       --ci                  display only log messages to CLI output (mutually exclusive with silent)

internal/console/assets/scan-flags.json

@@ -59,6 +59,13 @@
     "defaultValue": "false",
     "usage": "include bill of materials (BoM) in results output"
   },
+  "experimental-queries": {
+    "flagType": "multiStr",
+    "shorthandFlag": "",
+    "defaultValue": null,
+    "usage": "include experimental queries (queries not yet thoroughly reviewed) by providing the path of the queries folder\ncan be provided multiple times or as a comma-separated string (platform/cloudProvider or platform)\nexample: 'terraform/databricks'\npossible values found in: '/assets/utils/experimental-queries.json'",
+    "validation": "validateMultiStr"
+  },
   "fail-on": {
     "flagType": "multiStr",
     "shorthandFlag": "",

internal/console/flags/scan_flags.go

@@ -2,35 +2,36 @@ package flags
 
 // Flags constants for scan
 const (
-	BomFlag                = "bom"
-	CloudProviderFlag      = "cloud-provider"
-	ConfigFlag             = "config"
-	DisableFullDescFlag    = "disable-full-descriptions"
-	ExcludeCategoriesFlag  = "exclude-categories"
-	ExcludePathsFlag       = "exclude-paths"
-	ExcludeQueriesFlag     = "exclude-queries"
-	ExcludeResultsFlag     = "exclude-results"
-	ExcludeSeveritiesFlag  = "exclude-severities"
-	IncludeQueriesFlag     = "include-queries"
-	InputDataFlag          = "input-data"
-	FailOnFlag             = "fail-on"
-	IgnoreOnExitFlag       = "ignore-on-exit"
-	MinimalUIFlag          = "minimal-ui"
-	NoProgressFlag         = "no-progress"
-	OutputNameFlag         = "output-name"
-	OutputPathFlag         = "output-path"
-	PathFlag               = "path"
-	PayloadPathFlag        = "payload-path"
-	PreviewLinesFlag       = "preview-lines"
-	QueriesPath            = "queries-path"
-	LibrariesPath          = "libraries-path"
-	ReportFormatsFlag      = "report-formats"
-	TypeFlag               = "type"
-	ExcludeTypeFlag        = "exclude-type"
-	TerraformVarsPathFlag  = "terraform-vars-path"
-	QueryExecTimeoutFlag   = "timeout"
-	LineInfoPayloadFlag    = "payload-lines"
-	DisableSecretsFlag     = "disable-secrets"
-	SecretsRegexesPathFlag = "secrets-regexes-path" //nolint:gosec
-	ExcludeGitIgnore       = "exclude-gitignore"
+	BomFlag                 = "bom"
+	CloudProviderFlag       = "cloud-provider"
+	ConfigFlag              = "config"
+	DisableFullDescFlag     = "disable-full-descriptions"
+	ExcludeCategoriesFlag   = "exclude-categories"
+	ExcludePathsFlag        = "exclude-paths"
+	ExcludeQueriesFlag      = "exclude-queries"
+	ExcludeResultsFlag      = "exclude-results"
+	ExcludeSeveritiesFlag   = "exclude-severities"
+	ExperimentalQueriesFlag = "experimental-queries"
+	IncludeQueriesFlag      = "include-queries"
+	InputDataFlag           = "input-data"
+	FailOnFlag              = "fail-on"
+	IgnoreOnExitFlag        = "ignore-on-exit"
+	MinimalUIFlag           = "minimal-ui"
+	NoProgressFlag          = "no-progress"
+	OutputNameFlag          = "output-name"
+	OutputPathFlag          = "output-path"
+	PathFlag                = "path"
+	PayloadPathFlag         = "payload-path"
+	PreviewLinesFlag        = "preview-lines"
+	QueriesPath             = "queries-path"
+	LibrariesPath           = "libraries-path"
+	ReportFormatsFlag       = "report-formats"
+	TypeFlag                = "type"
+	ExcludeTypeFlag         = "exclude-type"
+	TerraformVarsPathFlag   = "terraform-vars-path"
+	QueryExecTimeoutFlag    = "timeout"
+	LineInfoPayloadFlag     = "payload-lines"
+	DisableSecretsFlag      = "disable-secrets"
+	SecretsRegexesPathFlag  = "secrets-regexes-path" //nolint:gosec
+	ExcludeGitIgnore        = "exclude-gitignore"
 )

internal/console/helpers/helpers.go

@@ -132,9 +132,31 @@ func GetExecutableDirectory() string {
 // GetDefaultQueryPath - returns the default query path
 func GetDefaultQueryPath(queriesPath string) (string, error) {
 	log.Debug().Msg("helpers.GetDefaultQueryPath()")
+	queriesDirectory, err := GetFullPath(queriesPath)
+	if err != nil {
+		return "", err
+	}
+	log.Debug().Msgf("Queries found in %s", queriesDirectory)
+	return queriesDirectory, nil
+}
+
+// GetDefaultExperimentalPath returns the default Experimental path
+func GetDefaultExperimentalPath(experimentalQueriesPath string) (string, error) {
+	log.Debug().Msg("helpers.GetDefaultExperimentalPath()")
+	experimentalQueriesFile, err := GetFullPath(experimentalQueriesPath)
+	if err != nil {
+		return "", err
+	}
+
+	log.Debug().Msgf("Experimental Queries found in %s", experimentalQueriesFile)
+	return experimentalQueriesFile, nil
+}
+
+// GetFulPath returns the full path of a partial path used for queries or experimental queries json path
+func GetFullPath(partialPath string) (string, error) {
 	executableDirPath := GetExecutableDirectory()
-	queriesDirectory := filepath.Join(executableDirPath, queriesPath)
-	if _, err := os.Stat(queriesDirectory); os.IsNotExist(err) {
+	fullPath := filepath.Join(executableDirPath, partialPath)
+	if _, err := os.Stat(fullPath); os.IsNotExist(err) {
 		currentWorkDir, err := os.Getwd()
 		if err != nil {
 			return "", err
@@ -143,14 +165,13 @@ func GetDefaultQueryPath(queriesPath string) (string, error) {
 		if idx != -1 {
 			currentWorkDir = currentWorkDir[:strings.LastIndex(currentWorkDir, "kics")] + "kics"
 		}
-		queriesDirectory = filepath.Join(currentWorkDir, queriesPath)
-		if _, err := os.Stat(queriesDirectory); os.IsNotExist(err) {
+		fullPath = filepath.Join(currentWorkDir, partialPath)
+		if _, err := os.Stat(fullPath); os.IsNotExist(err) {
 			return "", err
 		}
 	}
 
-	log.Debug().Msgf("Queries found in %s", queriesDirectory)
-	return queriesDirectory, nil
+	return fullPath, nil
 }
 
 // ListReportFormats return a slice with all supported report formats

internal/console/scan.go

@@ -116,6 +116,7 @@ func getScanParameters(changedDefaultQueryPath, changedDefaultLibrariesPath bool
 		ExcludeQueries:              flags.GetMultiStrFlag(flags.ExcludeQueriesFlag),
 		ExcludeResults:              flags.GetMultiStrFlag(flags.ExcludeResultsFlag),
 		ExcludeSeverities:           flags.GetMultiStrFlag(flags.ExcludeSeveritiesFlag),
+		ExperimentalQueries:         flags.GetMultiStrFlag(flags.ExperimentalQueriesFlag),
 		IncludeQueries:              flags.GetMultiStrFlag(flags.IncludeQueriesFlag),
 		InputData:                   flags.GetStrFlag(flags.InputDataFlag),
 		OutputName:                  flags.GetStrFlag(flags.OutputNameFlag),