Merge pull request #6670 from tomk-orca/6306

feat(cli): add new flag --max-file-size to control the max file size

docs/commands.md

@@ -83,6 +83,7 @@ Flags:
                                       example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db'
       --input-data string             path to query input data files
   -b, --libraries-path string         path to directory with libraries (default "./assets/libraries")
+      --max-file-size int             max file size permitted for scanning, in MB (default 5)
       --minimal-ui                    simplified version of CLI output
       --no-progress                   hides the progress bar
       --output-name string            name used on report creations (default "results")

e2e/fixtures/E2E_CLI_076_RESULT.json

@@ -0,0 +1,26 @@
+{
+	"kics_version": "development",
+	"files_scanned": 0,
+	"lines_scanned": 0,
+	"files_parsed": 0,
+	"lines_parsed": 0,
+	"lines_ignored": 0,
+	"files_failed_to_scan": 0,
+	"queries_total": 0,
+	"queries_failed_to_execute": 0,
+	"queries_failed_to_compute_similarity_id": 0,
+	"scan_id": "console",
+	"severity_counters": {
+		"HIGH": 0,
+		"INFO": 0,
+		"LOW": 0,
+		"MEDIUM": 0,
+		"TRACE": 0
+	},
+	"total_counter": 0,
+	"total_bom_resources": 0,
+	"start": "2023-12-19T15:48:05.8014232Z",
+	"end": "2023-12-19T15:48:05.9361693Z",
+	"paths": [],
+	"queries": []
+}

e2e/fixtures/assets/scan_help

@@ -43,6 +43,7 @@ Flags:
                                       example: 'e69890e6-fce5-461d-98ad-cb98318dfc96,4728cd65-a20c-49da-8b31-9c08b423e4db'
       --input-data string             path to query input data files
   -b, --libraries-path string         path to directory with libraries (default "./assets/libraries")
+      --max-file-size int             max file size permitted for scanning, in MB (default 5)
       --minimal-ui                    simplified version of CLI output
       --no-progress                   hides the progress bar
       --output-name string            name used on report creations (default "results")

e2e/testcases/e2e-cli-076_max_file_size.go

@@ -0,0 +1,27 @@
+package testcases
+
+// E2E-CLI-076 - KICS  scan
+// should perform a scan without detecting anything since no files are scanned because of max file size
+func init() { //nolint
+	testSample := TestCase{
+		Name: "should perform a scan without detecting anything since no files are scanned because of max file size [E2E-CLI-076]",
+		Args: args{
+			Args: []cmdArgs{
+				[]string{"scan", "-o", "/path/e2e/output",
+					"--output-name", "E2E_CLI_076_RESULT",
+					"-p", "\"/path/test/fixtures/max_file_size\"",
+					"--max-file-size", "3",
+				},
+			},
+			ExpectedResult: []ResultsValidation{
+				{
+					ResultsFile:    "E2E_CLI_076_RESULT",
+					ResultsFormats: []string{"json"},
+				},
+			},
+		},
+		WantStatus: []int{00},
+	}
+
+	Tests = append(Tests, testSample)
+}

internal/console/analyze.go

@@ -65,8 +65,9 @@ func analyze() error {
 
 func getAnalyzeParameters() *analyzer.Parameters {
 	analyzeParams := analyzer.Parameters{
-		Path:    flags.GetMultiStrFlag(flags.AnalyzePath),
-		Results: flags.GetStrFlag(flags.AnalyzeResults),
+		Path:        flags.GetMultiStrFlag(flags.AnalyzePath),
+		Results:     flags.GetStrFlag(flags.AnalyzeResults),
+		MaxFileSize: flags.GetIntFlag(flags.MaxFileSizeFlag),
 	}
 
 	return &analyzeParams
@@ -90,6 +91,7 @@ func executeAnalyze(analyzeParams *analyzer.Parameters) error {
 		Exc:               []string{""},
 		ExcludeGitIgnore:  false,
 		GitIgnoreFileName: "",
+		MaxFileSize:       analyzeParams.MaxFileSize,
 	}
 
 	analyzedPaths, err := analyzer.Analyze(analyzerStruct)

internal/console/assets/scan-flags.json

@@ -207,5 +207,11 @@
     "shorthandFlag": "",
     "defaultValue": "false",
     "usage": "resolve the file reference, on OpenAPI files"
+  },
+  "max-file-size": {
+    "flagType": "int",
+    "shorthandFlag": "",
+    "defaultValue": "5",
+    "usage": "max file size permitted for scanning, in MB"
   }
 }

internal/console/flags/scan_flags.go

@@ -35,4 +35,5 @@ const (
 	SecretsRegexesPathFlag  = "secrets-regexes-path" //nolint:gosec
 	ExcludeGitIgnore        = "exclude-gitignore"
 	OpenAPIReferencesFlag   = "enable-openapi-refs"
+	MaxFileSizeFlag         = "max-file-size"
 )

internal/console/pre_scan.go

@@ -157,6 +157,8 @@ func (console *console) preScan() {
 	cpu := consoleHelpers.GetNumCPU()
 	log.Info().Msgf("CPU: %.1f", cpu)
 
+	log.Info().Msgf("Max file size permitted for scanning: %d MB", flags.GetIntFlag(flags.MaxFileSizeFlag))
+
 	noProgress := flags.GetBoolFlag(flags.NoProgressFlag)
 	if strings.EqualFold(flags.GetStrFlag(flags.LogLevelFlag), "debug") {
 		noProgress = true

internal/console/scan.go

@@ -140,6 +140,7 @@ func getScanParameters(changedDefaultQueryPath, changedDefaultLibrariesPath bool
 		BillOfMaterials:             flags.GetBoolFlag(flags.BomFlag),
 		ExcludeGitIgnore:            flags.GetBoolFlag(flags.ExcludeGitIgnore),
 		OpenAPIResolveReferences:    flags.GetBoolFlag(flags.OpenAPIReferencesFlag),
+		MaxFileSizeFlag:             flags.GetIntFlag(flags.MaxFileSizeFlag),
 	}
 
 	return &scanParams

pkg/analyzer/analyzer.go

@@ -123,11 +123,13 @@ const (
 	dockerfile = "dockerfile"
 	crossplane = "crossplane"
 	knative    = "knative"
+	sizeMb     = 1048576
 )
 
 type Parameters struct {
-	Results string
-	Path    []string
+	Results     string
+	Path        []string
+	MaxFileSize int
 }
 
 // regexSlice is a struct to contain a slice of regex
@@ -149,6 +151,7 @@ type Analyzer struct {
 	Exc               []string
 	GitIgnoreFileName string
 	ExcludeGitIgnore  bool
+	MaxFileSize       int
 }
 
 // types is a map that contains the regex by type
@@ -301,10 +304,7 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) {
 
 			ext := utils.GetExtension(path)
 
-			if (hasGitIgnoreFile && gitIgnore.MatchesPath(path)) || isDeadSymlink(path) {
-				ignoreFiles = append(ignoreFiles, path)
-				a.Exc = append(a.Exc, path)
-			}
+			ignoreFiles = a.checkIgnore(info.Size(), hasGitIgnoreFile, gitIgnore, path, ignoreFiles)
 
 			if isConfigFile(path, defaultConfigFiles) {
 				projectConfigFiles = append(projectConfigFiles, path)
@@ -324,13 +324,7 @@ func Analyze(a *Analyzer) (model.AnalyzedPaths, error) {
 	// unwanted is the channel shared by the workers that contains the unwanted files that the parser will ignore
 	unwanted := make(chan string, len(files))
 
-	for i := range a.Types {
-		a.Types[i] = strings.ToLower(a.Types[i])
-	}
-
-	for i := range a.ExcludeTypes {
-		a.ExcludeTypes[i] = strings.ToLower(a.ExcludeTypes[i])
-	}
+	a.Types, a.ExcludeTypes = typeLower(a.Types, a.ExcludeTypes)
 
 	// Start the workers
 	for _, file := range files {
@@ -725,3 +719,31 @@ func (a *analyzerInfo) isAvailableType(typeName string) bool {
 	// no valid behavior detected
 	return false
 }
+
+func (a *Analyzer) checkIgnore(fileSize int64, hasGitIgnoreFile bool,
+	gitIgnore *ignore.GitIgnore,
+	path string, ignoreFiles []string) []string {
+	exceededFileSize := a.MaxFileSize >= 0 && float64(fileSize)/float64(sizeMb) > float64(a.MaxFileSize)
+
+	if (hasGitIgnoreFile && gitIgnore.MatchesPath(path)) || isDeadSymlink(path) || exceededFileSize {
+		ignoreFiles = append(ignoreFiles, path)
+		a.Exc = append(a.Exc, path)
+
+		if exceededFileSize {
+			log.Error().Msgf("file %s exceeds maximum file size of %d Mb", path, a.MaxFileSize)
+		}
+	}
+	return ignoreFiles
+}
+
+func typeLower(types, exclTypes []string) (typesRes, exclTypesRes []string) {
+	for i := range types {
+		types[i] = strings.ToLower(types[i])
+	}
+
+	for i := range exclTypes {
+		exclTypes[i] = strings.ToLower(exclTypes[i])
+	}
+
+	return types, exclTypes
+}