feat(logs): added debug log messages

docs/bom.md

@@ -190,43 +190,43 @@ Results will be found in the [JSON](results.md) output and placed separately und
 { 
     // etc...
     "bill_of_materials": [
-            {
-                "query_name": "BOM - AWS S3 Buckets",
-                "query_id": "b5d6a2e0-8f15-4664-bd5b-68ec5c9bab83",
-                "query_url": "https://kics.io",
-                "severity": "TRACE",
-                "platform": "CloudFormation",
-                "cloud_provider": "AWS",
-                "category": "Bill Of Materials",
-                "description": "A list of S3 resources found. Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
-                "description_id": "a46851fb",
-                "files": [
-                    {
-                        "file_name": "positive2.json",
-                        "similarity_id": "a307e0f377932f42880de350fc69f83084aa8451a2e1e2a37cc97fc4eae7cf94",
-                        "line": 5,
-                        "issue_type": "BillOfMaterials",
-                        "search_key": "Resources.JenkinsArtifacts03",
-                        "search_line": 0,
-                        "search_value": "",
-                        "expected_value": "",
-                        "actual_value": "",
-                        "value": "{\"acl\":\"BucketOwnerFullControl\",\"resource_accessibility\":\"BucketOwnerFullControl\",\"resource_category\":\"Storage\",\"resource_encryption\":\"unencrypted\",\"resource_name\":\"jenkins-artifacts\",\"resource_type\":\"AWS::S3::Bucket\",\"resource_vendor\":\"AWS\"}"
-                    },
-                    {
-                        "file_name": "positive1.yaml",
-                        "similarity_id": "24a0036d2e94676f33c505c5cfd6686ef414072a14e576b08283e9a77596f7eb",
-                        "line": 4,
-                        "issue_type": "BillOfMaterials",
-                        "search_key": "Resources.MyBucket",
-                        "search_line": 0,
-                        "search_value": "",
-                        "expected_value": "",
-                        "actual_value": "",
-                        "value": "{\"acl\":\"BucketOwnerFullControl\",\"policy\":{\"Statement\":[{\"Action\":[\"s3:GetObject\"],\"Condition\":{\"StringLike\":{\"aws:Referer\":[\"http://www.example.com/*\",\"http://example.net/*\"]}},\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":[\"\",{\"playbooks\":[\"arn:aws:s3:::\",\"DOC-EXAMPLE-BUCKET\",\"/*\"]}]}],\"Version\":\"2012-10-17\"},\"resource_accessibility\":\"hasPolicy\",\"resource_category\":\"Storage\",\"resource_encryption\":\"encrypted\",\"resource_name\":\"jenkins-artifacts\",\"resource_type\":\"AWS::S3::Bucket\",\"resource_vendor\":\"AWS\"}"
-                    }
-                ]
-            }
-    ]
+		{
+			"query_name": "BOM - AWS S3 Buckets",
+			"query_id": "b5d6a2e0-8f15-4664-bd5b-68ec5c9bab83",
+			"query_url": "https://kics.io",
+			"severity": "TRACE",
+			"platform": "CloudFormation",
+			"cloud_provider": "AWS",
+			"category": "Bill Of Materials",
+			"description": "A list of S3 resources found. Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.",
+			"description_id": "a46851fb",
+			"files": [
+				{
+					"file_name": "positive2.json",
+					"similarity_id": "a307e0f377932f42880de350fc69f83084aa8451a2e1e2a37cc97fc4eae7cf94",
+					"line": 5,
+					"issue_type": "BillOfMaterials",
+					"search_key": "Resources.JenkinsArtifacts03",
+					"search_line": 0,
+					"search_value": "",
+					"expected_value": "",
+					"actual_value": "",
+					"value": "{\"acl\":\"BucketOwnerFullControl\",\"resource_accessibility\":\"unknown\",\"resource_category\":\"Storage\",\"resource_encryption\":\"unencrypted\",\"resource_name\":\"jenkins-artifacts\",\"resource_type\":\"AWS::S3::Bucket\",\"resource_vendor\":\"AWS\"}"
+				},
+				{
+					"file_name": "positive1.yaml",
+					"similarity_id": "24a0036d2e94676f33c505c5cfd6686ef414072a14e576b08283e9a77596f7eb",
+					"line": 4,
+					"issue_type": "BillOfMaterials",
+					"search_key": "Resources.MyBucket",
+					"search_line": 0,
+					"search_value": "",
+					"expected_value": "",
+					"actual_value": "",
+					"value": "{\"acl\":\"BucketOwnerFullControl\",\"policy\":{\"Statement\":[{\"Action\":[\"s3:GetObject\"],\"Condition\":{\"StringLike\":{\"aws:Referer\":[\"http://www.example.com/*\",\"http://example.net/*\"]}},\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":[\"\",{\"playbooks\":[\"arn:aws:s3:::\",\"DOC-EXAMPLE-BUCKET\",\"/*\"]}]}],\"Version\":\"2012-10-17\"},\"resource_accessibility\":\"hasPolicy\",\"resource_category\":\"Storage\",\"resource_encryption\":\"encrypted\",\"resource_name\":\"jenkins-artifacts\",\"resource_type\":\"AWS::S3::Bucket\",\"resource_vendor\":\"AWS\"}"
+				}
+			]
+		}
+	]
 }
 ```

go.mod

@@ -4,6 +4,7 @@ go 1.16
 
 require (
 	cloud.google.com/go/monitoring v1.2.0 // indirect
+	code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5
 	github.com/BurntSushi/toml v1.1.0
 	github.com/GoogleCloudPlatform/terraformer v0.8.18
 	github.com/agnivade/levenshtein v1.1.1
@@ -23,10 +24,12 @@ require (
 	github.com/hashicorp/hcl/v2 v2.11.1
 	github.com/hashicorp/terraform-json v0.13.0
 	github.com/johnfercher/maroto v0.36.1
+	github.com/mackerelio/go-osstat v0.2.2
 	github.com/mailru/easyjson v0.7.7
 	github.com/mitchellh/go-wordwrap v1.0.1 // indirect
 	github.com/moby/buildkit v0.10.1
 	github.com/open-policy-agent/opa v0.39.0
+	github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect
 	github.com/pkg/errors v0.9.1
 	github.com/rs/zerolog v1.26.1
 	github.com/sosedoff/ansible-vault-go v0.1.1

go.sum

@@ -66,6 +66,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX
 cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0=
 cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU=
 cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo=
+code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5 h1:tM5+dn2C9xZw1RzgI6WTQW1rGqdUimKB3RFbyu4h6Hc=
+code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5/go.mod h1:v4VVB6oBMz/c9fRY6vZrwr5xKRWOH5NPDjQZlPk0Gbs=
 code.gitea.io/sdk/gitea v0.12.0/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY=
 contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA=
 contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0=
@@ -1618,6 +1620,8 @@ github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z
 github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4=
 github.com/lusis/go-artifactory v0.0.0-20160115162124-7e4ce345df82/go.mod h1:y54tfGmO3NKssKveTEFFzH8C/akrSOy/iW9qEAUDV84=
 github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w=
+github.com/mackerelio/go-osstat v0.2.2 h1:7jVyXGXTkQL3+6lDVUDBY+Fpo8VQPfyOkZeXxxsXX4c=
+github.com/mackerelio/go-osstat v0.2.2/go.mod h1:G2A1f01HIHVRhMdS1qnigXxS6C8ahppy5lCwBrbRp0s=
 github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ=
 github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls=
@@ -1857,6 +1861,7 @@ github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDs
 github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48=
 github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak=
 github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY=
+github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY=
 github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk=
 github.com/open-policy-agent/opa v0.39.0 h1:nus6g0UC4+6adN5GV2W7K/gsL9QoELBlv5iBVyLVzWI=
 github.com/open-policy-agent/opa v0.39.0/go.mod h1:M+l9UHc2T3PCZ/RMPxiHNKZZhukiOWfmaSapx71TmGM=
@@ -1901,6 +1906,8 @@ github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627 h1:pSCLCl6joCFR
 github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ=
 github.com/paultyng/go-newrelic/v4 v4.10.0 h1:6R2aC3vONWnfxbW00nAYZ9YSj7nJ3IvQRw2rG5KGijY=
 github.com/paultyng/go-newrelic/v4 v4.10.0/go.mod h1:RmSnSvZnV267IBAqv2/2RACv1YVmxaf+/ujOFS9DRb8=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0=
+github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y=
 github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g=
 github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k=
 github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic=
@@ -2733,6 +2740,8 @@ golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBc
 golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0=
 golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs=
+golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg=
 golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw=
 golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=
 golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo=

internal/console/pre_scan.go

@@ -5,14 +5,17 @@ import (
 	"fmt"
 	"os"
 	"path/filepath"
+	"runtime"
 	"strings"
 
+	"code.cloudfoundry.org/bytefmt"
 	"github.com/Checkmarx/kics/internal/console/flags"
 	consoleHelpers "github.com/Checkmarx/kics/internal/console/helpers"
 	"github.com/Checkmarx/kics/internal/constants"
 	"github.com/Checkmarx/kics/internal/metrics"
 	internalPrinter "github.com/Checkmarx/kics/pkg/printer"
 	"github.com/Checkmarx/kics/pkg/progress"
+	"github.com/mackerelio/go-osstat/memory"
 	"github.com/pkg/errors"
 	"github.com/rs/zerolog/log"
 	"github.com/spf13/cobra"
@@ -136,6 +139,18 @@ func (console *console) preScan() {
 	fmt.Println(versionMsg)
 	log.Info().Msgf(strings.ReplaceAll(versionMsg, "\n", ""))
 
+	log.Info().Msgf("Operating system: %s", runtime.GOOS)
+
+	mem, err := memory.Get()
+	if err != nil {
+		log.Info().Msgf("%s\n", err)
+	} else {
+		log.Info().Msgf("Total memory: %s", bytefmt.ByteSize(mem.Total))
+	}
+
+	cpu := runtime.NumCPU()
+	log.Info().Msgf("CPU: %d", cpu)
+
 	noProgress := flags.GetBoolFlag(flags.NoProgressFlag)
 	if strings.EqualFold(flags.GetStrFlag(flags.LogLevelFlag), "debug") {
 		noProgress = true

pkg/engine/inspector.go

@@ -199,6 +199,9 @@ func (c *Inspector) Inspect(
 			continue
 		}
 
+		log.Debug().Msgf("Starting to run query %s", queryMeta.Query)
+		queryStartTime := time.Now()
+
 		query := &preparedQuery{
 			opaQuery: *queryOpa,
 			metadata: queryMeta,
@@ -227,6 +230,8 @@ func (c *Inspector) Inspect(
 			continue
 		}
 
+		log.Debug().Msgf("Finished to run query %s after %v", queryMeta.Query, time.Since(queryStartTime))
+
 		vulnerabilities = append(vulnerabilities, vuls...)
 
 		c.tracker.TrackQueryExecution(query.metadata.Aggregation)

pkg/kics/sink.go

@@ -26,6 +26,7 @@ var (
 
 func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reader) error {
 	s.Tracker.TrackFileFound()
+	log.Debug().Msgf("Starting to process file %s", filename)
 
 	c, err := getContent(rc)
 
@@ -76,6 +77,7 @@ func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reade
 		s.saveToFile(ctx, &file)
 	}
 	s.Tracker.TrackFileParse()
+	log.Debug().Msgf("Finished to process file %s", filename)
 	s.Tracker.TrackFileParseCountLines(documents.CountLines)
 
 	return errors.Wrap(err, "failed to save file content")

pkg/scan/utils.go

@@ -39,6 +39,8 @@ func (c *Client) prepareAndAnalyzePaths() (provider.ExtractedPath, error) {
 
 	allPaths := combinePaths(terraformerExPaths, regularExPaths)
 
+	log.Info().Msgf("Total files in the project: %d", getTotalFiles(allPaths.Path))
+
 	pathTypes, errAnalyze :=
 		analyzePaths(
 			allPaths.Path,
@@ -211,3 +213,23 @@ func printVersionCheck(customPrint *consolePrinter.Printer, s *model.Summary) {
 		log.Warn().Msgf(message)
 	}
 }
+
+func getTotalFiles(paths []string) int {
+	files := 0
+	for _, path := range paths {
+		if err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error {
+			if err != nil {
+				return err
+			}
+
+			if !info.IsDir() {
+				files++
+			}
+
+			return nil
+		}); err != nil {
+			log.Error().Msgf("failed to walk path %s: %s", path, err)
+		}
+	}
+	return files
+}