From 7be22a139a4f8f4fe40bcad9069181ea2e89a0cb Mon Sep 17 00:00:00 2001 From: rafaela-soares Date: Thu, 14 Apr 2022 11:58:05 +0100 Subject: [PATCH 1/7] added debug log messages --- pkg/engine/inspector.go | 5 +++++ pkg/kics/sink.go | 2 ++ 2 files changed, 7 insertions(+) diff --git a/pkg/engine/inspector.go b/pkg/engine/inspector.go index a5bfb77c2b1..4411008b5ce 100644 --- a/pkg/engine/inspector.go +++ b/pkg/engine/inspector.go @@ -199,6 +199,9 @@ func (c *Inspector) Inspect( continue } + log.Debug().Msgf("Starting to load query %s ", queryMeta.Query) + queryStartTime := time.Now() + query := &preparedQuery{ opaQuery: *queryOpa, metadata: queryMeta, @@ -227,6 +230,8 @@ func (c *Inspector) Inspect( continue } + log.Debug().Msgf("Finishing to load query %s after %v", queryMeta.Query, time.Since(queryStartTime)) + vulnerabilities = append(vulnerabilities, vuls...) c.tracker.TrackQueryExecution(query.metadata.Aggregation) diff --git a/pkg/kics/sink.go b/pkg/kics/sink.go index 13120b1931b..0edc5bfa8e2 100644 --- a/pkg/kics/sink.go +++ b/pkg/kics/sink.go @@ -26,6 +26,7 @@ var ( func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reader) error { s.Tracker.TrackFileFound() + log.Debug().Msgf("Starting to process file %s", filename) c, err := getContent(rc) @@ -76,6 +77,7 @@ func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reade s.saveToFile(ctx, &file) } s.Tracker.TrackFileParse() + log.Debug().Msgf("Finishing to process file %s", filename) s.Tracker.TrackFileParseCountLines(documents.CountLines) return errors.Wrap(err, "failed to save file content") From 8d2591f4eb7a511244f5e0c528244d03cb82f013 Mon Sep 17 00:00:00 2001 From: rafaela-soares Date: Thu, 14 Apr 2022 11:59:18 +0100 Subject: [PATCH 2/7] removing space --- pkg/engine/inspector.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/engine/inspector.go b/pkg/engine/inspector.go index 4411008b5ce..c1c6118220f 100644 --- a/pkg/engine/inspector.go +++ b/pkg/engine/inspector.go @@ -199,7 +199,7 @@ func (c *Inspector) Inspect( continue } - log.Debug().Msgf("Starting to load query %s ", queryMeta.Query) + log.Debug().Msgf("Starting to load query %s", queryMeta.Query) queryStartTime := time.Now() query := &preparedQuery{ From b830b03c5086a1f1a40650d40c5a5f676a6b7f32 Mon Sep 17 00:00:00 2001 From: rafaela-soares Date: Thu, 14 Apr 2022 12:10:36 +0100 Subject: [PATCH 3/7] requested changes --- pkg/engine/inspector.go | 2 +- pkg/kics/sink.go | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/engine/inspector.go b/pkg/engine/inspector.go index c1c6118220f..32507c1758b 100644 --- a/pkg/engine/inspector.go +++ b/pkg/engine/inspector.go @@ -230,7 +230,7 @@ func (c *Inspector) Inspect( continue } - log.Debug().Msgf("Finishing to load query %s after %v", queryMeta.Query, time.Since(queryStartTime)) + log.Debug().Msgf("Finished to load query %s after %v", queryMeta.Query, time.Since(queryStartTime)) vulnerabilities = append(vulnerabilities, vuls...) diff --git a/pkg/kics/sink.go b/pkg/kics/sink.go index 0edc5bfa8e2..f7f8a5a86ff 100644 --- a/pkg/kics/sink.go +++ b/pkg/kics/sink.go @@ -77,7 +77,7 @@ func (s *Service) sink(ctx context.Context, filename, scanID string, rc io.Reade s.saveToFile(ctx, &file) } s.Tracker.TrackFileParse() - log.Debug().Msgf("Finishing to process file %s", filename) + log.Debug().Msgf("Finished to process file %s", filename) s.Tracker.TrackFileParseCountLines(documents.CountLines) return errors.Wrap(err, "failed to save file content") From 379dccab10873148a7491b953af43650f0406469 Mon Sep 17 00:00:00 2001 From: rafaela-soares Date: Thu, 14 Apr 2022 13:00:48 +0100 Subject: [PATCH 4/7] updated doc --- docs/bom.md | 76 ++++++++++++++++++++++++++--------------------------- 1 file changed, 38 insertions(+), 38 deletions(-) diff --git a/docs/bom.md b/docs/bom.md index 70143c62bcb..78a06d196d3 100644 --- a/docs/bom.md +++ b/docs/bom.md @@ -190,43 +190,43 @@ Results will be found in the [JSON](results.md) output and placed separately und { // etc... "bill_of_materials": [ - { - "query_name": "BOM - AWS S3 Buckets", - "query_id": "b5d6a2e0-8f15-4664-bd5b-68ec5c9bab83", - "query_url": "https://kics.io", - "severity": "TRACE", - "platform": "CloudFormation", - "cloud_provider": "AWS", - "category": "Bill Of Materials", - "description": "A list of S3 resources found. Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.", - "description_id": "a46851fb", - "files": [ - { - "file_name": "positive2.json", - "similarity_id": "a307e0f377932f42880de350fc69f83084aa8451a2e1e2a37cc97fc4eae7cf94", - "line": 5, - "issue_type": "BillOfMaterials", - "search_key": "Resources.JenkinsArtifacts03", - "search_line": 0, - "search_value": "", - "expected_value": "", - "actual_value": "", - "value": "{\"acl\":\"BucketOwnerFullControl\",\"resource_accessibility\":\"BucketOwnerFullControl\",\"resource_category\":\"Storage\",\"resource_encryption\":\"unencrypted\",\"resource_name\":\"jenkins-artifacts\",\"resource_type\":\"AWS::S3::Bucket\",\"resource_vendor\":\"AWS\"}" - }, - { - "file_name": "positive1.yaml", - "similarity_id": "24a0036d2e94676f33c505c5cfd6686ef414072a14e576b08283e9a77596f7eb", - "line": 4, - "issue_type": "BillOfMaterials", - "search_key": "Resources.MyBucket", - "search_line": 0, - "search_value": "", - "expected_value": "", - "actual_value": "", - "value": "{\"acl\":\"BucketOwnerFullControl\",\"policy\":{\"Statement\":[{\"Action\":[\"s3:GetObject\"],\"Condition\":{\"StringLike\":{\"aws:Referer\":[\"http://www.example.com/*\",\"http://example.net/*\"]}},\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":[\"\",{\"playbooks\":[\"arn:aws:s3:::\",\"DOC-EXAMPLE-BUCKET\",\"/*\"]}]}],\"Version\":\"2012-10-17\"},\"resource_accessibility\":\"hasPolicy\",\"resource_category\":\"Storage\",\"resource_encryption\":\"encrypted\",\"resource_name\":\"jenkins-artifacts\",\"resource_type\":\"AWS::S3::Bucket\",\"resource_vendor\":\"AWS\"}" - } - ] - } - ] + { + "query_name": "BOM - AWS S3 Buckets", + "query_id": "b5d6a2e0-8f15-4664-bd5b-68ec5c9bab83", + "query_url": "https://kics.io", + "severity": "TRACE", + "platform": "CloudFormation", + "cloud_provider": "AWS", + "category": "Bill Of Materials", + "description": "A list of S3 resources found. Amazon Simple Storage Service (Amazon S3) is an object storage service that offers industry-leading scalability, data availability, security, and performance.", + "description_id": "a46851fb", + "files": [ + { + "file_name": "positive2.json", + "similarity_id": "a307e0f377932f42880de350fc69f83084aa8451a2e1e2a37cc97fc4eae7cf94", + "line": 5, + "issue_type": "BillOfMaterials", + "search_key": "Resources.JenkinsArtifacts03", + "search_line": 0, + "search_value": "", + "expected_value": "", + "actual_value": "", + "value": "{\"acl\":\"BucketOwnerFullControl\",\"resource_accessibility\":\"unknown\",\"resource_category\":\"Storage\",\"resource_encryption\":\"unencrypted\",\"resource_name\":\"jenkins-artifacts\",\"resource_type\":\"AWS::S3::Bucket\",\"resource_vendor\":\"AWS\"}" + }, + { + "file_name": "positive1.yaml", + "similarity_id": "24a0036d2e94676f33c505c5cfd6686ef414072a14e576b08283e9a77596f7eb", + "line": 4, + "issue_type": "BillOfMaterials", + "search_key": "Resources.MyBucket", + "search_line": 0, + "search_value": "", + "expected_value": "", + "actual_value": "", + "value": "{\"acl\":\"BucketOwnerFullControl\",\"policy\":{\"Statement\":[{\"Action\":[\"s3:GetObject\"],\"Condition\":{\"StringLike\":{\"aws:Referer\":[\"http://www.example.com/*\",\"http://example.net/*\"]}},\"Effect\":\"Allow\",\"Principal\":\"*\",\"Resource\":[\"\",{\"playbooks\":[\"arn:aws:s3:::\",\"DOC-EXAMPLE-BUCKET\",\"/*\"]}]}],\"Version\":\"2012-10-17\"},\"resource_accessibility\":\"hasPolicy\",\"resource_category\":\"Storage\",\"resource_encryption\":\"encrypted\",\"resource_name\":\"jenkins-artifacts\",\"resource_type\":\"AWS::S3::Bucket\",\"resource_vendor\":\"AWS\"}" + } + ] + } + ] } ``` From 7c8b174ebf61a258faae251d8bcf38bcadb96c2d Mon Sep 17 00:00:00 2001 From: rafaela-soares Date: Mon, 18 Apr 2022 09:31:03 +0100 Subject: [PATCH 5/7] requested changes --- pkg/engine/inspector.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/engine/inspector.go b/pkg/engine/inspector.go index 32507c1758b..be978f138e7 100644 --- a/pkg/engine/inspector.go +++ b/pkg/engine/inspector.go @@ -199,7 +199,7 @@ func (c *Inspector) Inspect( continue } - log.Debug().Msgf("Starting to load query %s", queryMeta.Query) + log.Debug().Msgf("Starting to run query %s", queryMeta.Query) queryStartTime := time.Now() query := &preparedQuery{ @@ -230,7 +230,7 @@ func (c *Inspector) Inspect( continue } - log.Debug().Msgf("Finished to load query %s after %v", queryMeta.Query, time.Since(queryStartTime)) + log.Debug().Msgf("Finished to run query %s after %v", queryMeta.Query, time.Since(queryStartTime)) vulnerabilities = append(vulnerabilities, vuls...) From fb9f6739468bf0a3605a6b45416b142296cb2698 Mon Sep 17 00:00:00 2001 From: rafaela-soares Date: Mon, 18 Apr 2022 14:13:10 +0100 Subject: [PATCH 6/7] added more log infos --- go.mod | 3 +++ go.sum | 9 +++++++++ internal/console/pre_scan.go | 15 +++++++++++++++ pkg/scan/utils.go | 22 ++++++++++++++++++++++ 4 files changed, 49 insertions(+) diff --git a/go.mod b/go.mod index 598acb6a719..411d9f0c430 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.16 require ( cloud.google.com/go/monitoring v1.2.0 // indirect + code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5 github.com/BurntSushi/toml v1.1.0 github.com/GoogleCloudPlatform/terraformer v0.8.18 github.com/agnivade/levenshtein v1.1.1 @@ -23,10 +24,12 @@ require ( github.com/hashicorp/hcl/v2 v2.11.1 github.com/hashicorp/terraform-json v0.13.0 github.com/johnfercher/maroto v0.36.1 + github.com/mackerelio/go-osstat v0.2.2 github.com/mailru/easyjson v0.7.7 github.com/mitchellh/go-wordwrap v1.0.1 // indirect github.com/moby/buildkit v0.10.1 github.com/open-policy-agent/opa v0.39.0 + github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 // indirect github.com/pkg/errors v0.9.1 github.com/rs/zerolog v1.26.1 github.com/sosedoff/ansible-vault-go v0.1.1 diff --git a/go.sum b/go.sum index efef97c4a4b..009955ea8b9 100644 --- a/go.sum +++ b/go.sum @@ -66,6 +66,8 @@ cloud.google.com/go/storage v1.8.0/go.mod h1:Wv1Oy7z6Yz3DshWRJFhqM/UCfaWIRTdp0RX cloud.google.com/go/storage v1.10.0/go.mod h1:FLPqc6j+Ki4BU591ie1oL6qBQGu2Bl/tZ9ullr3+Kg0= cloud.google.com/go/storage v1.14.0 h1:6RRlFMv1omScs6iq2hfE3IvgE+l6RfJPampq8UZc5TU= cloud.google.com/go/storage v1.14.0/go.mod h1:GrKmX003DSIwi9o29oFT7YDnHYwZoctc3fOKtUw0Xmo= +code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5 h1:tM5+dn2C9xZw1RzgI6WTQW1rGqdUimKB3RFbyu4h6Hc= +code.cloudfoundry.org/bytefmt v0.0.0-20211005130812-5bb3c17173e5/go.mod h1:v4VVB6oBMz/c9fRY6vZrwr5xKRWOH5NPDjQZlPk0Gbs= code.gitea.io/sdk/gitea v0.12.0/go.mod h1:z3uwDV/b9Ls47NGukYM9XhnHtqPh/J+t40lsUrR6JDY= contrib.go.opencensus.io/exporter/aws v0.0.0-20181029163544-2befc13012d0/go.mod h1:uu1P0UCM/6RbsMrgPa98ll8ZcHM858i/AD06a9aLRCA= contrib.go.opencensus.io/exporter/ocagent v0.5.0/go.mod h1:ImxhfLRpxoYiSq891pBrLVhN+qmP8BTVvdH2YLs7Gl0= @@ -1618,6 +1620,8 @@ github.com/lithammer/dedent v1.1.0/go.mod h1:jrXYCQtgg0nJiN+StA2KgR7w6CiQNv9Fd/Z github.com/logrusorgru/aurora v0.0.0-20181002194514-a7b3b318ed4e/go.mod h1:7rIyQOR62GCctdiQpZ/zOJlFyk6y+94wXzv6RNZgaR4= github.com/lusis/go-artifactory v0.0.0-20160115162124-7e4ce345df82/go.mod h1:y54tfGmO3NKssKveTEFFzH8C/akrSOy/iW9qEAUDV84= github.com/lyft/protoc-gen-star v0.5.3/go.mod h1:V0xaHgaf5oCCqmcxYcWiDfTiKsZsRc87/1qhoTACD8w= +github.com/mackerelio/go-osstat v0.2.2 h1:7jVyXGXTkQL3+6lDVUDBY+Fpo8VQPfyOkZeXxxsXX4c= +github.com/mackerelio/go-osstat v0.2.2/go.mod h1:G2A1f01HIHVRhMdS1qnigXxS6C8ahppy5lCwBrbRp0s= github.com/magiconair/properties v1.8.0/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.1/go.mod h1:PppfXfuXeibc/6YijjN8zIbojt8czPbwD3XqdrwzmxQ= github.com/magiconair/properties v1.8.5 h1:b6kJs+EmPFMYGkow9GiUyCyOvIwYetYJ3fSaWak/Gls= @@ -1857,6 +1861,7 @@ github.com/onsi/gomega v1.10.3/go.mod h1:V9xEwhxec5O8UDM77eCW8vLymOMltsqPVYWrpDs github.com/onsi/gomega v1.10.5/go.mod h1:gza4q3jKQJijlu05nKWRCW/GavJumGt8aNRxWg7mt48= github.com/onsi/gomega v1.13.0 h1:7lLHu94wT9Ij0o6EWWclhu0aOh32VxhkwEJvzuWPeak= github.com/onsi/gomega v1.13.0/go.mod h1:lRk9szgn8TxENtWd0Tp4c3wjlRfMTMH27I+3Je41yGY= +github.com/onsi/gomega v1.16.0/go.mod h1:HnhC7FXeEQY45zxNK3PPoIUhzk/80Xly9PcubAlGdZY= github.com/op/go-logging v0.0.0-20160315200505-970db520ece7/go.mod h1:HzydrMdWErDVzsI23lYNej1Htcns9BCg93Dk0bBINWk= github.com/open-policy-agent/opa v0.39.0 h1:nus6g0UC4+6adN5GV2W7K/gsL9QoELBlv5iBVyLVzWI= github.com/open-policy-agent/opa v0.39.0/go.mod h1:M+l9UHc2T3PCZ/RMPxiHNKZZhukiOWfmaSapx71TmGM= @@ -1901,6 +1906,8 @@ github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627 h1:pSCLCl6joCFR github.com/patrickmn/go-cache v0.0.0-20180815053127-5633e0862627/go.mod h1:3Qf8kWWT7OJRJbdiICTKqZju1ZixQ/KpMGzzAfe6+WQ= github.com/paultyng/go-newrelic/v4 v4.10.0 h1:6R2aC3vONWnfxbW00nAYZ9YSj7nJ3IvQRw2rG5KGijY= github.com/paultyng/go-newrelic/v4 v4.10.0/go.mod h1:RmSnSvZnV267IBAqv2/2RACv1YVmxaf+/ujOFS9DRb8= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58 h1:onHthvaw9LFnH4t2DcNVpwGmV9E1BkGknEliJkfwQj0= +github.com/pbnjay/memory v0.0.0-20210728143218-7b4eea64cf58/go.mod h1:DXv8WO4yhMYhSNPKjeNKa5WY9YCIEBRbNzFFPJbWO6Y= github.com/pborman/uuid v1.2.0 h1:J7Q5mO4ysT1dv8hyrUGHb9+ooztCXu1D8MY8DZYsu3g= github.com/pborman/uuid v1.2.0/go.mod h1:X/NO0urCmaxf9VXbdlT7C2Yzkj2IKimNn4k+gtPdI/k= github.com/pelletier/go-toml v1.2.0/go.mod h1:5z9KED0ma1S8pY6P1sdut58dfprrGBbd/94hg7ilaic= @@ -2733,6 +2740,8 @@ golang.org/x/sys v0.0.0-20211210111614-af8b64212486/go.mod h1:oPkhp1MJrh7nUepCBc golang.org/x/sys v0.0.0-20211216021012-1d35b9e2eb4e/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9 h1:XfKQ4OlFl8okEOr5UvAqFRVj8pY/4yfcXrddB8qAbU0= golang.org/x/sys v0.0.0-20220114195835-da31bd327af9/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8 h1:OH54vjqzRWmbJ62fjuhxy7AxFFgoHN0/DPc/UrL8cAs= +golang.org/x/sys v0.0.0-20220319134239-a9b59b0215f8/go.mod h1:oPkhp1MJrh7nUepCBck5+mAzfO9JrbApNNgaTdGDITg= golang.org/x/term v0.0.0-20201117132131-f5c789dd3221/go.mod h1:Nr5EML6q2oocZ2LXRh80K7BxOlk5/8JxuGnuhpl+muw= golang.org/x/term v0.0.0-20201126162022-7de9c90e9dd1/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= golang.org/x/term v0.0.0-20210220032956-6a3ed077a48d/go.mod h1:bj7SfCRtBDWHUb9snDiAeCFNEtKQo2Wmx5Cou7ajbmo= diff --git a/internal/console/pre_scan.go b/internal/console/pre_scan.go index 8c425e15e95..b2c49b2dce6 100644 --- a/internal/console/pre_scan.go +++ b/internal/console/pre_scan.go @@ -5,14 +5,17 @@ import ( "fmt" "os" "path/filepath" + "runtime" "strings" + "code.cloudfoundry.org/bytefmt" "github.com/Checkmarx/kics/internal/console/flags" consoleHelpers "github.com/Checkmarx/kics/internal/console/helpers" "github.com/Checkmarx/kics/internal/constants" "github.com/Checkmarx/kics/internal/metrics" internalPrinter "github.com/Checkmarx/kics/pkg/printer" "github.com/Checkmarx/kics/pkg/progress" + "github.com/mackerelio/go-osstat/memory" "github.com/pkg/errors" "github.com/rs/zerolog/log" "github.com/spf13/cobra" @@ -136,6 +139,18 @@ func (console *console) preScan() { fmt.Println(versionMsg) log.Info().Msgf(strings.ReplaceAll(versionMsg, "\n", "")) + log.Info().Msgf("Operating system: %s", runtime.GOOS) + + mem, err := memory.Get() + if err != nil { + log.Info().Msgf("%s\n", err) + } else { + log.Info().Msgf("Total memory: %s", bytefmt.ByteSize(mem.Total)) + } + + cpu := runtime.NumCPU() + log.Info().Msgf("CPU: %d", cpu) + noProgress := flags.GetBoolFlag(flags.NoProgressFlag) if strings.EqualFold(flags.GetStrFlag(flags.LogLevelFlag), "debug") { noProgress = true diff --git a/pkg/scan/utils.go b/pkg/scan/utils.go index 32a9be894ff..c95762b0a3f 100644 --- a/pkg/scan/utils.go +++ b/pkg/scan/utils.go @@ -39,6 +39,8 @@ func (c *Client) prepareAndAnalyzePaths() (provider.ExtractedPath, error) { allPaths := combinePaths(terraformerExPaths, regularExPaths) + log.Info().Msgf("Total files in the project: %d", getTotalFiles(allPaths.Path)) + pathTypes, errAnalyze := analyzePaths( allPaths.Path, @@ -211,3 +213,23 @@ func printVersionCheck(customPrint *consolePrinter.Printer, s *model.Summary) { log.Warn().Msgf(message) } } + +func getTotalFiles(paths []string) int { + files := 0 + for _, path := range paths { + if err := filepath.Walk(path, func(path string, info os.FileInfo, err error) error { + if err != nil { + return err + } + + if !info.IsDir() { + files++ + } + + return nil + }); err != nil { + log.Error().Msgf("failed to walk path %s: %s", path, err) + } + } + return files +} From 65a543d51dbc8344da042cd7e73234d72c86c52e Mon Sep 17 00:00:00 2001 From: rafaela-soares Date: Mon, 18 Apr 2022 14:23:15 +0100 Subject: [PATCH 7/7] requested changes --- internal/console/pre_scan.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/internal/console/pre_scan.go b/internal/console/pre_scan.go index b2c49b2dce6..f5c662070a2 100644 --- a/internal/console/pre_scan.go +++ b/internal/console/pre_scan.go @@ -143,7 +143,7 @@ func (console *console) preScan() { mem, err := memory.Get() if err != nil { - log.Info().Msgf("%s\n", err) + log.Err(err) } else { log.Info().Msgf("Total memory: %s", bytefmt.ByteSize(mem.Total)) }