diff --git a/docs/benchmark.md b/docs/benchmark.md deleted file mode 100644 index b1017752499..00000000000 --- a/docs/benchmark.md +++ /dev/null @@ -1,37 +0,0 @@ -## KICS Benchmark - -The values below were obtained after scanning 150 open source projects with KICS (v1.2.0) covering -the supported IaC technologies (c.f., Terraform, Ansible, Kubernetes, Docker, AWS Cloudformation). - - -| IaC Technology | Query Accuracy1 | Query Coverage2 | Scanned IaC files​ | Number of Results​ | Average Scan Time​ (s) | Average Project Size (MB) | -| :--- | :--- | :--- | :--- | :--- | :---| :---| -| Terraform​ | 99.7%​ | 46% | 1176​ | 709 | 6.6 | 33.4​ | -| Docker​ | 98.8%​​ | 68%​ | 1017​ | 5109 | 11 | 0.7 |​ -| Kubernetes​ | 99.3%​​ | 88.7%​ | 6089​ | 21753 | 7 | 90 |​ -| CloudFormation​ | 95%​ | 73%​ | 1769​ | 5343 | 10.2 | 4.8 |​ -| Ansible ​ | 100% |​ 54%​ | 3367​ | 1320 | 23.3 | 4.1 |​ - ---- - -1 Query Accuracy = TP results / results - -2 Query Coverage = Query with results / Queries - ---- - -
- -### Global Measures - -|Measure | Value | -| :--- | :--- | -| **Average Accuracy** | 98.6% | -| **Total Number of Results** | 34234 | -| **Average Query Coverage** | 66.4% | -| **Total Scanned IaC Files** | 13418 | -| **Average Scan Time (s)** | 11.2 | -| **Average Project Size (MB)** | 26.6 | - - - diff --git a/docs/configuration-file.md b/docs/configuration-file.md index f0e3d000366..50c8632429a 100644 --- a/docs/configuration-file.md +++ b/docs/configuration-file.md @@ -70,7 +70,8 @@ KICS is able to infer the format without the need of file extension. "queries-path": "path to directory with queries (default ./assets/queries) (default './assets/queries')", "report-formats": "formats in which the results will be exported (json, sarif, html)", "type": "type of queries to use in the scan", - "verbose": true + "verbose": true, + "profiling": "enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM)" } ``` @@ -96,6 +97,7 @@ queries-path: "path to directory with queries (default ./assets/queries) (defaul report-formats: "formats in which the results will be exported (json, sarif, html)" type: "type of queries to use in the scan" verbose: true +profiling: "enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM)" ``` #### TOML Format @@ -120,6 +122,7 @@ queries-path = "path to directory with queries (default ./assets/queries) (defau report-formats = "formats in which the results will be exported (json, sarif, html)" type = "type of queries to use in the scan" verbose = true +profiling = "enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM)" ``` #### HCL Format @@ -144,6 +147,7 @@ verbose = true "report-formats" = "formats in which the results will be exported (json, sarif, html)" "type" = "type of queries to use in the scan" "verbose" = true +"profiling" = "enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM)" ``` --- diff --git a/docs/performance.md b/docs/performance.md new file mode 100644 index 00000000000..f1c56b40f8e --- /dev/null +++ b/docs/performance.md @@ -0,0 +1,92 @@ +## KICS Accuracy Benchmark + +The values below were obtained after scanning 150 open source projects with KICS (1.2.x) covering +the supported IaC technologies (c.f., Terraform, Ansible, Kubernetes, Docker, AWS Cloudformation). + + +| IaC Technology | Query Accuracy1 | Query Coverage2 | Scanned IaC files​ | Number of Results​ | Average Scan Time​ (s) | Average Project Size (MB) | +| :--- | :--- | :--- | :--- | :--- | :---| :---| +| Terraform​ | 99.7%​ | 46% | 1176​ | 709 | 6.6 | 33.4​ | +| Docker​ | 98.8%​​ | 68%​ | 1017​ | 5109 | 11 | 0.7 |​ +| Kubernetes​ | 99.3%​​ | 88.7%​ | 6089​ | 21753 | 7 | 90 |​ +| CloudFormation​ | 95%​ | 73%​ | 1769​ | 5343 | 10.2 | 4.8 |​ +| Ansible ​ | 100% |​ 54%​ | 3367​ | 1320 | 23.3 | 4.1 |​ + +--- + +1 Query Accuracy = TP results / results + +2 Query Coverage = Query with results / Queries + +--- + +
+ +### Global Measures + +|Measure | Value | +| :--- | :--- | +| **Average Accuracy** | 98.6% | +| **Total Number of Results** | 34234 | +| **Average Query Coverage** | 66.4% | +| **Total Scanned IaC Files** | 13418 | +| **Average Scan Time (s)** | 11.2 | +| **Average Project Size (MB)** | 26.6 | + +--- +## KICS Profiling + +Running Kics with `--profiling` flag will log the CPU/MEM metrics used for: + +- Getting Queries +- Parsing Files +- Scanning Vulnerabilities +- Generating Reports + +Keep in mind that profiling will periodically stop KICS to retrieve the wanted metrics, meaning KICS execution time will increase substantially. + +--- + +### CPU Profiling + +Flag: `--profiling CPU` + +```text +9:43AM INF Scanning with Keeping Infrastructure as Code Secure dev +9:43AM INF Total CPU usage for get_queries: 6.56s <- +9:43AM INF Inspector initialized, number of queries=1385 +9:43AM INF Total CPU usage for get_sources: 200.00ms <- +9:43AM INF Total CPU usage for inspect: 15.43s <- +9:43AM INF Results saved to file results/results.json fileName=results.json +9:43AM INF Results saved to file results/results.sarif fileName=results.sarif +9:43AM INF Results saved to file results/results.html fileName=results.html +9:43AM INF Total CPU usage for generate_report: 290.00ms <- +9:43AM INF Files scanned: 221 +9:43AM INF Parsed files: 221 +9:43AM INF Queries loaded: 1385 +9:43AM INF Queries failed to execute: 0 +9:43AM INF Inspector stopped +9:43AM INF Scan duration: 21.1476197s +``` +--- +### MEM Profiling + +Flag: `--profiling MEM` + +```text +9:43AM INF Scanning with Keeping Infrastructure as Code Secure dev +9:43AM INF Total MEM usage for get_queries: 237.96MB <- +9:43AM INF Inspector initialized, number of queries=1385 +9:43AM INF Total MEM usage for get_sources: 280.53MB <- +9:43AM INF Total MEM usage for inspect: 335.44MB <- +9:43AM INF Results saved to file results/results.json fileName=results.json +9:43AM INF Results saved to file results/results.sarif fileName=results.sarif +9:43AM INF Results saved to file results/results.html fileName=results.html +9:43AM INF Total MEM usage for generate_report: 333.38MB <- +9:43AM INF Files scanned: 221 +9:43AM INF Parsed files: 221 +9:43AM INF Queries loaded: 1385 +9:43AM INF Queries failed to execute: 0 +9:43AM INF Inspector stopped +9:43AM INF Scan duration: 21.1476197s +``` diff --git a/docs/usage/commands.md b/docs/usage/commands.md index a9fd3d270cc..59c741fefb6 100644 --- a/docs/usage/commands.md +++ b/docs/usage/commands.md @@ -22,6 +22,7 @@ Flags: --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") --log-path string path to log files, (defaults to ${PWD}/info.log) --no-color disable CLI color output + --profiling string enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM) -s, --silent silence stdout messages (mutually exclusive with verbose and ci) -v, --verbose write logs to stdout too (mutually exclusive with silent) @@ -74,6 +75,7 @@ Global Flags: --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") --log-path string path to log files, (defaults to ${PWD}/info.log) --no-color disable CLI color output + --profiling string enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM) -s, --silent silence stdout messages (mutually exclusive with verbose and ci) -v, --verbose write logs to stdout too (mutually exclusive with silent) ``` diff --git a/e2e/fixtures/E2E_CLI_001 b/e2e/fixtures/E2E_CLI_001 index 3a630d7f83c..8f488390f16 100644 --- a/e2e/fixtures/E2E_CLI_001 +++ b/e2e/fixtures/E2E_CLI_001 @@ -17,6 +17,7 @@ Flags: --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") --log-path string path to log files, (defaults to ${PWD}/info.log) --no-color disable CLI color output + --profiling string enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM) -s, --silent silence stdout messages (mutually exclusive with verbose and ci) -v, --verbose write logs to stdout too (mutually exclusive with silent) diff --git a/e2e/fixtures/E2E_CLI_002 b/e2e/fixtures/E2E_CLI_002 index fdb4a0524bb..54c024f634c 100644 --- a/e2e/fixtures/E2E_CLI_002 +++ b/e2e/fixtures/E2E_CLI_002 @@ -41,5 +41,6 @@ Global Flags: --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") --log-path string path to log files, (defaults to ${PWD}/info.log) --no-color disable CLI color output + --profiling string enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM) -s, --silent silence stdout messages (mutually exclusive with verbose and ci) -v, --verbose write logs to stdout too (mutually exclusive with silent) diff --git a/e2e/fixtures/E2E_CLI_003 b/e2e/fixtures/E2E_CLI_003 index bd1fa4bcabf..321c559cee9 100644 --- a/e2e/fixtures/E2E_CLI_003 +++ b/e2e/fixtures/E2E_CLI_003 @@ -40,6 +40,7 @@ Global Flags: --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") --log-path string path to log files, (defaults to ${PWD}/info.log) --no-color disable CLI color output + --profiling string enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM) -s, --silent silence stdout messages (mutually exclusive with verbose and ci) -v, --verbose write logs to stdout too (mutually exclusive with silent) diff --git a/e2e/fixtures/E2E_CLI_004 b/e2e/fixtures/E2E_CLI_004 index 7dfb3aa45a8..f01daee2fdb 100644 --- a/e2e/fixtures/E2E_CLI_004 +++ b/e2e/fixtures/E2E_CLI_004 @@ -40,6 +40,7 @@ Global Flags: --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") --log-path string path to log files, (defaults to ${PWD}/info.log) --no-color disable CLI color output + --profiling string enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM) -s, --silent silence stdout messages (mutually exclusive with verbose and ci) -v, --verbose write logs to stdout too (mutually exclusive with silent) diff --git a/e2e/fixtures/E2E_CLI_016_INVALID_FLAG b/e2e/fixtures/E2E_CLI_016_INVALID_FLAG index 1b3b15af99d..658e3812437 100644 --- a/e2e/fixtures/E2E_CLI_016_INVALID_FLAG +++ b/e2e/fixtures/E2E_CLI_016_INVALID_FLAG @@ -16,6 +16,7 @@ Flags: --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") --log-path string path to log files, (defaults to ${PWD}/info.log) --no-color disable CLI color output + --profiling string enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM) -s, --silent silence stdout messages (mutually exclusive with verbose and ci) -v, --verbose write logs to stdout too (mutually exclusive with silent) diff --git a/e2e/fixtures/E2E_CLI_016_INVALID_SCAN_FLAG b/e2e/fixtures/E2E_CLI_016_INVALID_SCAN_FLAG index ced7b637081..7346b138a64 100644 --- a/e2e/fixtures/E2E_CLI_016_INVALID_SCAN_FLAG +++ b/e2e/fixtures/E2E_CLI_016_INVALID_SCAN_FLAG @@ -40,6 +40,7 @@ Global Flags: --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") --log-path string path to log files, (defaults to ${PWD}/info.log) --no-color disable CLI color output + --profiling string enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM) -s, --silent silence stdout messages (mutually exclusive with verbose and ci) -v, --verbose write logs to stdout too (mutually exclusive with silent) diff --git a/e2e/fixtures/E2E_CLI_016_INVALID_SHOTHAND b/e2e/fixtures/E2E_CLI_016_INVALID_SHOTHAND index 591970d0db0..36e161b02a7 100644 --- a/e2e/fixtures/E2E_CLI_016_INVALID_SHOTHAND +++ b/e2e/fixtures/E2E_CLI_016_INVALID_SHOTHAND @@ -16,6 +16,7 @@ Flags: --log-level string determines log level (TRACE,DEBUG,INFO,WARN,ERROR,FATAL) (default "INFO") --log-path string path to log files, (defaults to ${PWD}/info.log) --no-color disable CLI color output + --profiling string enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM) -s, --silent silence stdout messages (mutually exclusive with verbose and ci) -v, --verbose write logs to stdout too (mutually exclusive with silent) diff --git a/go.mod b/go.mod index 2ad4eff8b48..68b4ad14e5a 100644 --- a/go.mod +++ b/go.mod @@ -10,6 +10,7 @@ require ( github.com/getsentry/sentry-go v0.10.0 github.com/golang/mock v1.5.0 github.com/google/go-cmp v0.5.4 // indirect + github.com/google/pprof v0.0.0-20210413054141-7c2eacd09c8d github.com/google/uuid v1.2.0 github.com/gookit/color v1.4.2 github.com/gregjones/httpcache v0.0.0-20190611155906-901d90724c79 // indirect diff --git a/go.sum b/go.sum index f14b3fbb873..58f63e49fd1 100644 --- a/go.sum +++ b/go.sum @@ -591,6 +591,8 @@ github.com/google/pprof v0.0.0-20191218002539-d4f498aebedc/go.mod h1:ZgVRPoUq/hf github.com/google/pprof v0.0.0-20200212024743-f11f1df84d12/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200229191704-1ebb73c60ed3/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= github.com/google/pprof v0.0.0-20200430221834-fc25d7d30c6d/go.mod h1:ZgVRPoUq/hfqzAqh7sHMqb3I9Rq5C59dIz2SbBwJ4eM= +github.com/google/pprof v0.0.0-20210413054141-7c2eacd09c8d h1:X4vWSRcXmxBANxWPGUsfWv291lZUjENBew0l1R/RVRk= +github.com/google/pprof v0.0.0-20210413054141-7c2eacd09c8d/go.mod h1:kpwsk12EmLew5upagYY7GY0pfYCcupk39gWOCRROcvE= github.com/google/renameio v0.1.0/go.mod h1:KWCgfxg9yswjAJkECMjeO8J8rahYeXnNhOm40UhjYkI= github.com/google/rpmpack v0.0.0-20191226140753-aa36bfddb3a0/go.mod h1:RaTPr0KUf2K7fnZYLNDrr8rxAamWs3iNywJLtQ2AzBg= github.com/google/shlex v0.0.0-20191202100458-e7afc7fbc510/go.mod h1:pupxD2MaaD3pAXIBCelhxNneeOaAeabZDe5s4K6zSpQ= @@ -692,6 +694,7 @@ github.com/huandu/xstrings v1.3.1 h1:4jgBlKK6tLKFvO8u5pmYjG91cqytmDCDvGh7ECVFfFs github.com/huandu/xstrings v1.3.1/go.mod h1:y5/lhBue+AyNmUVz9RLU9xbLR0o4KIIExikq4ovT0aE= github.com/hudl/fargo v1.3.0/go.mod h1:y3CKSmjA+wD2gak7sUSXTAoopbhU08POFhmITJgmKTg= github.com/ianlancetaylor/demangle v0.0.0-20181102032728-5e5cf60278f6/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= +github.com/ianlancetaylor/demangle v0.0.0-20200824232613-28f6c0f3b639/go.mod h1:aSSvb/t6k1mPoxDqO4vJh6VOCGPwU4O0C2/Eqndh1Sc= github.com/imdario/mergo v0.3.5/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.8/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= github.com/imdario/mergo v0.3.9/go.mod h1:2EnlNZ0deacrJVfApfmtdGgDfMuh/nq6Ok1EcJh5FfA= diff --git a/internal/console/helpers/helpers.go b/internal/console/helpers/helpers.go index 732ed304315..899d150dc47 100644 --- a/internal/console/helpers/helpers.go +++ b/internal/console/helpers/helpers.go @@ -12,6 +12,7 @@ import ( "sync" "github.com/BurntSushi/toml" + "github.com/Checkmarx/kics/internal/metrics" "github.com/Checkmarx/kics/pkg/model" "github.com/Checkmarx/kics/pkg/report" "github.com/gookit/color" @@ -244,6 +245,7 @@ func FileAnalyzer(path string) (string, error) { // GenerateReport execute each report function to generate report func GenerateReport(path, filename string, body interface{}, formats []string) error { log.Debug().Msgf("helpers.GenerateReport()") + metrics.Metric.Start("generate_report") var err error = nil for _, format := range formats { if err = reportGenerators[format](path, filename, body); err != nil { @@ -251,6 +253,7 @@ func GenerateReport(path, filename string, body interface{}, formats []string) e break } } + metrics.Metric.Stop() return err } diff --git a/internal/console/kics.go b/internal/console/kics.go index adce027b006..e1e5ac35e7c 100644 --- a/internal/console/kics.go +++ b/internal/console/kics.go @@ -31,6 +31,7 @@ var ( logPath string noColor bool silent bool + profiling string verbose bool warning []string @@ -90,9 +91,13 @@ func initialize(rootCmd *cobra.Command) error { "", false, "display only log messages to CLI output (mutually exclusive with silent)") + rootCmd.PersistentFlags().StringVarP(&profiling, + "profiling", + "", + "", + "enables performance profiler that prints resource consumption metrics in the logs during the execution (CPU, MEM)") - err := rootCmd.PersistentFlags().MarkDeprecated(printer.LogFileFlag, "please use --log-path instead") - if err != nil { + if err := rootCmd.PersistentFlags().MarkDeprecated(printer.LogFileFlag, "please use --log-path instead"); err != nil { return err } @@ -133,7 +138,7 @@ func Execute() error { } defer sentry.Flush(timeMult * time.Second) - if err := initialize(rootCmd); err != nil { + if err = initialize(rootCmd); err != nil { sentry.CaptureException(err) log.Err(err).Msg("Failed to initialize CLI") return err diff --git a/internal/console/scan.go b/internal/console/scan.go index e870226121e..ff11d71df21 100644 --- a/internal/console/scan.go +++ b/internal/console/scan.go @@ -13,6 +13,7 @@ import ( consoleHelpers "github.com/Checkmarx/kics/internal/console/helpers" internalPrinter "github.com/Checkmarx/kics/internal/console/printer" "github.com/Checkmarx/kics/internal/constants" + "github.com/Checkmarx/kics/internal/metrics" "github.com/Checkmarx/kics/internal/storage" "github.com/Checkmarx/kics/internal/tracker" "github.com/Checkmarx/kics/pkg/engine" @@ -98,6 +99,10 @@ func NewScanCmd() *cobra.Command { if err != nil { return errors.New("initialization error - " + err.Error()) } + err = metrics.InitializeMetrics(cmd.InheritedFlags().Lookup("profiling")) + if err != nil { + return errors.New("initialization error - " + err.Error()) + } return nil }, RunE: func(cmd *cobra.Command, args []string) error { diff --git a/internal/metrics/cpu_metric.go b/internal/metrics/cpu_metric.go new file mode 100644 index 00000000000..f6763225696 --- /dev/null +++ b/internal/metrics/cpu_metric.go @@ -0,0 +1,59 @@ +package metrics + +import ( + "bytes" + "runtime/pprof" + "time" + + "github.com/rs/zerolog/log" +) + +type cpuMetric struct { + close func() + writer *bytes.Buffer + idx int + typeMap map[string]float64 +} + +var cpuMap = map[string]float64{ + "ns": float64(time.Nanosecond), + "us": float64(time.Microsecond), + "ms": float64(time.Millisecond), + "s": float64(time.Second), + "hrs": float64(time.Hour), +} + +// Start - start gathering metrics for CPU usage +func (c *cpuMetric) start() { + c.idx = 1 + c.typeMap = cpuMap + + c.writer = bytes.NewBuffer([]byte{}) + + if err := pprof.StartCPUProfile(c.writer); err != nil { + log.Error().Msgf("failed to write cpu profile") + } + c.close = func() { + pprof.StopCPUProfile() + } +} + +// Stop - stop gathering metrics for CPU usage +func (c *cpuMetric) stop() { + c.close() +} + +// getWriter returns the profile buffer +func (c *cpuMetric) getWriter() *bytes.Buffer { + return c.writer +} + +// getIndex returns the cpu sample index +func (c *cpuMetric) getIndex() int { + return c.idx +} + +// getMap returns the map used to format total value +func (c *cpuMetric) getMap() map[string]float64 { + return c.typeMap +} diff --git a/internal/metrics/mem_metric.go b/internal/metrics/mem_metric.go new file mode 100644 index 00000000000..d9047ad8437 --- /dev/null +++ b/internal/metrics/mem_metric.go @@ -0,0 +1,72 @@ +package metrics + +import ( + "bytes" + "runtime" + "runtime/pprof" + + "github.com/rs/zerolog/log" +) + +type memMetric struct { + close func() + writer *bytes.Buffer + idx int + typeMap map[string]float64 +} + +var ( + b = 1 + kb = 10 + mb = 20 + gb = 30 + tb = 40 + pb = 50 +) + +var memoryMap = map[string]float64{ + "B": float64(b), + "kB": float64(b << kb), + "MB": float64(b << mb), + "GB": float64(b << gb), + "TB": float64(b << tb), + "PB": float64(b << pb), +} + +// Start - start gathering metrics for Memory usage +func (c *memMetric) start() { + c.idx = 3 + c.typeMap = memoryMap + + old := runtime.MemProfileRate + runtime.MemProfileRate = 4096 // set default memory rate + + c.writer = bytes.NewBuffer([]byte{}) + c.close = func() { + if err := pprof.Lookup("heap").WriteTo(c.writer, 0); err != nil { + log.Error().Msgf("failed to write mem profile") + } + + runtime.MemProfileRate = old + } +} + +// Stop - stop gathering metrics for Memory usage +func (c *memMetric) stop() { + c.close() +} + +// getWriter returns the profile buffer +func (c *memMetric) getWriter() *bytes.Buffer { + return c.writer +} + +// getIndex returns the memory sample index +func (c *memMetric) getIndex() int { + return c.idx +} + +// getMap returns the map used to format total value +func (c *memMetric) getMap() map[string]float64 { + return c.typeMap +} diff --git a/internal/metrics/metrics.go b/internal/metrics/metrics.go new file mode 100644 index 00000000000..780e9bebc57 --- /dev/null +++ b/internal/metrics/metrics.go @@ -0,0 +1,134 @@ +package metrics + +import ( + "bytes" + "fmt" + "strings" + + "github.com/google/pprof/profile" + "github.com/rs/zerolog/log" + "github.com/spf13/pflag" +) + +var ( + // Metric is the global metrics object + Metric = &Metrics{ + Disable: true, + } +) + +// Start - starts gathering metrics based on the type of metrics and writes metrics to string +// Stop - stops gathering metrics for the type of metrics specified +type metricType interface { + start() + stop() + getWriter() *bytes.Buffer + getIndex() int + getMap() map[string]float64 +} + +// Metrics - structure to keep information relevant to the metrics calculation +// Disable - disables metric calculations +type Metrics struct { + metric metricType + metricsID string + location string + Disable bool +} + +// InitializeMetrics - creates a new instance of a Metrics based on the type of metrics specified +func InitializeMetrics(metric *pflag.Flag) error { + metricStr := metric.Value.String() + var err error + switch strings.ToLower(metricStr) { + case "cpu": + Metric.Disable = false + Metric.metric = &cpuMetric{} + case "mem": + Metric.metric = &memMetric{} + Metric.Disable = false + case "": + Metric.Disable = true + default: + Metric.Disable = true + err = fmt.Errorf("unknonwn metric: %s (available metrics: CPU, MEM)", metricStr) + } + + // Create temporary dir to keep pprof file + if !Metric.Disable { + Metric.metricsID = metricStr + } + + return err +} + +// Start - starts gathering metrics for the location specified +func (m *Metrics) Start(location string) { + if m.Disable { + return + } + + log.Debug().Msgf("Started %s profiling for %s", m.metricsID, location) + + m.location = location + m.metric.start() +} + +// Stop - stops gathering metrics and logs the result +func (m *Metrics) Stop() { + if m.Disable { + return + } + log.Debug().Msgf("Stopped %s profiling for %s", m.metricsID, m.location) + + m.metric.stop() + + p, err := profile.Parse(m.metric.getWriter()) + if err != nil { + log.Error().Msgf("failed to parse profile on %s: %s", m.location, err) + } + + if err := p.CheckValid(); err != nil { + log.Error().Msgf("invalid profile on %s: %s", m.location, err) + } + + total := getTotal(p, m.metric.getIndex()) + log.Info(). + Msgf("Total %s usage for %s: %s", strings.ToUpper(m.metricsID), m.location, formatTotal(total, m.metric.getMap())) +} + +// getTotal goes through the profile samples summing their values according to +// the type of profile +func getTotal(prof *profile.Profile, idx int) int64 { + var total, diffTotal int64 + for _, sample := range prof.Sample { + var v int64 + v = sample.Value[idx] + if v < 0 { + v = -v + } + total += v + if sample.DiffBaseSample() { + diffTotal += v + } + } + if diffTotal > 0 { + total = diffTotal + } + + return total +} + +// formatTotal parses total value into a human readble way +func formatTotal(b int64, typeMap map[string]float64) string { + value := float64(b) + var formatter float64 + var mesure string + for k, u := range typeMap { + if u >= formatter && (value/u) >= 1.0 { + formatter = u + mesure = k + } + } + return fmt.Sprintf("%.2f%s", value/formatter, mesure) +} diff --git a/mkdocs.yml b/mkdocs.yml index c7395304ab9..b47e8485757 100644 --- a/mkdocs.yml +++ b/mkdocs.yml @@ -18,20 +18,20 @@ nav: - Results: results.md - Architecture: architecture.md - Usage: - - KICS Commands: usage/commands.md - - Technologies: usage/technologies.md + - KICS Commands: usage/commands.md + - Technologies: usage/technologies.md - Queries: - General Info: queries.md - Creating Queries: creating-queries.md - Queries List: - - All: queries/all-queries.md - - Ansible: queries/ansible-queries.md - - CloudFormation: queries/cloudformation-queries.md - - Common: queries/common-queries.md - - Dockerfile: queries/dockerfile-queries.md - - Kubernetes: queries/kubernetes-queries.md - - Terraform: queries/terraform-queries.md - - OpenAPI: queries/openapi-queries.md + - All: queries/all-queries.md + - Ansible: queries/ansible-queries.md + - CloudFormation: queries/cloudformation-queries.md + - Common: queries/common-queries.md + - Dockerfile: queries/dockerfile-queries.md + - Kubernetes: queries/kubernetes-queries.md + - Terraform: queries/terraform-queries.md + - OpenAPI: queries/openapi-queries.md - Integrations: - Overview: integrations.md - Github Actions: integrations_ghactions.md @@ -43,7 +43,7 @@ nav: - Plans: "https://github.com/Checkmarx/kics/projects" - Issues: "https://github.com/Checkmarx/kics/issues" - Releases: releases.md - - Benchmark: benchmark.md + - Performance: performance.md - Contribution: CONTRIBUTING.md - About: about.md @@ -55,7 +55,6 @@ theme: include_sidebar: true sticky_navigation: true - extra_css: - css/custom.css diff --git a/pkg/engine/inspector.go b/pkg/engine/inspector.go index 10653ae9f15..045c0302183 100644 --- a/pkg/engine/inspector.go +++ b/pkg/engine/inspector.go @@ -9,6 +9,7 @@ import ( "time" consoleHelpers "github.com/Checkmarx/kics/internal/console/helpers" + "github.com/Checkmarx/kics/internal/metrics" "github.com/Checkmarx/kics/pkg/detector" "github.com/Checkmarx/kics/pkg/detector/docker" "github.com/Checkmarx/kics/pkg/detector/helm" @@ -106,6 +107,7 @@ func NewInspector( excludeResults map[string]bool) (*Inspector, error) { log.Debug().Msg("engine.NewInspector()") + metrics.Metric.Start("get_queries") queries, err := queriesSource.GetQueries(excludeQueries) if err != nil { return nil, errors.Wrap(err, "failed to get queries") @@ -159,6 +161,8 @@ func NewInspector( queriesNumber := sumAllAggregatedQueries(opaQueries) + metrics.Metric.Stop() + log.Info(). Msgf("Inspector initialized, number of queries=%d", queriesNumber) diff --git a/pkg/kics/service.go b/pkg/kics/service.go index 86a85dbef20..5132284c1d1 100644 --- a/pkg/kics/service.go +++ b/pkg/kics/service.go @@ -4,6 +4,7 @@ import ( "context" "io" + "github.com/Checkmarx/kics/internal/metrics" "github.com/Checkmarx/kics/pkg/engine" "github.com/Checkmarx/kics/pkg/engine/provider" "github.com/Checkmarx/kics/pkg/model" @@ -49,6 +50,7 @@ type Service struct { // StartScan executes scan over the context, using the scanID as reference func (s *Service) StartScan(ctx context.Context, scanID string, hideProgress bool) error { log.Debug().Msg("service.StartScan()") + metrics.Metric.Start("get_sources") if err := s.SourceProvider.GetSources( ctx, s.Parser.SupportedExtensions(), @@ -61,14 +63,15 @@ func (s *Service) StartScan(ctx context.Context, scanID string, hideProgress boo ); err != nil { return errors.Wrap(err, "failed to read sources") } - + metrics.Metric.Stop() + metrics.Metric.Start("inspect") vulnerabilities, err := s.Inspector.Inspect(ctx, scanID, s.files, hideProgress, s.SourceProvider.GetBasePath()) if err != nil { return errors.Wrap(err, "failed to inspect files") } err = s.Storage.SaveVulnerabilities(ctx, vulnerabilities) - + metrics.Metric.Stop() return errors.Wrap(err, "failed to save vulnerabilities") }