From 6f6ccdfdef96acb88ce3f252d51f3ffdf507c427 Mon Sep 17 00:00:00 2001 From: Teppei Fukuda Date: Wed, 15 Jan 2020 15:00:46 +0200 Subject: [PATCH] feat: add -compare option (#10) * feat: add -compare option * test: fix arguments --- README.md | 48 +++++++++++++++++++++++++++++++++++++++++++++++- config.go | 2 ++ main.go | 45 +++++++++++++++++++++++++++++++++++++++------ main_test.go | 27 ++++++++++++++++++++++----- 4 files changed, 110 insertions(+), 12 deletions(-) diff --git a/README.md b/README.md index abae2c6..3151bee 100644 --- a/README.md +++ b/README.md @@ -29,6 +29,7 @@ - [Show only benchmarks with worse score](#show-only-benchmarks-with-worse-score) - [Specify a threshold](#specify-a-threshold) - [Specify a base commit compared with HEAD](#specify-a-base-commit-compared-with-head) + - [Compare only memory allocation](#compare-only-memory-allocation) - [Usage](#usage) - [Q&A](#qa) - [A result of benchmarks is unstable](#a-result-of-benchmarks-is-unstable) @@ -172,7 +173,6 @@ Comparison +-----------------------------+---------+-------------------+ 2020/01/12 17:48:39 This commit makes benchmarks worse - ``` @@ -192,6 +192,51 @@ By default, `cob` uses `HEAD~1`. If you compare benchmarks with different commit $ cob --base origin/master ./... ``` +## Compare only memory allocation +You can use `-compare` option. + +``` +$ cob -compare B/op +``` + +
+Result + +``` +2020/01/15 14:46:31 Run Benchmark: 4363944cbed3da7a8245cbcdc8d8240b8976eb24 HEAD~1 +2020/01/15 14:46:33 Run Benchmark: 599a5523729d4d99a331b9d3f71dde9e1e6daef0 HEAD + +Result +====== + ++-----------------------------+----------+---------------+-------------------+ +| Name | Commit | NsPerOp | AllocedBytesPerOp | ++-----------------------------+----------+---------------+-------------------+ +| BenchmarkAppend_Allocate-16 | HEAD | 179.00 ns/op | 121 B/op | ++ +----------+---------------+-------------------+ +| | HEAD@{1} | 104.00 ns/op | 23 B/op | ++-----------------------------+----------+---------------+-------------------+ +| BenchmarkCall-16 | HEAD | 0.50 ns/op | 0 B/op | ++ +----------+---------------+ + +| | HEAD@{1} | 0.49 ns/op | | ++-----------------------------+----------+---------------+-------------------+ + +Comparison +========== + ++-----------------------------+---------+-------------------+ +| Name | NsPerOp | AllocedBytesPerOp | ++-----------------------------+---------+-------------------+ +| BenchmarkAppend_Allocate-16 | - | 426.09% | ++-----------------------------+---------+-------------------+ +| BenchmarkCall-16 | - | 0.00% | ++-----------------------------+---------+-------------------+ + +2020/01/15 14:46:35 This commit makes benchmarks worse +``` + +
+ # Usage ``` @@ -208,6 +253,7 @@ GLOBAL OPTIONS: --only-degression Show only benchmarks with worse score (default: false) --threshold value The program fails if the benchmark gets worse than the threshold (default: 0.2) --base value Specify a base commit compared with HEAD (default: "HEAD~1") + --compare value Which score to compare (default: "ns/op,B/op") --bench-cmd value Specify a command to measure benchmarks (default: "go") --bench-args value Specify arguments passed to -cmd (default: "test -run '^$' -bench . -benchmem ./...") --help, -h show help (default: false) diff --git a/config.go b/config.go index 5dccb78..b8e4769 100644 --- a/config.go +++ b/config.go @@ -10,6 +10,7 @@ type config struct { onlyDegression bool threshold float64 base string + compare []string benchCmd string benchArgs []string } @@ -19,6 +20,7 @@ func newConfig(c *cli.Context) config { onlyDegression: c.Bool("only-degression"), threshold: c.Float64("threshold"), base: c.String("base"), + compare: strings.Split(c.String("compare"), ","), benchCmd: c.String("bench-cmd"), benchArgs: strings.Fields(c.String("bench-args")), } diff --git a/main.go b/main.go index f9a1223..1f8e839 100644 --- a/main.go +++ b/main.go @@ -25,6 +25,11 @@ type result struct { RatioAllocedBytesPerOp float64 } +type comparedScore struct { + nsPerOp bool + allocedBytesPerOp bool +} + func main() { app := &cli.App{ Name: "cob", @@ -47,6 +52,11 @@ func main() { Usage: "Specify a base commit compared with HEAD", Value: "HEAD~1", }, + &cli.StringFlag{ + Name: "compare", + Usage: "Which score to compare", + Value: "ns/op,B/op", + }, &cli.StringFlag{ Name: "bench-cmd", Usage: "Specify a command to measure benchmarks", @@ -159,7 +169,7 @@ func run(c config) error { showResult(os.Stdout, rows) } - degression := showRatio(os.Stdout, ratios, c.threshold, c.onlyDegression) + degression := showRatio(os.Stdout, ratios, c.threshold, whichScoreToCompare(c.compare), c.onlyDegression) if degression { return xerrors.New("This commit makes benchmarks worse") } @@ -201,7 +211,7 @@ func showResult(w io.Writer, rows [][]string) { table.Render() } -func showRatio(w io.Writer, results []result, threshold float64, onlyDegression bool) bool { +func showRatio(w io.Writer, results []result, threshold float64, comparedScore comparedScore, onlyDegression bool) bool { table := tablewriter.NewWriter(w) table.SetAutoFormatHeaders(false) table.SetAlignment(tablewriter.ALIGN_CENTER) @@ -211,7 +221,9 @@ func showRatio(w io.Writer, results []result, threshold float64, onlyDegression var degression bool for _, result := range results { - if threshold < result.RatioNsPerOp || threshold < result.RatioAllocedBytesPerOp { + if comparedScore.nsPerOp && threshold < result.RatioNsPerOp { + degression = true + } else if comparedScore.allocedBytesPerOp && threshold < result.RatioAllocedBytesPerOp { degression = true } else { if onlyDegression { @@ -219,9 +231,15 @@ func showRatio(w io.Writer, results []result, threshold float64, onlyDegression } } row := []string{result.Name, generateRatioItem(result.RatioNsPerOp), generateRatioItem(result.RatioAllocedBytesPerOp)} - colors := []tablewriter.Colors{{}} - colors = append(colors, generateColor(result.RatioNsPerOp)) - colors = append(colors, generateColor(result.RatioAllocedBytesPerOp)) + colors := []tablewriter.Colors{{}, generateColor(result.RatioNsPerOp), generateColor(result.RatioAllocedBytesPerOp)} + if !comparedScore.nsPerOp { + row[1] = "-" + colors[1] = tablewriter.Colors{} + } + if !comparedScore.allocedBytesPerOp { + row[2] = "-" + colors[2] = tablewriter.Colors{} + } table.Rich(row, colors) } if table.NumLines() > 0 { @@ -250,3 +268,18 @@ func generateColor(ratio float64) tablewriter.Colors { } return tablewriter.Colors{tablewriter.Bold, tablewriter.FgBlueColor} } + +func whichScoreToCompare(c []string) comparedScore { + var comparedScore comparedScore + for _, cc := range c { + switch cc { + case "ns/op": + fmt.Println("cpu") + comparedScore.nsPerOp = true + case "B/op": + fmt.Println("memory") + comparedScore.allocedBytesPerOp = true + } + } + return comparedScore +} diff --git a/main_test.go b/main_test.go index b34662c..46b6d53 100644 --- a/main_test.go +++ b/main_test.go @@ -61,6 +61,7 @@ func Test_showRatio(t *testing.T) { type args struct { results []result threshold float64 + compare comparedScore onlyDegression bool } tests := []struct { @@ -79,7 +80,11 @@ func Test_showRatio(t *testing.T) { RatioAllocedBytesPerOp: 0.5, }, }, - threshold: 0.2, + threshold: 0.2, + compare: comparedScore{ + nsPerOp: true, + allocedBytesPerOp: true, + }, onlyDegression: false, }, want: true, @@ -110,7 +115,11 @@ Comparison RatioAllocedBytesPerOp: 0.5, }, }, - threshold: 0.95, + threshold: 0.95, + compare: comparedScore{ + nsPerOp: true, + allocedBytesPerOp: true, + }, onlyDegression: false, }, want: false, @@ -143,7 +152,11 @@ Comparison RatioAllocedBytesPerOp: 0.5, }, }, - threshold: 0.4, + threshold: 0.4, + compare: comparedScore{ + nsPerOp: true, + allocedBytesPerOp: true, + }, onlyDegression: true, }, want: true, @@ -174,7 +187,11 @@ Comparison RatioAllocedBytesPerOp: 0.5, }, }, - threshold: 0.6, + threshold: 0.6, + compare: comparedScore{ + nsPerOp: true, + allocedBytesPerOp: true, + }, onlyDegression: true, }, want: false, @@ -184,7 +201,7 @@ Comparison for _, tt := range tests { t.Run(tt.name, func(t *testing.T) { w := &bytes.Buffer{} - got := showRatio(w, tt.args.results, tt.args.threshold, tt.args.onlyDegression) + got := showRatio(w, tt.args.results, tt.args.threshold, tt.args.compare, tt.args.onlyDegression) gotTable := w.String() assert.Equal(t, tt.wantTable, gotTable, tt.name) assert.Equal(t, tt.want, got, tt.name)