Skip to content

Commit

Permalink
feat(gnovm): add benchmark system (#1624)
Browse files Browse the repository at this point in the history
I was attempting some changes with performance ramifications and wanted
to measure their impact. Eventually, they turned out to be bad
performance-wise, but the code I made for measuring performance is
probably still useful.

This PR sets up a basic performance measurement system for the GnoVM. It
leverages Go's existing benchmarking system together with templates. It
allows us to measure impact on performance using standard go tools like
[benchstat](https://pkg.go.dev/golang.org/x/perf/cmd/benchstat).

```console
$ go test -bench . -run NONE .
goos: linux
goarch: amd64
pkg: github.com/gnolang/gno/gnovm/pkg/gnolang
cpu: AMD Ryzen 7 PRO 4750U with Radeon Graphics
BenchmarkPreprocess-16                      7737            152430 ns/op
BenchmarkBenchdata/fib.gno_param:4-16              47374             24664 ns/op
BenchmarkBenchdata/fib.gno_param:8-16              10000            213632 ns/op
BenchmarkBenchdata/fib.gno_param:16-16               100          10590462 ns/op
BenchmarkBenchdata/loop.gno-16                   2556032               509.3 ns/op
BenchmarkBenchdata/matrix_mult.gno_param:3-16       1976            590535 ns/op
BenchmarkBenchdata/matrix_mult.gno_param:4-16        740           1703166 ns/op
BenchmarkBenchdata/matrix_mult.gno_param:5-16        180           6797221 ns/op
BenchmarkBenchdata/matrix_mult.gno_param:6-16         36          34389320 ns/op
BenchmarkCreateNewMachine-16                       80518             13819 ns/op
PASS
ok      github.com/gnolang/gno/gnovm/pkg/gnolang        16.790s
```

A few benchmarks are provided to show its functioning and benchmark some
initial functions. `loop.gno` brings over the `LoopyMain` existing
benchmark, and makes its output more useful by correlating it with
`b.N`.

Making this to have a preliminary review and open up discussions:

- I removed `go_bench_data.go`, because as the name suggests it tests
out things in Go and isn't actually at all related to the GnoVM.
- Do we want to keep these benchmarks in `misc/` or somewhere? Or has
the information gathered from these tests already been used to act, and
as such they can be removed?
- Was something similar already in the works? @deelawn
@petar-dambovaliev
- Is it OK to place the benchmarking files in `benchdata`, or would we
prefer another location?
- Would these benchmarks be automatically picked up for
gnolang/benchmarks? cc/ @ajnavarro @albttx
  • Loading branch information
thehowl authored Feb 8, 2024
1 parent 371876c commit 78bd2a9
Show file tree
Hide file tree
Showing 5 changed files with 186 additions and 448 deletions.
16 changes: 16 additions & 0 deletions gnovm/pkg/gnolang/benchdata/fib.gno
Original file line number Diff line number Diff line change
@@ -0,0 +1,16 @@
// param: 4 8 16

package main

func main() {
for i := 0; i < {{ .N }}; i++ {
fib({{ .Param }})
}
}

func fib(n int) int {
if n < 2 {
return 1
}
return fib(n-1) + fib(n-2)
}
5 changes: 5 additions & 0 deletions gnovm/pkg/gnolang/benchdata/loop.gno
Original file line number Diff line number Diff line change
@@ -0,0 +1,5 @@
package main

func main() {
for i := 0; i < {{ .N }}; i++ {}
}
90 changes: 90 additions & 0 deletions gnovm/pkg/gnolang/benchdata/matrix.gno
Original file line number Diff line number Diff line change
@@ -0,0 +1,90 @@
// param: 3 4 5 6

package main

func main() {
const p = {{ .Param }}
for i := 0; i < {{ .N }}; i++ {
_ = det(mul(dist(p), mul(id(p), dist(p))))
}
}

// identity matrix
func id(sz int) [][]int {
r := make([][]int, sz)
for i := range r {
r[i] = make([]int, sz)
r[i][i] = 1
}
return r
}

// distance from corner
func dist(sz int) [][]int {
half := sz / 2
r := make([][]int, sz)
for i := range r {
r[i] = make([]int, sz)
vdist := i
if vdist >= half {
vdist = sz - 1 - i
}
for j := range r[i] {
hdist := j
if hdist >= half {
hdist = sz - 1 - j
}
r[i][j] = vdist + hdist
}
}
return r
}

func det(m [][]int) int {
size := len(m)
if size == 2 {
return m[0][0]*m[1][1] - m[0][1]*m[1][0]
}
subMatrix := make([][]int, size-1)
for j := range subMatrix {
subMatrix[j] = make([]int, size-1)
}

determinant := 0
for i := 0; i < size; i++ {
for j := 1; j < size; j++ {
t := 0
for k := 0; k < size; k++ {
if k == i {
continue
}
subMatrix[j-1][t] = m[j][k]
t++
}
}
sign := 1
if i % 2 == 1 {
sign = -1
}
determinant += m[0][i] * det(subMatrix) * sign
}
return determinant
}

func mul(m1, m2 [][]int) [][]int {
size := len(m1)
result := make([][]int, size)
for i := range result {
result[i] = make([]int, size)
}

for i := 0; i < size; i++ {
for j := 0; j < size; j++ {
for k := 0; k < size; k++ {
result[i][j] += m1[i][k] * m2[k][j]
}
}
}

return result
}
103 changes: 75 additions & 28 deletions gnovm/pkg/gnolang/gno_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -3,12 +3,18 @@ package gnolang
import (
"bytes"
"fmt"
"io"
"os"
"path/filepath"
"reflect"
"strings"
"testing"
"text/template"
"unsafe"

// "github.com/davecgh/go-spew/spew"
"github.com/jaekwon/testify/assert"
"github.com/jaekwon/testify/require"
)

// run empty main().
Expand Down Expand Up @@ -192,43 +198,84 @@ func main() {
// Benchmarks

func BenchmarkPreprocess(b *testing.B) {
for i := 0; i < b.N; i++ {
// stop timer
b.StopTimer()
pkg := &PackageNode{
PkgName: "main",
PkgPath: ".main",
FileSet: nil,
}
pkg.InitStaticBlock(pkg, nil)
main := FuncD("main", nil, nil, Ss(
A("mx", ":=", "1000000"),
For(
A("i", ":=", "0"),
X("i < mx"),
Inc("i"),
),
))
b.StartTimer()
// timer started
main = Preprocess(nil, pkg, main).(*FuncDecl)
pkg := &PackageNode{
PkgName: "main",
PkgPath: ".main",
FileSet: nil,
}
}

func BenchmarkLoopyMain(b *testing.B) {
m := NewMachine("test", nil)
pkg.InitStaticBlock(pkg, nil)
main := FuncD("main", nil, nil, Ss(
A("mx", ":=", "10000000"),
A("mx", ":=", "1000000"),
For(
A("i", ":=", "0"),
// X("i < 10000000"),
X("i < mx"),
Inc("i"),
),
))
m.RunDeclaration(main)
copies := make([]*FuncDecl, b.N)
for i := 0; i < b.N; i++ {
copies[i] = main.Copy().(*FuncDecl)
}
b.ResetTimer()

for i := 0; i < b.N; i++ {
m.RunMain()
main = Preprocess(nil, pkg, copies[i]).(*FuncDecl)
}
}

type bdataParams struct {
N int
Param string
}

func BenchmarkBenchdata(b *testing.B) {
const bdDir = "./benchdata"
files, err := os.ReadDir(bdDir)
require.NoError(b, err)
for _, file := range files {
// Read file and parse template.
bcont, err := os.ReadFile(filepath.Join(bdDir, file.Name()))
cont := string(bcont)
require.NoError(b, err)
tpl, err := template.New("").Parse(cont)
require.NoError(b, err)

// Determine parameters.
const paramString = "// param: "
var params []string
pos := strings.Index(cont, paramString)
if pos >= 0 {
paramsRaw := strings.SplitN(cont[pos+len(paramString):], "\n", 2)[0]
params = strings.Fields(paramsRaw)
} else {
params = []string{""}
}

for _, param := range params {
name := file.Name()
if param != "" {
name += "_param:" + param
}
b.Run(name, func(b *testing.B) {
// Gen template with N and param.
var buf bytes.Buffer
require.NoError(b, tpl.Execute(&buf, bdataParams{
N: b.N,
Param: param,
}))

// Set up machine.
m := NewMachineWithOptions(MachineOptions{
PkgPath: "main",
Output: io.Discard,
})
n := MustParseFile("main.go", buf.String())
m.RunFiles(n)

b.ResetTimer()
m.RunMain()
})
}
}
}

Expand Down
Loading

0 comments on commit 78bd2a9

Please sign in to comment.