cmd/bench: add benchmark wrapper

The coordinator is getting support for running the benchmarks in this
repository. Since the benchmarks and interface are in flux, encoding all
of the details of running Go tests, bent arguments, etc into the
coordinator will likely cause churn and frustrating migration issues.

Instead, add cmd/bench which serves as the simple entrypoint for the
coordinator. The coordinator runs cmd/bench with the GOROOT to test
(eventually multiple GOROOTs), and this binary takes care of the
remaining details.

Right now, we just do a basic go test golang.org/x/benchmarks/... and
simple invocation of bent. Note that bent does not pass without
https://golang.org/cl/354634.

For golang/go#49207

Change-Id: I5c9cf89540cab605c0a64e17af85311d37985c25
Reviewed-on: https://go-review.googlesource.com/c/benchmarks/+/359854
Trust: Michael Pratt <[email protected]>
Run-TryBot: Michael Pratt <[email protected]>
TryBot-Result: Go Bot <[email protected]>
Reviewed-by: Michael Knyszek <[email protected]>

cmd/bench/bent.go

@@ -0,0 +1,120 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"runtime"
+	"text/template"
+)
+
+// TODO(prattmic): refactor bent to export Todo so we can directly build this
+// in Go.
+var configurationTmpl = template.Must(template.New("configuration").Parse(`
+[[Configurations]]
+  Name = "Benchmark"
+  Root = "{{.}}"
+`))
+
+func writeConfiguration(filename, goroot string) error {
+	var buf bytes.Buffer
+	if err := configurationTmpl.Execute(&buf, goroot); err != nil {
+		return fmt.Errorf("error generating configuration: %w", err)
+	}
+
+	log.Printf("bent configuration for GOROOT %s:\n%s", goroot, buf.String())
+
+	if err := os.WriteFile(filename, buf.Bytes(), 0644); err != nil {
+		return fmt.Errorf("error creating configurations.toml: %w", err)
+	}
+
+	return nil
+}
+
+// removeAllIncludingReadonly is like os.RemoveAll except that it'll
+// also try to change permissions to work around permission errors
+// when deleting.
+func removeAllIncludingReadonly(dir string) error {
+	err := os.RemoveAll(dir)
+	if err == nil || !os.IsPermission(err) || runtime.GOOS == "windows" /* different fs permission model */ {
+		return err
+	}
+	// Make a best effort (ignoring errors) attempt to make all
+	// files and directories writable before we try to delete them
+	// all again.
+	filepath.Walk(dir, func(path string, fi os.FileInfo, err error) error {
+		const ownerWritable = 0200
+		if err != nil || fi.Mode().Perm()&ownerWritable != 0 {
+			return nil
+		}
+		os.Chmod(path, fi.Mode().Perm()|ownerWritable)
+		return nil
+	})
+	return os.RemoveAll(dir)
+}
+
+func bent(goroot string) (err error) {
+	dir, err := os.MkdirTemp("", "bent")
+	if err != nil {
+		return fmt.Errorf("error creating temporary directory: %w", err)
+	}
+	defer func() {
+		err = removeAllIncludingReadonly(dir)
+		if err != nil {
+			err = fmt.Errorf("error removing temporary directory: %w", err)
+		}
+	}()
+	log.Printf("Bent temporary directory: %s", dir)
+
+	bentPath := filepath.Join(dir, "bent")
+
+	log.Printf("Building bent...")
+
+	// Build bent itself. N.B. we don't need to do this with the goroot
+	// under test since we aren't testing bent itself, but we are sure that
+	// this toolchain exists.
+	//
+	// TODO(prattmic): do this only once on first call?
+	cmd := goCommand(goroot, "build", "-o", bentPath, "golang.org/x/benchmarks/cmd/bent")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("error building bent: %w", err)
+	}
+
+	log.Printf("Initializing bent...")
+
+	// Initialize scratch dir for bent.
+	cmd = exec.Command(bentPath, "-I")
+	cmd.Dir = dir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("error running bent -I: %w", err)
+	}
+
+	confFile := filepath.Join(dir, "configurations.toml")
+	if err := writeConfiguration(confFile, goroot); err != nil {
+		return fmt.Errorf("error writing configuration: %w", err)
+	}
+
+	log.Printf("Running bent...")
+
+	// Finally we can actually run the benchmarks.
+	cmd = exec.Command(bentPath, "-C", confFile, "-B", filepath.Join(dir, "benchmarks-50.toml"))
+	cmd.Dir = dir
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+	if err := cmd.Run(); err != nil {
+		return fmt.Errorf("error running bent -I: %w", err)
+	}
+
+	return nil
+}

cmd/bench/gotest.go

@@ -0,0 +1,34 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+package main
+
+import (
+	"log"
+	"os"
+	"strings"
+)
+
+func goTest(goroot string) error {
+	log.Printf("Running Go test benchmarks for GOROOT %s", goroot)
+
+	cmd := goCommand(goroot, "test", "-v", "-run=none", "-bench=.", "-count=5", "golang.org/x/benchmarks/...")
+	cmd.Stdout = os.Stdout
+	cmd.Stderr = os.Stderr
+
+	env := os.Environ()
+	needGOROOT := true
+	for i := range env {
+		if strings.HasPrefix(env[i], "GOROOT=") {
+			env[i] = "GOROOT=" + goroot
+			needGOROOT = false
+		}
+	}
+	if needGOROOT {
+		env = append(env, "GOROOT=" + goroot)
+	}
+	cmd.Env = env
+
+	return cmd.Run()
+}

cmd/bench/main.go

@@ -0,0 +1,61 @@
+// Copyright 2021 The Go Authors. All rights reserved.
+// Use of this source code is governed by a BSD-style
+// license that can be found in the LICENSE file.
+
+// Binary bench provides a unified wrapper around the different types of
+// benchmarks in x/benchmarks.
+package main
+
+import (
+	"log"
+	"os"
+	"os/exec"
+	"path/filepath"
+	"strings"
+)
+
+func determineGOROOT() (string, error) {
+	g, ok := os.LookupEnv("GOROOT")
+	if ok {
+		return g, nil
+	}
+
+	cmd := exec.Command("go", "env", "GOROOT")
+	b, err := cmd.Output()
+	if err != nil {
+		return "", err
+	}
+	return strings.TrimSpace(string(b)), nil
+}
+
+func goCommand(goroot string, args ...string) *exec.Cmd {
+	bin := filepath.Join(goroot, "bin/go")
+	cmd := exec.Command(bin, args...)
+	return cmd
+}
+
+func main() {
+	goroot, err := determineGOROOT()
+	if err != nil {
+		log.Fatalf("Unable to determine GOROOT: %v", err)
+	}
+
+	log.Printf("GOROOT under test: %s", goroot)
+
+	pass := true
+
+	if err := goTest(goroot); err != nil {
+		pass = false
+		log.Printf("Error running Go tests: %v", err)
+	}
+
+	if err := bent(goroot); err != nil {
+		pass = false
+		log.Printf("Error running bent: %v", err)
+	}
+
+	if !pass {
+		log.Printf("FAIL")
+		os.Exit(1)
+	}
+}