Skip to content

Commit

Permalink
cmd/devp2p, internal/utesting: implement TAP output (ethereum#21760)
Browse files Browse the repository at this point in the history
TAP is a text format for test results. Parsers for it are available in many languages,
making it easy to consume. I want TAP output from our protocol tests because the
Hive wrapper around them needs to know about the test names and their individual
results and logs. It would also be possible to just write this info as JSON, but I don't
want to invent a new format.

This also improves the normal console output for tests (when running without --tap).
It now prints -- RUN lines before any output from the test, and indents the log output
by one space.
  • Loading branch information
fjl authored and enriquefynn committed Feb 15, 2021
1 parent 9109549 commit 7387d1a
Show file tree
Hide file tree
Showing 6 changed files with 329 additions and 81 deletions.
38 changes: 9 additions & 29 deletions cmd/devp2p/discv4cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,14 +19,12 @@ package main
import (
"fmt"
"net"
"os"
"strings"
"time"

"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v4test"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/utesting"
"github.com/ethereum/go-ethereum/p2p/discover"
"github.com/ethereum/go-ethereum/p2p/enode"
"github.com/ethereum/go-ethereum/params"
Expand Down Expand Up @@ -82,7 +80,13 @@ var (
Name: "test",
Usage: "Runs tests against a node",
Action: discv4Test,
Flags: []cli.Flag{remoteEnodeFlag, testPatternFlag, testListen1Flag, testListen2Flag},
Flags: []cli.Flag{
remoteEnodeFlag,
testPatternFlag,
testTAPFlag,
testListen1Flag,
testListen2Flag,
},
}
)

Expand Down Expand Up @@ -113,20 +117,6 @@ var (
Usage: "Enode of the remote node under test",
EnvVar: "REMOTE_ENODE",
}
testPatternFlag = cli.StringFlag{
Name: "run",
Usage: "Pattern of test suite(s) to run",
}
testListen1Flag = cli.StringFlag{
Name: "listen1",
Usage: "IP address of the first tester",
Value: v4test.Listen1,
}
testListen2Flag = cli.StringFlag{
Name: "listen2",
Usage: "IP address of the second tester",
Value: v4test.Listen2,
}
)

func discv4Ping(ctx *cli.Context) error {
Expand Down Expand Up @@ -213,6 +203,7 @@ func discv4Crawl(ctx *cli.Context) error {
return nil
}

// discv4Test runs the protocol test suite.
func discv4Test(ctx *cli.Context) error {
// Configure test package globals.
if !ctx.IsSet(remoteEnodeFlag.Name) {
Expand All @@ -221,18 +212,7 @@ func discv4Test(ctx *cli.Context) error {
v4test.Remote = ctx.String(remoteEnodeFlag.Name)
v4test.Listen1 = ctx.String(testListen1Flag.Name)
v4test.Listen2 = ctx.String(testListen2Flag.Name)

// Filter and run test cases.
tests := v4test.AllTests
if ctx.IsSet(testPatternFlag.Name) {
tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
}
results := utesting.RunTests(tests, os.Stdout)
if fails := utesting.CountFailures(results); fails > 0 {
return fmt.Errorf("%v/%v tests passed.", len(tests)-fails, len(tests))
}
fmt.Printf("%v/%v passed\n", len(tests), len(tests))
return nil
return runTests(ctx, v4test.AllTests)
}

// startV4 starts an ephemeral discovery V4 node.
Expand Down
28 changes: 8 additions & 20 deletions cmd/devp2p/discv5cmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -18,13 +18,10 @@ package main

import (
"fmt"
"os"
"time"

"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v5test"
"github.com/ethereum/go-ethereum/common"
"github.com/ethereum/go-ethereum/internal/utesting"
"github.com/ethereum/go-ethereum/log"
"github.com/ethereum/go-ethereum/p2p/discover"
"gopkg.in/urfave/cli.v1"
)
Expand Down Expand Up @@ -62,7 +59,12 @@ var (
Name: "test",
Usage: "Runs protocol tests against a node",
Action: discv5Test,
Flags: []cli.Flag{testPatternFlag, testListen1Flag, testListen2Flag},
Flags: []cli.Flag{
testPatternFlag,
testTAPFlag,
testListen1Flag,
testListen2Flag,
},
}
discv5ListenCommand = cli.Command{
Name: "listen",
Expand Down Expand Up @@ -114,28 +116,14 @@ func discv5Crawl(ctx *cli.Context) error {
return nil
}

// discv5Test runs the protocol test suite.
func discv5Test(ctx *cli.Context) error {
// Disable logging unless explicitly enabled.
if !ctx.GlobalIsSet("verbosity") && !ctx.GlobalIsSet("vmodule") {
log.Root().SetHandler(log.DiscardHandler())
}

// Filter and run test cases.
suite := &v5test.Suite{
Dest: getNodeArg(ctx),
Listen1: ctx.String(testListen1Flag.Name),
Listen2: ctx.String(testListen2Flag.Name),
}
tests := suite.AllTests()
if ctx.IsSet(testPatternFlag.Name) {
tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
}
results := utesting.RunTests(tests, os.Stdout)
if fails := utesting.CountFailures(results); fails > 0 {
return fmt.Errorf("%v/%v tests passed.", len(tests)-fails, len(tests))
}
fmt.Printf("%v/%v passed\n", len(tests), len(tests))
return nil
return runTests(ctx, suite.AllTests())
}

func discv5Listen(ctx *cli.Context) error {
Expand Down
24 changes: 7 additions & 17 deletions cmd/devp2p/rlpxcmd.go
Original file line number Diff line number Diff line change
Expand Up @@ -19,11 +19,9 @@ package main
import (
"fmt"
"net"
"os"

"github.com/ethereum/go-ethereum/cmd/devp2p/internal/ethtest"
"github.com/ethereum/go-ethereum/crypto"
"github.com/ethereum/go-ethereum/internal/utesting"
"github.com/ethereum/go-ethereum/p2p"
"github.com/ethereum/go-ethereum/p2p/rlpx"
"github.com/ethereum/go-ethereum/rlp"
Expand All @@ -47,9 +45,12 @@ var (
rlpxEthTestCommand = cli.Command{
Name: "eth-test",
Usage: "Runs tests against a node",
ArgsUsage: "<node> <path_to_chain.rlp_file>",
ArgsUsage: "<node> <chain.rlp> <genesis.json>",
Action: rlpxEthTest,
Flags: []cli.Flag{testPatternFlag},
Flags: []cli.Flag{
testPatternFlag,
testTAPFlag,
},
}
)

Expand Down Expand Up @@ -88,22 +89,11 @@ func rlpxPing(ctx *cli.Context) error {
return nil
}

// rlpxEthTest runs the eth protocol test suite.
func rlpxEthTest(ctx *cli.Context) error {
if ctx.NArg() < 3 {
exit("missing path to chain.rlp as command-line argument")
}

suite := ethtest.NewSuite(getNodeArg(ctx), ctx.Args()[1], ctx.Args()[2])

// Filter and run test cases.
tests := suite.AllTests()
if ctx.IsSet(testPatternFlag.Name) {
tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
}
results := utesting.RunTests(tests, os.Stdout)
if fails := utesting.CountFailures(results); fails > 0 {
return fmt.Errorf("%v of %v tests passed.", len(tests)-fails, len(tests))
}
fmt.Printf("all tests passed\n")
return nil
return runTests(ctx, suite.AllTests())
}
69 changes: 69 additions & 0 deletions cmd/devp2p/runtest.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,69 @@
// Copyright 2020 The go-ethereum Authors
// This file is part of go-ethereum.
//
// go-ethereum is free software: you can redistribute it and/or modify
// it under the terms of the GNU General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// go-ethereum is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU General Public License for more details.
//
// You should have received a copy of the GNU General Public License
// along with go-ethereum. If not, see <http://www.gnu.org/licenses/>.

package main

import (
"os"

"github.com/ethereum/go-ethereum/cmd/devp2p/internal/v4test"
"github.com/ethereum/go-ethereum/internal/utesting"
"github.com/ethereum/go-ethereum/log"
"gopkg.in/urfave/cli.v1"
)

var (
testPatternFlag = cli.StringFlag{
Name: "run",
Usage: "Pattern of test suite(s) to run",
}
testTAPFlag = cli.BoolFlag{
Name: "tap",
Usage: "Output TAP",
}
// These two are specific to the discovery tests.
testListen1Flag = cli.StringFlag{
Name: "listen1",
Usage: "IP address of the first tester",
Value: v4test.Listen1,
}
testListen2Flag = cli.StringFlag{
Name: "listen2",
Usage: "IP address of the second tester",
Value: v4test.Listen2,
}
)

func runTests(ctx *cli.Context, tests []utesting.Test) error {
// Filter test cases.
if ctx.IsSet(testPatternFlag.Name) {
tests = utesting.MatchTests(tests, ctx.String(testPatternFlag.Name))
}
// Disable logging unless explicitly enabled.
if !ctx.GlobalIsSet("verbosity") && !ctx.GlobalIsSet("vmodule") {
log.Root().SetHandler(log.DiscardHandler())
}
// Run the tests.
var run = utesting.RunTests
if ctx.Bool(testTAPFlag.Name) {
run = utesting.RunTAP
}
results := run(tests, os.Stdout)
if utesting.CountFailures(results) > 0 {
os.Exit(1)
}
return nil
}
Loading

0 comments on commit 7387d1a

Please sign in to comment.