Migrate polybft e2e framework to edge (#797)

Back-port the e2e framework + tests from v3 to edge as a separate set of tests.

.github/workflows/e2e-polybft.yml

@@ -0,0 +1,36 @@
+---
+name: PolyBFT E2E tests
+on:  # yamllint disable-line rule:truthy
+  push:
+    branches:
+      - main
+      - develop
+  pull_request:
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    env:
+      E2E_TESTS: true
+      E2E_LOGS: true
+      CI_VERBOSE: true
+    steps:
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          token: ${{ secrets.GH_TOKEN_CLONE_PRIVATE }}
+      - name: Install Go
+        uses: actions/setup-go@v3
+        with:
+          go-version: 1.18.x
+      - name: Compile v3 contracts
+        run: make compile-v3-contracts
+      - name: Run tests
+        run: make test-e2e-polybft
+      - name: Archive test logs
+        if: always()
+        uses: actions/upload-artifact@v3
+        with:
+          name: e2e-logs
+          path: e2e-logs-*/
+          retention-days: 30

.github/workflows/e2e.yaml

@@ -15,7 +15,10 @@ jobs:
       E2E_LOGS: true
       CI_VERBOSE: true
     steps:
-      - uses: actions/checkout@v2
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          token: ${{ secrets.GH_TOKEN_CLONE_PRIVATE }}
       - name: Install Go
         uses: actions/setup-go@v3
         with:

Makefile

@@ -53,6 +53,12 @@ test-e2e:
 	go build -race -o artifacts/polygon-edge .
 	env EDGE_BINARY=${PWD}/artifacts/polygon-edge go test -v -timeout=30m ./e2e/...
 
+.PHONY: test-e2ev3
+test-e2e-polybft:
+    # We can not build with race because of a bug in boltdb dependency
+	go build -o artifacts/polygon-edge .
+	env EDGE_BINARY=${PWD}/artifacts/polygon-edge E2E_TESTS=true E2E_LOGS=true go test -v -timeout=30m ./e2e-polybft/...
+
 .PHONY: run-local
 run-local:
 	$(MAKE) compile-v3-contracts

command/genesis/genesis.go

@@ -185,6 +185,13 @@ func setFlags(cmd *cobra.Command) {
 			command.DefaultPremineBalance,
 			"the amount which will be premined to all the validators",
 		)
+		cmd.Flags().StringVar(
+			&params.smartContractsRootPath,
+			smartContractsRootPathFlag,
+			defaultContractsRootFolder,
+			"the smart contracts folder",
+		)
+
 		cmd.MarkFlagsMutuallyExclusive(premineFlag, premineValidatorsFlag)
 		cmd.MarkFlagsMutuallyExclusive(validatorsFlag, premineValidatorsFlag)
 	}

command/genesis/params.go

@@ -82,6 +82,8 @@ type genesisParams struct {
 
 	polyBftValidatorPrefixPath string
 	premineValidators          string
+
+	smartContractsRootPath string
 }
 
 func (p *genesisParams) validateFlags() error {

command/genesis/polybft_params.go

@@ -20,6 +20,7 @@ import (
 const (
 	premineValidatorsFlag          = "premine-validators"
 	polyBftValidatorPrefixPathFlag = "validator-prefix"
+	smartContractsRootPathFlag     = "contracts-path"
 
 	validatorSetSizeFlag = "validator-set-size"
 	sprintSizeFlag       = "sprint-size"
@@ -34,6 +35,8 @@ const (
 	defaultPolyBftValidatorPrefixPath = "test-chain-"
 
 	bootnodePortStart = 30301
+
+	defaultContractsRootFolder = "consensus/polybft/polybftcontracts/artifacts/contracts"
 )
 
 var (
@@ -42,8 +45,6 @@ var (
 	sidechainBridgeAddr      = types.StringToAddress("0x5a443704dd4B594B382c22a083e2BD3090A6feF3")
 	sidechainERC20Addr       = types.StringToAddress("0x47e9Fbef8C83A1714F1951F142132E6e90F5fa5D")
 	sidechainERC20BridgeAddr = types.StringToAddress("0x8Be503bcdEd90ED42Eff31f56199399B2b0154CA")
-
-	contractsRootFolder = "consensus/polybft/polybftcontracts/artifacts/contracts"
 )
 
 func (p *genesisParams) generatePolyBFTConfig() (*chain.Chain, error) {
@@ -52,7 +53,7 @@ func (p *genesisParams) generatePolyBFTConfig() (*chain.Chain, error) {
 		return nil, err
 	}
 
-	smartContracts, err := deployContracts()
+	smartContracts, err := p.deployContracts()
 	if err != nil {
 		return nil, err
 	}
@@ -151,7 +152,7 @@ func (p *genesisParams) generatePolyBftGenesis() error {
 	return helper.WriteGenesisConfigToDisk(config, params.genesisPath)
 }
 
-func deployContracts() ([]polybft.SmartContract, error) {
+func (p *genesisParams) deployContracts() ([]polybft.SmartContract, error) {
 	predefinedContracts := []struct {
 		name     string
 		input    []interface{}
@@ -190,7 +191,7 @@ func deployContracts() ([]polybft.SmartContract, error) {
 	result := make([]polybft.SmartContract, 0, len(predefinedContracts))
 
 	for _, contract := range predefinedContracts {
-		artifact, err := polybftcontracts.ReadArtifact(contractsRootFolder, contract.chain, contract.name)
+		artifact, err := polybftcontracts.ReadArtifact(p.smartContractsRootPath, contract.chain, contract.name)
 		if err != nil {
 			return nil, err
 		}

consensus/polybft/state.go

@@ -9,8 +9,8 @@ import (
 	"sort"
 
 	"github.com/0xPolygon/pbft-consensus"
-	"github.com/boltdb/bolt"
 	"github.com/hashicorp/go-hclog"
+	bolt "go.etcd.io/bbolt"
 
 	"github.com/umbracle/ethgo"
 	"github.com/umbracle/ethgo/abi"

e2e-polybft/README.md

@@ -0,0 +1,22 @@
+
+# End-to-End testing
+
+The implemented E2E tests start a local instance of V3.
+
+As such, they require the binary 'polygon-edge' to be available in the $PATH variable.
+
+## Step 1: Build the polygon-edge
+
+```bash
+go build -race -o artifacts/polygon-edge . && mv artifacts/polygon-edge $GOPATH/bin
+```
+
+In this case we expect that $GOPATH is set and $GOPATH/bin is defined in $PATH as it is required for a complete Go installation.
+
+## Step 2: Run the tests
+
+```bash
+export E2E_TESTS=TRUE && go test -v ./e2e-polybft/...
+```
+
+To enable logs in the e2e test set `E2E_LOGS=true`.

e2e-polybft/consensus_test.go

@@ -0,0 +1,91 @@
+package e2e
+
+import (
+	"testing"
+	"time"
+
+	"github.com/0xPolygon/polygon-edge/e2e-polybft/framework"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestE2E_Consensus_Basic_WithNonValidators(t *testing.T) {
+	cluster := framework.NewTestCluster(t, 7,
+		framework.WithNonValidators(2), framework.WithValidatorSnapshot(5))
+	defer cluster.Stop()
+
+	assert.NoError(t, cluster.WaitForBlock(22, 1*time.Minute))
+}
+
+func TestE2E_Consensus_Sync_WithNonValidators(t *testing.T) {
+	// one non-validator node from the ensemble gets disconnected and connected again.
+	// It should be able to pick up from the synchronization protocol again.
+	cluster := framework.NewTestCluster(t, 7,
+		framework.WithNonValidators(2), framework.WithValidatorSnapshot(5))
+	defer cluster.Stop()
+
+	// wait for the start
+	assert.NoError(t, cluster.WaitForBlock(20, 1*time.Minute))
+
+	// stop one non-validator node
+	node := cluster.Servers[6]
+	node.Stop()
+
+	// wait for at least 15 more blocks before starting again
+	assert.NoError(t, cluster.WaitForBlock(35, 1*time.Minute))
+
+	// start the node again
+	node.Start()
+
+	// wait for block 55
+	assert.NoError(t, cluster.WaitForBlock(55, 2*time.Minute))
+}
+
+func TestE2E_Consensus_Sync(t *testing.T) {
+	// one node from the ensemble gets disconnected and connected again.
+	// It should be able to pick up from the synchronization protocol again.
+	cluster := framework.NewTestCluster(t, 5, framework.WithValidatorSnapshot(5))
+	defer cluster.Stop()
+
+	// wait for the start
+	assert.NoError(t, cluster.WaitForBlock(5, 1*time.Minute))
+
+	// stop one node
+	node := cluster.Servers[0]
+	node.Stop()
+
+	// wait for at least 15 more blocks before starting again
+	assert.NoError(t, cluster.WaitForBlock(20, 1*time.Minute))
+
+	// start the node again
+	node.Start()
+
+	// wait for block 35
+	assert.NoError(t, cluster.WaitForBlock(35, 2*time.Minute))
+}
+
+func TestE2E_Consensus_Bulk_Drop(t *testing.T) {
+	clusterSize := 5
+	bulkToDrop := 3
+
+	cluster := framework.NewTestCluster(t, clusterSize)
+	defer cluster.Stop()
+
+	// wait for cluster to start
+	require.NoError(t, cluster.WaitForBlock(5, 1*time.Minute))
+
+	// drop bulk of nodes from cluster
+	for i := 0; i < bulkToDrop; i++ {
+		node := cluster.Servers[i]
+		node.Stop()
+	}
+
+	// start dropped nodes again
+	for i := 0; i < bulkToDrop; i++ {
+		node := cluster.Servers[i]
+		node.Start()
+	}
+
+	// wait for block 10
+	assert.NoError(t, cluster.WaitForBlock(10, 2*time.Minute))
+}

e2e-polybft/framework/node.go

@@ -0,0 +1,77 @@
+package framework
+
+import (
+	"io"
+	"os"
+	"os/exec"
+	"sync/atomic"
+)
+
+type node struct {
+	shuttingDown uint64
+	cmd          *exec.Cmd
+	doneCh       chan struct{}
+	exitResult   *exitResult
+}
+
+func newNode(binary string, args []string, stdout io.Writer) (*node, error) {
+	cmd := exec.Command(binary, args...)
+	cmd.Stdout = stdout
+	cmd.Stderr = stdout
+
+	if err := cmd.Start(); err != nil {
+		return nil, err
+	}
+
+	n := &node{
+		cmd:    cmd,
+		doneCh: make(chan struct{}),
+	}
+	go n.run()
+
+	return n, nil
+}
+
+func (n *node) ExitResult() *exitResult {
+	return n.exitResult
+}
+
+func (n *node) Wait() <-chan struct{} {
+	return n.doneCh
+}
+
+func (n *node) run() {
+	err := n.cmd.Wait()
+
+	n.exitResult = &exitResult{
+		Signaled: n.IsShuttingDown(),
+		Err:      err,
+	}
+	close(n.doneCh)
+	n.cmd = nil
+}
+
+func (n *node) IsShuttingDown() bool {
+	return atomic.LoadUint64(&n.shuttingDown) == 1
+}
+
+func (n *node) Stop() error {
+	if n.cmd == nil {
+		// the server is already stopped
+		return nil
+	}
+
+	if err := n.cmd.Process.Signal(os.Interrupt); err != nil {
+		return err
+	}
+
+	atomic.StoreUint64(&n.shuttingDown, 1)
+	<-n.Wait()
+
+	return nil
+}
+
+type exitResult struct {
+	Signaled bool
+	Err      error
+}