chore(systemtests): better support for app v2 (backport #21128) (#21160)

Co-authored-by: Alexander Peters <[email protected]>

scripts/build/testing.mk

@@ -29,7 +29,7 @@ test-all: test-unit test-e2e test-integration test-ledger-mock test-race
 .PHONY: test-system
 test-system: build
 	mkdir -p ./tests/systemtests/binaries/
-	cp $(BUILDDIR)/simd ./tests/systemtests/binaries/
+	cp $(BUILDDIR)/simd$(if $(findstring v2,$(COSMOS_BUILD_OPTIONS)),v2) ./tests/systemtests/binaries/
 	$(MAKE) -C tests/systemtests test
 
 

tests/systemtests/Makefile

@@ -5,7 +5,7 @@ WAIT_TIME ?= 45s
 all: test
 
 test:
-	go test -mod=readonly -failfast -tags='system_test' ./... --wait-time=$(WAIT_TIME) --verbose
+	go test -mod=readonly -failfast -tags='system_test' ./... --wait-time=$(WAIT_TIME) --verbose $(if $(findstring v2,$(COSMOS_BUILD_OPTIONS)),--binary=simdv2)
 
 format:
 	find . -name '*.go' -type f -not -path "./vendor*" -not -path "*.git*" -not -path "./client/lcd/statik/statik.go" | xargs gofumpt -w

tests/systemtests/cli.go

@@ -223,7 +223,7 @@ func (c CLIWrapper) runWithInput(args []string, input io.Reader) (output string,
 		cmd := exec.Command(locateExecutable(c.execBinary), args...) //nolint:gosec // test code only
 		cmd.Dir = WorkDir
 		cmd.Stdin = input
-		return cmd.CombinedOutput()
+		return cmd.Output()
 	}()
 	gotOut = filterProtoNoise(gotOut)
 	ok = c.assertErrorFn(c.t, gotErr, string(gotOut))

tests/systemtests/fraud_test.go

@@ -29,9 +29,10 @@ func TestValidatorDoubleSign(t *testing.T) {
 	t.Log(rsp)
 	nodePowerBefore := QueryCometValidatorPowerForNode(t, sut, 0)
 	require.NotEmpty(t, nodePowerBefore)
+	t.Logf("nodePowerBefore: %v", nodePowerBefore)
 
 	var validatorPubKey cryptotypes.PubKey
-	newNode := sut.AddFullnode(t, "0.001stake", func(nodeNumber int, nodePath string) {
+	newNode := sut.AddFullnode(t, func(nodeNumber int, nodePath string) {
 		valKeyFile := filepath.Join(WorkDir, nodePath, "config", "priv_validator_key.json")
 		_ = os.Remove(valKeyFile)
 		_, err := copyFile(filepath.Join(WorkDir, sut.nodePath(0), "config", "priv_validator_key.json"), valKeyFile)
@@ -54,9 +55,15 @@ func TestValidatorDoubleSign(t *testing.T) {
 	// then comet status updated
 	nodePowerAfter := QueryCometValidatorPowerForNode(t, sut, 0)
 	require.Empty(t, nodePowerAfter)
+	t.Logf("nodePowerAfter: %v", nodePowerAfter)
 
 	// and sdk status updated
 	byzantineOperatorAddr := cli.GetKeyAddrPrefix("node0", "val")
 	rsp = cli.CustomQuery("q", "staking", "validator", byzantineOperatorAddr)
 	assert.True(t, gjson.Get(rsp, "validator.jailed").Bool(), rsp)
+
+	t.Log("let's run for some blocks to confirm all good")
+	for i := 0; i < 10; i++ {
+		sut.AwaitNextBlock(t)
+	}
 }

tests/systemtests/system.go

@@ -23,6 +23,7 @@ import (
 	tmtypes "github.com/cometbft/cometbft/types"
 	"github.com/stretchr/testify/require"
 	"github.com/tidwall/sjson"
+	"golang.org/x/exp/maps"
 
 	"github.com/cosmos/cosmos-sdk/server"
 	sdk "github.com/cosmos/cosmos-sdk/types"
@@ -154,7 +155,7 @@ func (s *SystemUnderTest) StartChain(t *testing.T, xargs ...string) {
 	t.Helper()
 	s.Log("Start chain\n")
 	s.ChainStarted = true
-	s.startNodesAsync(t, append([]string{"start", "--trace", "--log_level=info", "--log_no_color"}, xargs...)...)
+	s.startNodesAsync(t, append([]string{"start", "--log_level=info", "--log_no_color"}, xargs...)...)
 
 	s.AwaitNodeUp(t, s.rpcAddr)
 
@@ -332,10 +333,10 @@ func (s *SystemUnderTest) StopChain() {
 
 func (s *SystemUnderTest) withEachPid(cb func(p *os.Process)) {
 	s.pidsLock.RLock()
-	pids := s.pids
+	pids := maps.Keys(s.pids)
 	s.pidsLock.RUnlock()
 
-	for pid := range pids {
+	for _, pid := range pids {
 		p, err := os.FindProcess(pid)
 		if err != nil {
 			continue
@@ -528,28 +529,28 @@ func (s *SystemUnderTest) ForEachNodeExecAndWait(t *testing.T, cmds ...[]string)
 		for j, xargs := range cmds {
 			xargs = append(xargs, "--home", home)
 			s.Logf("Execute `%s %s`\n", s.execBinary, strings.Join(xargs, " "))
-			out := runShellCmd(t, s.execBinary, xargs...)
+			out := MustRunShellCmd(t, s.execBinary, xargs...)
 			s.Logf("Result: %s\n", out)
 			result[i][j] = out
 		}
 	})
 	return result
 }
 
-func runShellCmd(t *testing.T, cmd string, args ...string) string {
+func MustRunShellCmd(t *testing.T, cmd string, args ...string) string {
 	t.Helper()
-	out, err := runShellCmdX(cmd, args...)
+	out, err := RunShellCmd(cmd, args...)
 	require.NoError(t, err)
 	return out
 }
 
-func runShellCmdX(cmd string, args ...string) (string, error) {
+func RunShellCmd(cmd string, args ...string) (string, error) {
 	c := exec.Command( //nolint:gosec // used by tests only
 		locateExecutable(cmd),
 		args...,
 	)
 	c.Dir = WorkDir
-	out, err := c.CombinedOutput()
+	out, err := c.Output()
 	if err != nil {
 		return string(out), fmt.Errorf("run `%s %s`: out: %s: %w", cmd, strings.Join(args, " "), string(out), err)
 	}
@@ -560,7 +561,7 @@ func runShellCmdX(cmd string, args ...string) (string, error) {
 func (s *SystemUnderTest) startNodesAsync(t *testing.T, xargs ...string) {
 	t.Helper()
 	s.withEachNodeHome(func(i int, home string) {
-		args := append(xargs, "--home", home)
+		args := append(xargs, "--home="+home)
 		s.Logf("Execute `%s %s`\n", s.execBinary, strings.Join(args, " "))
 		cmd := exec.Command( //nolint:gosec // used by tests only
 			locateExecutable(s.execBinary),
@@ -569,24 +570,27 @@ func (s *SystemUnderTest) startNodesAsync(t *testing.T, xargs ...string) {
 		cmd.Dir = WorkDir
 		s.watchLogs(i, cmd)
 		require.NoError(t, cmd.Start(), "node %d", i)
-
-		pid := cmd.Process.Pid
-		s.pidsLock.Lock()
-		s.pids[pid] = struct{}{}
-		s.pidsLock.Unlock()
-		s.Logf("Node started: %d\n", pid)
+		s.Logf("Node started: %d\n", cmd.Process.Pid)
 
 		// cleanup when stopped
-		go func(pid int) {
-			_ = cmd.Wait() // blocks until shutdown
-			s.pidsLock.Lock()
-			delete(s.pids, pid)
-			s.pidsLock.Unlock()
-			s.Logf("Node stopped: %d\n", pid)
-		}(pid)
+		s.awaitProcessCleanup(cmd)
 	})
 }
 
+func (s *SystemUnderTest) awaitProcessCleanup(cmd *exec.Cmd) {
+	pid := cmd.Process.Pid
+	s.pidsLock.Lock()
+	s.pids[pid] = struct{}{}
+	s.pidsLock.Unlock()
+	go func() {
+		_ = cmd.Wait() // blocks until shutdown
+		s.Logf("Node stopped: %d\n", pid)
+		s.pidsLock.Lock()
+		delete(s.pids, pid)
+		s.pidsLock.Unlock()
+	}()
+}
+
 func (s *SystemUnderTest) withEachNodeHome(cb func(i int, home string)) {
 	for i := 0; i < s.nodesCount; i++ {
 		cb(i, s.nodePath(i))
@@ -658,7 +662,7 @@ func (s *SystemUnderTest) resetBuffers() {
 }
 
 // AddFullnode starts a new fullnode that connects to the existing chain but is not a validator.
-func (s *SystemUnderTest) AddFullnode(t *testing.T, minGasPrices string, beforeStart ...func(nodeNumber int, nodePath string)) Node {
+func (s *SystemUnderTest) AddFullnode(t *testing.T, beforeStart ...func(nodeNumber int, nodePath string)) Node {
 	t.Helper()
 	s.MarkDirty()
 	s.nodesCount++
@@ -668,7 +672,7 @@ func (s *SystemUnderTest) AddFullnode(t *testing.T, minGasPrices string, beforeS
 
 	// prepare new node
 	moniker := fmt.Sprintf("node%d", nodeNumber)
-	args := []string{"init", moniker, "--home", nodePath, "--overwrite"}
+	args := []string{"init", moniker, "--home=" + nodePath, "--overwrite"}
 	s.Logf("Execute `%s %s`\n", s.execBinary, strings.Join(args, " "))
 	cmd := exec.Command( //nolint:gosec // used by tests only
 		locateExecutable(s.execBinary),
@@ -679,12 +683,15 @@ func (s *SystemUnderTest) AddFullnode(t *testing.T, minGasPrices string, beforeS
 	require.NoError(t, cmd.Run(), "failed to start node with id %d", nodeNumber)
 	require.NoError(t, saveGenesis(nodePath, []byte(s.ReadGenesisJSON(t))))
 
-	// quick hack: copy config and overwrite by start params
-	configFile := filepath.Join(WorkDir, nodePath, "config", "config.toml")
-	_ = os.Remove(configFile)
-	_, err := copyFile(filepath.Join(WorkDir, s.nodePath(0), "config", "config.toml"), configFile)
-	require.NoError(t, err)
+	configPath := filepath.Join(WorkDir, nodePath, "config")
 
+	// quick hack: copy config and overwrite by start params
+	for _, tomlFile := range []string{"config.toml", "app.toml"} {
+		configFile := filepath.Join(configPath, tomlFile)
+		_ = os.Remove(configFile)
+		_, err := copyFile(filepath.Join(WorkDir, s.nodePath(0), "config", tomlFile), configFile)
+		require.NoError(t, err)
+	}
 	// start node
 	allNodes := s.AllNodes(t)
 	node := allNodes[len(allNodes)-1]
@@ -701,7 +708,6 @@ func (s *SystemUnderTest) AddFullnode(t *testing.T, minGasPrices string, beforeS
 		fmt.Sprintf("--p2p.laddr=tcp://localhost:%d", node.P2PPort),
 		fmt.Sprintf("--rpc.laddr=tcp://localhost:%d", node.RPCPort),
 		fmt.Sprintf("--grpc.address=localhost:%d", 9090+nodeNumber),
-		fmt.Sprintf("--minimum-gas-prices=%s", minGasPrices),
 		"--p2p.pex=false",
 		"--moniker=" + moniker,
 		"--log_level=info",
@@ -716,6 +722,7 @@ func (s *SystemUnderTest) AddFullnode(t *testing.T, minGasPrices string, beforeS
 	cmd.Dir = WorkDir
 	s.watchLogs(nodeNumber, cmd)
 	require.NoError(t, cmd.Start(), "node %d", nodeNumber)
+	s.awaitProcessCleanup(cmd)
 	return node
 }
 

tests/systemtests/testnet_init.go

@@ -58,7 +58,7 @@ func (s SingleHostTestnetCmdInitializer) Initialize() {
 		"--single-host",
 	}
 	s.log(fmt.Sprintf("+++ %s %s\n", s.execBinary, strings.Join(args, " ")))
-	out, err := runShellCmdX(s.execBinary, args...)
+	out, err := RunShellCmd(s.execBinary, args...)
 	if err != nil {
 		panic(err)
 	}
@@ -112,7 +112,7 @@ func (s ModifyConfigYamlInitializer) Initialize() {
 	}
 	s.log(fmt.Sprintf("+++ %s %s\n", s.execBinary, strings.Join(args, " ")))
 
-	out, err := runShellCmdX(s.execBinary, args...)
+	out, err := RunShellCmd(s.execBinary, args...)
 	if err != nil {
 		panic(err)
 	}

tests/systemtests/upgrade_test.go

@@ -106,8 +106,8 @@ func FetchExecutable(t *testing.T, version string) string {
 	}
 	destFile := cacheFile
 	t.Log("+++ version not in cache, downloading from docker image")
-	runShellCmd(t, "docker", "pull", "ghcr.io/cosmos/simapp:"+version)
-	runShellCmd(t, "docker", "create", "--name=ci_temp", "ghcr.io/cosmos/simapp:"+version)
-	runShellCmd(t, "docker", "cp", "ci_temp:/usr/bin/simd", destFile)
+	MustRunShellCmd(t, "docker", "pull", "ghcr.io/cosmos/simapp:"+version)
+	MustRunShellCmd(t, "docker", "create", "--name=ci_temp", "ghcr.io/cosmos/simapp:"+version)
+	MustRunShellCmd(t, "docker", "cp", "ci_temp:/usr/bin/simd", destFile)
 	return destFile
 }