Merge PR #2187: simulation: Add benchmarking

Makefile

@@ -163,6 +163,17 @@ test_sim_gaia_slow:
 	@echo "Running full Gaia simulation. This may take awhile!"
 	@go test ./cmd/gaia/app -run TestFullGaiaSimulation -SimulationEnabled=true -SimulationNumBlocks=1000 -SimulationVerbose=true -v -timeout 24h
 
+SIM_NUM_BLOCKS ?= 210
+SIM_BLOCK_SIZE ?= 200
+SIM_COMMIT ?= true
+test_sim_gaia_benchmark:
+	@echo "Running Gaia benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
+	@go test -benchmem -run=^$$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$$  -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h
+
+test_sim_gaia_profile:
+	@echo "Running Gaia benchmark for numBlocks=$(SIM_NUM_BLOCKS), blockSize=$(SIM_BLOCK_SIZE). This may take awhile!"
+	@go test -benchmem -run=^$$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$$ -SimulationEnabled=true -SimulationNumBlocks=$(SIM_NUM_BLOCKS) -SimulationBlockSize=$(SIM_BLOCK_SIZE) -SimulationCommit=$(SIM_COMMIT) -timeout 24h -cpuprofile cpu.out -memprofile mem.out
+
 test_cover:
 	@bash tests/test_cover.sh
 

PENDING.md

@@ -60,6 +60,7 @@ FEATURES
 * SDK
   * [querier] added custom querier functionality, so ABCI query requests can be handled by keepers
   * [simulation] \#1924 allow operations to specify future operations
+  * [simulation] \#1924 Add benchmarking capabilities, with makefile commands "test_sim_gaia_benchmark, test_sim_gaia_profile"
 
 * Tendermint
 
@@ -79,7 +80,7 @@ IMPROVEMENTS
     * [x/stake] [#2023](https://github.com/cosmos/cosmos-sdk/pull/2023) Terminate iteration loop in `UpdateBondedValidators` and `UpdateBondedValidatorsFull` when the first revoked validator is encountered and perform a sanity check.
     * [x/auth] Signature verification's gas cost now accounts for pubkey type. [#2046](https://github.com/tendermint/tendermint/pull/2046)
     * [x/stake] [x/slashing] Ensure delegation invariants to jailed validators [#1883](https://github.com/cosmos/cosmos-sdk/issues/1883).
-
+    * [x/stake] Improve speed of GetValidator, which was shown to be a performance bottleneck. [#2046](https://github.com/tendermint/tendermint/pull/2200)
 * SDK
     * [tools] Make get_vendor_deps deletes `.vendor-new` directories, in case scratch files are present.
     * [cli] \#1632 Add integration tests to ensure `basecoind init && basecoind` start sequences run successfully for both `democoin` and `basecoin` examples.

cmd/gaia/app/sim_test.go

@@ -3,7 +3,9 @@ package app
 import (
 	"encoding/json"
 	"flag"
+	"fmt"
 	"math/rand"
+	"os"
 	"testing"
 
 	"github.com/stretchr/testify/require"
@@ -15,6 +17,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/baseapp"
 	sdk "github.com/cosmos/cosmos-sdk/types"
 	banksim "github.com/cosmos/cosmos-sdk/x/bank/simulation"
+	"github.com/cosmos/cosmos-sdk/x/gov"
 	govsim "github.com/cosmos/cosmos-sdk/x/gov/simulation"
 	"github.com/cosmos/cosmos-sdk/x/mock/simulation"
 	slashingsim "github.com/cosmos/cosmos-sdk/x/slashing/simulation"
@@ -28,6 +31,7 @@ var (
 	blockSize int
 	enabled   bool
 	verbose   bool
+	commit    bool
 )
 
 func init() {
@@ -36,6 +40,7 @@ func init() {
 	flag.IntVar(&blockSize, "SimulationBlockSize", 200, "Operations per block")
 	flag.BoolVar(&enabled, "SimulationEnabled", false, "Enable the simulation")
 	flag.BoolVar(&verbose, "SimulationVerbose", false, "Verbose log output")
+	flag.BoolVar(&commit, "SimulationCommit", false, "Have the simulation commit")
 }
 
 func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json.RawMessage {
@@ -49,7 +54,7 @@ func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json
 			Coins:   coins,
 		})
 	}
-
+	govGenesis := gov.DefaultGenesisState()
 	// Default genesis state
 	stakeGenesis := stake.DefaultGenesisState()
 	var validators []stake.Validator
@@ -73,6 +78,7 @@ func appStateFn(r *rand.Rand, keys []crypto.PrivKey, accs []sdk.AccAddress) json
 	genesis := GenesisState{
 		Accounts:  genesisAccounts,
 		StakeData: stakeGenesis,
+		GovData:   govGenesis,
 	}
 
 	// Marshal genesis
@@ -112,6 +118,39 @@ func invariants(app *GaiaApp) []simulation.Invariant {
 	}
 }
 
+// Profile with:
+// /usr/local/go/bin/go test -benchmem -run=^$ github.com/cosmos/cosmos-sdk/cmd/gaia/app -bench ^BenchmarkFullGaiaSimulation$ -SimulationCommit=true -cpuprofile cpu.out
+func BenchmarkFullGaiaSimulation(b *testing.B) {
+	// Setup Gaia application
+	var logger log.Logger
+	logger = log.NewNopLogger()
+	var db dbm.DB
+	dir := os.TempDir()
+	db, _ = dbm.NewGoLevelDB("Simulation", dir)
+	defer func() {
+		db.Close()
+		os.RemoveAll(dir)
+	}()
+	app := NewGaiaApp(logger, db, nil)
+
+	// Run randomized simulation
+	// TODO parameterize numbers, save for a later PR
+	simulation.SimulateFromSeed(
+		b, app.BaseApp, appStateFn, seed,
+		testAndRunTxs(app),
+		[]simulation.RandSetup{},
+		invariants(app), // these shouldn't get ran
+		numBlocks,
+		blockSize,
+		commit,
+	)
+	if commit {
+		fmt.Println("GoLevelDB Stats")
+		fmt.Println(db.Stats()["leveldb.stats"])
+		fmt.Println("GoLevelDB cached block size", db.Stats()["leveldb.cachedblock"])
+	}
+}
+
 func TestFullGaiaSimulation(t *testing.T) {
 	if !enabled {
 		t.Skip("Skipping Gaia simulation")
@@ -136,9 +175,11 @@ func TestFullGaiaSimulation(t *testing.T) {
 		invariants(app),
 		numBlocks,
 		blockSize,
-		false,
+		commit,
 	)
-
+	if commit {
+		fmt.Println("Database Size", db.Stats()["database.size"])
+	}
 }
 
 // TODO: Make another test for the fuzzer itself, which just has noOp txs

x/bank/simulation/msgs.go

@@ -21,7 +21,7 @@ import (
 // SimulateSingleInputMsgSend tests and runs a single msg send, with one input and one output, where both
 // accounts already exist.
 func SimulateSingleInputMsgSend(mapper auth.AccountMapper) simulation.Operation {
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) {
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) {
 		fromKey := simulation.RandomKey(r, keys)
 		fromAddr := sdk.AccAddress(fromKey.PubKey().Address())
 		toKey := simulation.RandomKey(r, keys)
@@ -58,15 +58,15 @@ func SimulateSingleInputMsgSend(mapper auth.AccountMapper) simulation.Operation
 			Inputs:  []bank.Input{bank.NewInput(fromAddr, coins)},
 			Outputs: []bank.Output{bank.NewOutput(toAddr, coins)},
 		}
-		sendAndVerifyMsgSend(t, app, mapper, msg, ctx, log, []crypto.PrivKey{fromKey})
+		sendAndVerifyMsgSend(tb, app, mapper, msg, ctx, log, []crypto.PrivKey{fromKey})
 		event("bank/sendAndVerifyMsgSend/ok")
 
 		return action, nil, nil
 	}
 }
 
 // Sends and verifies the transition of a msg send. This fails if there are repeated inputs or outputs
-func sendAndVerifyMsgSend(t *testing.T, app *baseapp.BaseApp, mapper auth.AccountMapper, msg bank.MsgSend, ctx sdk.Context, log string, privkeys []crypto.PrivKey) {
+func sendAndVerifyMsgSend(tb testing.TB, app *baseapp.BaseApp, mapper auth.AccountMapper, msg bank.MsgSend, ctx sdk.Context, log string, privkeys []crypto.PrivKey) {
 	initialInputAddrCoins := make([]sdk.Coins, len(msg.Inputs))
 	initialOutputAddrCoins := make([]sdk.Coins, len(msg.Outputs))
 	AccountNumbers := make([]int64, len(msg.Inputs))
@@ -91,24 +91,22 @@ func sendAndVerifyMsgSend(t *testing.T, app *baseapp.BaseApp, mapper auth.Accoun
 		// TODO: Do this in a more 'canonical' way
 		fmt.Println(res)
 		fmt.Println(log)
-		t.FailNow()
+		tb.FailNow()
 	}
 
 	for i := 0; i < len(msg.Inputs); i++ {
 		terminalInputCoins := mapper.GetAccount(ctx, msg.Inputs[i].Address).GetCoins()
-		require.Equal(t,
+		require.Equal(tb,
 			initialInputAddrCoins[i].Minus(msg.Inputs[i].Coins),
 			terminalInputCoins,
 			fmt.Sprintf("Input #%d had an incorrect amount of coins\n%s", i, log),
 		)
 	}
 	for i := 0; i < len(msg.Outputs); i++ {
 		terminalOutputCoins := mapper.GetAccount(ctx, msg.Outputs[i].Address).GetCoins()
-		require.Equal(t,
-			initialOutputAddrCoins[i].Plus(msg.Outputs[i].Coins),
-			terminalOutputCoins,
-			fmt.Sprintf("Output #%d had an incorrect amount of coins\n%s", i, log),
-		)
+		if !terminalOutputCoins.IsEqual(initialOutputAddrCoins[i].Plus(msg.Outputs[i].Coins)) {
+			tb.Fatalf("Output #%d had an incorrect amount of coins\n%s", i, log)
+		}
 	}
 }
 

x/gov/simulation/msgs.go

@@ -5,8 +5,6 @@ import (
 	"math/rand"
 	"testing"
 
-	"github.com/stretchr/testify/require"
-
 	"github.com/tendermint/tendermint/crypto"
 
 	"github.com/cosmos/cosmos-sdk/baseapp"
@@ -23,24 +21,15 @@ const (
 // SimulateMsgSubmitProposal simulates a msg Submit Proposal
 // Note: Currently doesn't ensure that the proposal txt is in JSON form
 func SimulateMsgSubmitProposal(k gov.Keeper, sk stake.Keeper) simulation.Operation {
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) {
-		key := simulation.RandomKey(r, keys)
-		addr := sdk.AccAddress(key.PubKey().Address())
-		deposit := randomDeposit(r)
-		msg := gov.NewMsgSubmitProposal(
-			simulation.RandStringOfLength(r, 5),
-			simulation.RandStringOfLength(r, 5),
-			gov.ProposalTypeText,
-			addr,
-			deposit,
-		)
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+	handler := gov.NewHandler(k)
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOps []simulation.FutureOperation, err sdk.Error) {
+		msg := simulationCreateMsgSubmitProposal(tb, r, keys, log)
 		ctx, write := ctx.CacheContext()
-		result := gov.NewHandler(k)(ctx, msg)
+		result := handler(ctx, msg)
 		if result.IsOK() {
 			// Update pool to keep invariants
 			pool := sk.GetPool(ctx)
-			pool.LooseTokens = pool.LooseTokens.Sub(sdk.NewDecFromInt(deposit.AmountOf(denom)))
+			pool.LooseTokens = pool.LooseTokens.Sub(sdk.NewDecFromInt(msg.InitialDeposit.AmountOf(denom)))
 			sk.SetPool(ctx, pool)
 			write()
 		}
@@ -50,9 +39,26 @@ func SimulateMsgSubmitProposal(k gov.Keeper, sk stake.Keeper) simulation.Operati
 	}
 }
 
+func simulationCreateMsgSubmitProposal(tb testing.TB, r *rand.Rand, keys []crypto.PrivKey, log string) gov.MsgSubmitProposal {
+	key := simulation.RandomKey(r, keys)
+	addr := sdk.AccAddress(key.PubKey().Address())
+	deposit := randomDeposit(r)
+	msg := gov.NewMsgSubmitProposal(
+		simulation.RandStringOfLength(r, 5),
+		simulation.RandStringOfLength(r, 5),
+		gov.ProposalTypeText,
+		addr,
+		deposit,
+	)
+	if msg.ValidateBasic() != nil {
+		tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+	}
+	return msg
+}
+
 // SimulateMsgDeposit
 func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation {
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
 		key := simulation.RandomKey(r, keys)
 		addr := sdk.AccAddress(key.PubKey().Address())
 		proposalID, ok := randomProposalID(r, k, ctx)
@@ -61,7 +67,9 @@ func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation {
 		}
 		deposit := randomDeposit(r)
 		msg := gov.NewMsgDeposit(addr, proposalID, deposit)
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+		if msg.ValidateBasic() != nil {
+			tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+		}
 		ctx, write := ctx.CacheContext()
 		result := gov.NewHandler(k)(ctx, msg)
 		if result.IsOK() {
@@ -79,7 +87,7 @@ func SimulateMsgDeposit(k gov.Keeper, sk stake.Keeper) simulation.Operation {
 
 // SimulateMsgVote
 func SimulateMsgVote(k gov.Keeper, sk stake.Keeper) simulation.Operation {
-	return func(t *testing.T, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
+	return func(tb testing.TB, r *rand.Rand, app *baseapp.BaseApp, ctx sdk.Context, keys []crypto.PrivKey, log string, event func(string)) (action string, fOp []simulation.FutureOperation, err sdk.Error) {
 		key := simulation.RandomKey(r, keys)
 		addr := sdk.AccAddress(key.PubKey().Address())
 		proposalID, ok := randomProposalID(r, k, ctx)
@@ -88,7 +96,9 @@ func SimulateMsgVote(k gov.Keeper, sk stake.Keeper) simulation.Operation {
 		}
 		option := randomVotingOption(r)
 		msg := gov.NewMsgVote(addr, proposalID, option)
-		require.Nil(t, msg.ValidateBasic(), "expected msg to pass ValidateBasic: %s", msg.GetSignBytes())
+		if msg.ValidateBasic() != nil {
+			tb.Fatalf("expected msg to pass ValidateBasic: %s, log %s", msg.GetSignBytes(), log)
+		}
 		ctx, write := ctx.CacheContext()
 		result := gov.NewHandler(k)(ctx, msg)
 		if result.IsOK() {