Tendermint Block Pruning

CHANGELOG.md

@@ -162,6 +162,7 @@ be used to retrieve the actual proposal `Content`. Also the `NewMsgSubmitProposa
 
 ### Features
 
+* [\#7265](https://github.com/cosmos/cosmos-sdk/pull/7265) Support Tendermint block pruning through a new `min-retain-blocks` configuration that can be set in either `app.toml` or via the CLI. This parameter is used in conjunction with other criteria to determine the height at which Tendermint should prune blocks.
 * (vesting) [\#7209](https://github.com/cosmos/cosmos-sdk/pull/7209) Create new `MsgCreateVestingAccount` message type along with CLI handler that allows for the creation of delayed and continuous vesting types.
 * (events) [\#7121](https://github.com/cosmos/cosmos-sdk/pull/7121) The application now drives what events are indexed by Tendermint via the `index-events` configuration in `app.toml`, which is a list of events taking the form `{eventType}.{attributeKey}`.
 * [\#6089](https://github.com/cosmos/cosmos-sdk/pull/6089) Transactions can now have a `TimeoutHeight` set which allows the transaction to be rejected if it's committed at a height greater than the timeout.

baseapp/abci.go

@@ -294,6 +294,7 @@ func (app *BaseApp) Commit() (res abci.ResponseCommit) {
 	defer telemetry.MeasureSince(time.Now(), "abci", "commit")
 
 	header := app.deliverState.ctx.BlockHeader()
+	retainHeight := app.GetBlockRetentionHeight(header.Height)
 
 	// Write the DeliverTx state which is cache-wrapped and commit the MultiStore.
 	// The write to the DeliverTx state writes all state transitions to the root
@@ -334,7 +335,8 @@ func (app *BaseApp) Commit() (res abci.ResponseCommit) {
 	}
 
 	return abci.ResponseCommit{
-		Data: commitID.Hash,
+		Data:         commitID.Hash,
+		RetainHeight: retainHeight,
 	}
 }
 
@@ -578,6 +580,93 @@ func (app *BaseApp) createQueryContext(height int64, prove bool) (sdk.Context, e
 	return ctx, nil
 }
 
+// GetBlockRetentionHeight returns the height for which all blocks below this height
+// are pruned from Tendermint. Given a commitment height and a non-zero local
+// minRetainBlocks configuration, the retentionHeight is the smallest height that
+// satisfies:
+//
+// - Unbonding (safety threshold) time: The block interval in which validators
+// can be economically punished for misbehavior. Blocks in this interval must be
+// auditable e.g. by the light client.
+//
+// - Logical store snapshot interval: The block interval at which the underlying
+// logical store database is persisted to disk, e.g. every 10000 heights. Blocks
+// since the last IAVL snapshot must be available for replay on application restart.
+//
+// - State sync snapshots: Blocks since the oldest available snapshot must be
+// available for state sync nodes to catch up (oldest because a node may be
+// restoring an old snapshot while a new snapshot was taken).
+//
+// - Local (minRetainBlocks) config: Archive nodes may want to retain more or
+// all blocks, e.g. via a local config option min-retain-blocks. There may also
+// be a need to vary retention for other nodes, e.g. sentry nodes which do not
+// need historical blocks.
+func (app *BaseApp) GetBlockRetentionHeight(commitHeight int64) int64 {
+	// pruning is disabled if minRetainBlocks is zero
+	if app.minRetainBlocks == 0 {
+		return 0
+	}
+
+	minNonZero := func(x, y int64) int64 {
+		switch {
+		case x == 0:
+			return y
+		case y == 0:
+			return x
+		case x < y:
+			return x
+		default:
+			return y
+		}
+	}
+
+	// Define retentionHeight as the minimum value that satisfies all non-zero
+	// constraints. All blocks below (commitHeight-retentionHeight) are pruned
+	// from Tendermint.
+	var retentionHeight int64
+
+	// Define the number of blocks needed to protect against misbehaving validators
+	// which allows light clients to operate safely. Note, we piggy back of the
+	// evidence parameters instead of computing an estimated nubmer of blocks based
+	// on the unbonding period and block commitment time as the two should be
+	// equivalent.
+	cp := app.GetConsensusParams(app.deliverState.ctx)
+	if cp != nil && cp.Evidence != nil && cp.Evidence.MaxAgeNumBlocks > 0 {
+		retentionHeight = commitHeight - cp.Evidence.MaxAgeNumBlocks
+	}
+
+	// Define the state pruning offset, i.e. the block offset at which the
+	// underlying logical database is persisted to disk.
+	statePruningOffset := int64(app.cms.GetPruning().KeepEvery)
+	if statePruningOffset > 0 {
+		if commitHeight > statePruningOffset {
+			v := commitHeight - (commitHeight % statePruningOffset)
+			retentionHeight = minNonZero(retentionHeight, v)
+		} else {
+			// Hitting this case means we have persisting enabled but have yet to reach
+			// a height in which we persist state, so we return zero regardless of other
+			// conditions. Otherwise, we could end up pruning blocks without having
+			// any state committed to disk.
+			return 0
+		}
+	}
+
+	if app.snapshotInterval > 0 && app.snapshotKeepRecent > 0 {
+		v := commitHeight - int64((app.snapshotInterval * uint64(app.snapshotKeepRecent)))
+		retentionHeight = minNonZero(retentionHeight, v)
+	}
+
+	v := commitHeight - int64(app.minRetainBlocks)
+	retentionHeight = minNonZero(retentionHeight, v)
+
+	if retentionHeight <= 0 {
+		// prune nothing in the case of a non-positive height
+		return 0
+	}
+
+	return retentionHeight
+}
+
 func handleQueryApp(app *BaseApp, path []string, req abci.RequestQuery) abci.ResponseQuery {
 	if len(path) >= 2 {
 		switch path[1] {

baseapp/abci_test.go

@@ -0,0 +1,118 @@
+package baseapp
+
+import (
+	"testing"
+
+	"github.com/stretchr/testify/require"
+	abci "github.com/tendermint/tendermint/abci/types"
+	tmprototypes "github.com/tendermint/tendermint/proto/tendermint/types"
+	dbm "github.com/tendermint/tm-db"
+
+	sdk "github.com/cosmos/cosmos-sdk/types"
+)
+
+func TestGetBlockRentionHeight(t *testing.T) {
+	logger := defaultLogger()
+	db := dbm.NewMemDB()
+	name := t.Name()
+
+	testCases := map[string]struct {
+		bapp         *BaseApp
+		maxAgeBlocks int64
+		commitHeight int64
+		expected     int64
+	}{
+		"defaults": {
+			bapp:         NewBaseApp(name, logger, db, nil),
+			maxAgeBlocks: 0,
+			commitHeight: 499000,
+			expected:     0,
+		},
+		"pruning unbonding time only": {
+			bapp:         NewBaseApp(name, logger, db, nil, SetMinRetainBlocks(1)),
+			maxAgeBlocks: 362880,
+			commitHeight: 499000,
+			expected:     136120,
+		},
+		"pruning iavl snapshot only": {
+			bapp: NewBaseApp(
+				name, logger, db, nil,
+				SetPruning(sdk.PruningOptions{KeepEvery: 10000}),
+				SetMinRetainBlocks(1),
+			),
+			maxAgeBlocks: 0,
+			commitHeight: 499000,
+			expected:     490000,
+		},
+		"pruning state sync snapshot only": {
+			bapp: NewBaseApp(
+				name, logger, db, nil,
+				SetSnapshotInterval(50000),
+				SetSnapshotKeepRecent(3),
+				SetMinRetainBlocks(1),
+			),
+			maxAgeBlocks: 0,
+			commitHeight: 499000,
+			expected:     349000,
+		},
+		"pruning min retention only": {
+			bapp: NewBaseApp(
+				name, logger, db, nil,
+				SetMinRetainBlocks(400000),
+			),
+			maxAgeBlocks: 0,
+			commitHeight: 499000,
+			expected:     99000,
+		},
+		"pruning all conditions": {
+			bapp: NewBaseApp(
+				name, logger, db, nil,
+				SetPruning(sdk.PruningOptions{KeepEvery: 10000}),
+				SetMinRetainBlocks(400000),
+				SetSnapshotInterval(50000), SetSnapshotKeepRecent(3),
+			),
+			maxAgeBlocks: 362880,
+			commitHeight: 499000,
+			expected:     99000,
+		},
+		"no pruning due to no persisted state": {
+			bapp: NewBaseApp(
+				name, logger, db, nil,
+				SetPruning(sdk.PruningOptions{KeepEvery: 10000}),
+				SetMinRetainBlocks(400000),
+				SetSnapshotInterval(50000), SetSnapshotKeepRecent(3),
+			),
+			maxAgeBlocks: 362880,
+			commitHeight: 10000,
+			expected:     0,
+		},
+		"disable pruning": {
+			bapp: NewBaseApp(
+				name, logger, db, nil,
+				SetPruning(sdk.PruningOptions{KeepEvery: 10000}),
+				SetMinRetainBlocks(0),
+				SetSnapshotInterval(50000), SetSnapshotKeepRecent(3),
+			),
+			maxAgeBlocks: 362880,
+			commitHeight: 499000,
+			expected:     0,
+		},
+	}
+
+	for name, tc := range testCases {
+		tc := tc
+
+		tc.bapp.SetParamStore(&paramStore{db: dbm.NewMemDB()})
+		tc.bapp.InitChain(abci.RequestInitChain{
+			ConsensusParams: &abci.ConsensusParams{
+				Evidence: &tmprototypes.EvidenceParams{
+					MaxAgeNumBlocks: tc.maxAgeBlocks,
+				},
+			},
+		})
+
+		t.Run(name, func(t *testing.T) {
+			require.Equal(t, tc.expected, tc.bapp.GetBlockRetentionHeight(tc.commitHeight))
+		})
+	}
+}

baseapp/baseapp.go

@@ -101,6 +101,18 @@ type BaseApp struct { // nolint: maligned
 	// minimum block time (in Unix seconds) at which to halt the chain and gracefully shutdown
 	haltTime uint64
 
+	// minRetainBlocks defines the minimum block height offset from the current
+	// block being committed, such that all blocks past this offset are pruned
+	// from Tendermint. It is used as part of the process of determining the
+	// ResponseCommit.RetainHeight value during ABCI Commit. A value of 0 indicates
+	// that no blocks should be pruned.
+	//
+	// Note: Tendermint block pruning is dependant on this parameter in conunction
+	// with the unbonding (safety threshold) period, state pruning and state sync
+	// snapshot parameters to determine the correct minimum value of
+	// ResponseCommit.RetainHeight.
+	minRetainBlocks uint64
+
 	// application's version string
 	appVersion string
 
@@ -298,6 +310,10 @@ func (app *BaseApp) setHaltTime(haltTime uint64) {
 	app.haltTime = haltTime
 }
 
+func (app *BaseApp) setMinRetainBlocks(minRetainBlocks uint64) {
+	app.minRetainBlocks = minRetainBlocks
+}
+
 func (app *BaseApp) setInterBlockCache(cache sdk.MultiStorePersistentCache) {
 	app.interBlockCache = cache
 }

baseapp/options.go

@@ -39,6 +39,13 @@ func SetHaltTime(haltTime uint64) func(*BaseApp) {
 	return func(bap *BaseApp) { bap.setHaltTime(haltTime) }
 }
 
+// SetMinRetainBlocks returns a BaseApp option function that sets the minimum
+// block retention height value when determining which heights to prune during
+// ABCI Commit.
+func SetMinRetainBlocks(minRetainBlocks uint64) func(*BaseApp) {
+	return func(bapp *BaseApp) { bapp.setMinRetainBlocks(minRetainBlocks) }
+}
+
 // SetTrace will turn on or off trace flag
 func SetTrace(trace bool) func(*BaseApp) {
 	return func(app *BaseApp) { app.setTrace(trace) }

server/config/config.go

@@ -43,6 +43,22 @@ type BaseConfig struct {
 	// Note: Commitment of state will be attempted on the corresponding block.
 	HaltTime uint64 `mapstructure:"halt-time"`
 
+	// MinRetainBlocks defines the minimum block height offset from the current
+	// block being committed, such that blocks past this offset may be pruned
+	// from Tendermint. It is used as part of the process of determining the
+	// ResponseCommit.RetainHeight value during ABCI Commit. A value of 0 indicates
+	// that no blocks should be pruned.
+	//
+	// This configuration value is only responsible for pruning Tendermint blocks.
+	// It has no bearing on application state pruning which is determined by the
+	// "pruning-*" configurations.
+	//
+	// Note: Tendermint block pruning is dependant on this parameter in conunction
+	// with the unbonding (safety threshold) period, state pruning and state sync
+	// snapshot parameters to determine the correct minimum value of
+	// ResponseCommit.RetainHeight.
+	MinRetainBlocks uint64 `mapstructure:"min-retain-blocks"`
+
 	// InterBlockCache enables inter-block caching.
 	InterBlockCache bool `mapstructure:"inter-block-cache"`
 
@@ -150,6 +166,7 @@ func DefaultConfig() *Config {
 			PruningKeepRecent: "0",
 			PruningKeepEvery:  "0",
 			PruningInterval:   "0",
+			MinRetainBlocks:   0,
 			IndexEvents:       make([]string, 0),
 		},
 		Telemetry: telemetry.Config{
@@ -197,6 +214,7 @@ func GetConfig(v *viper.Viper) Config {
 			HaltHeight:        v.GetUint64("halt-height"),
 			HaltTime:          v.GetUint64("halt-time"),
 			IndexEvents:       v.GetStringSlice("index-events"),
+			MinRetainBlocks:   v.GetUint64("min-retain-blocks"),
 		},
 		Telemetry: telemetry.Config{
 			ServiceName:             v.GetString("telemetry.service-name"),

server/config/toml.go

@@ -44,6 +44,22 @@ halt-height = {{ .BaseConfig.HaltHeight }}
 # Note: Commitment of state will be attempted on the corresponding block.
 halt-time = {{ .BaseConfig.HaltTime }}
 
+# MinRetainBlocks defines the minimum block height offset from the current
+# block being committed, such that all blocks past this offset are pruned
+# from Tendermint. It is used as part of the process of determining the
+# ResponseCommit.RetainHeight value during ABCI Commit. A value of 0 indicates
+# that no blocks should be pruned.
+#
+# This configuration value is only responsible for pruning Tendermint blocks.
+# It has no bearing on application state pruning which is determined by the
+# "pruning-*" configurations.
+#
+# Note: Tendermint block pruning is dependant on this parameter in conunction
+# with the unbonding (safety threshold) period, state pruning and state sync
+# snapshot parameters to determine the correct minimum value of
+# ResponseCommit.RetainHeight.
+min-retain-blocks = {{ .BaseConfig.MinRetainBlocks }}
+
 # InterBlockCache enables inter-block caching.
 inter-block-cache = {{ .BaseConfig.InterBlockCache }}
 

server/mock/store.go

@@ -55,6 +55,10 @@ func (ms multiStore) SetPruning(opts sdk.PruningOptions) {
 	panic("not implemented")
 }
 
+func (ms multiStore) GetPruning() sdk.PruningOptions {
+	panic("not implemented")
+}
+
 func (ms multiStore) GetCommitKVStore(key sdk.StoreKey) sdk.CommitKVStore {
 	panic("not implemented")
 }

server/start.go

@@ -48,6 +48,7 @@ const (
 	FlagPruningKeepEvery  = "pruning-keep-every"
 	FlagPruningInterval   = "pruning-interval"
 	FlagIndexEvents       = "index-events"
+	FlagMinRetainBlocks   = "min-retain-blocks"
 )
 
 // GRPC-related flags.
@@ -135,6 +136,7 @@ which accepts a path for the resulting pprof file.
 	cmd.Flags().Uint64(FlagPruningKeepEvery, 0, "Offset heights to keep on disk after 'keep-every' (ignored if pruning is not 'custom')")
 	cmd.Flags().Uint64(FlagPruningInterval, 0, "Height interval at which pruned heights are removed from disk (ignored if pruning is not 'custom')")
 	cmd.Flags().Uint(FlagInvCheckPeriod, 0, "Assert registered invariants every N blocks")
+	cmd.Flags().Uint64(FlagMinRetainBlocks, 0, "Minimum block height offset during ABCI commit to prune Tendermint blocks")
 
 	cmd.Flags().Bool(flagGRPCEnable, true, "Define if the gRPC server should be enabled")
 	cmd.Flags().String(flagGRPCAddress, config.DefaultGRPCAddress, "the gRPC server address to listen on")

simapp/simd/cmd/root.go

@@ -196,6 +196,7 @@ func newApp(logger log.Logger, db dbm.DB, traceStore io.Writer, appOpts serverty
 		baseapp.SetMinGasPrices(cast.ToString(appOpts.Get(server.FlagMinGasPrices))),
 		baseapp.SetHaltHeight(cast.ToUint64(appOpts.Get(server.FlagHaltHeight))),
 		baseapp.SetHaltTime(cast.ToUint64(appOpts.Get(server.FlagHaltTime))),
+		baseapp.SetMinRetainBlocks(cast.ToUint64(appOpts.Get(server.FlagMinRetainBlocks))),
 		baseapp.SetInterBlockCache(cache),
 		baseapp.SetTrace(cast.ToBool(appOpts.Get(server.FlagTrace))),
 		baseapp.SetIndexEvents(cast.ToStringSlice(appOpts.Get(server.FlagIndexEvents))),

store/iavl/store.go

@@ -125,6 +125,12 @@ func (st *Store) SetPruning(_ types.PruningOptions) {
 	panic("cannot set pruning options on an initialized IAVL store")
 }
 
+// SetPruning panics as pruning options should be provided at initialization
+// since IAVl accepts pruning options directly.
+func (st *Store) GetPruning() types.PruningOptions {
+	panic("cannot get pruning options on an initialized IAVL store")
+}
+
 // VersionExists returns whether or not a given version is stored.
 func (st *Store) VersionExists(version int64) bool {
 	return st.tree.VersionExists(version)