R4R: Outstanding per-validator rewards; correctly handle same-BeginBlock redelegation-double-slash

PENDING.md

@@ -61,6 +61,12 @@ CLI flag.
 
 ### SDK
 
+* \#3750 Track outstanding rewards per-validator instead of globally,
+         and fix the main simulation issue, which was that slashes of
+         re-delegations to a validator were not correctly accounted for
+         in fee distribution when the redelegation in question had itself
+          been slashed (from a fault committed by a different validator)
+         in the same BeginBlock
 * \#3753 Remove no-longer-used governance penalty parameter
 * \#3679 Consistent operators across Coins, DecCoins, Int, Dec
           replaced: Minus->Sub Plus->Add Div->Quo

client/lcd/lcd_test.go

@@ -945,7 +945,7 @@ func TestDistributionFlow(t *testing.T) {
 	operAddr := sdk.AccAddress(valAddr)
 
 	var rewards sdk.DecCoins
-	res, body := Request(t, port, "GET", fmt.Sprintf("/distribution/outstanding_rewards"), nil)
+	res, body := Request(t, port, "GET", fmt.Sprintf("/distribution/validators/%s/outstanding_rewards", valAddr), nil)
 	require.Equal(t, http.StatusOK, res.StatusCode, body)
 	require.NoError(t, cdc.UnmarshalJSON([]byte(body), &rewards))
 
@@ -967,7 +967,7 @@ func TestDistributionFlow(t *testing.T) {
 	require.Equal(t, uint32(0), resultTx.Code)
 
 	// Query outstanding rewards changed
-	res, body = Request(t, port, "GET", fmt.Sprintf("/distribution/outstanding_rewards"), nil)
+	res, body = Request(t, port, "GET", fmt.Sprintf("/distribution/validators/%s/outstanding_rewards", valAddr), nil)
 	require.Equal(t, http.StatusOK, res.StatusCode, body)
 	require.NoError(t, cdc.UnmarshalJSON([]byte(body), &rewards))
 

client/lcd/swagger-ui/swagger.yaml

@@ -1558,6 +1558,28 @@ paths:
           description: Invalid validator address
         500:
           description: Internal Server Error
+  /distribution/validators/{validatorAddr}/outstanding_rewards:
+    parameters:
+      - in: path
+        name: validatorAddr
+        description: Bech32 OperatorAddress of validator
+        required: true
+        type: string
+    get:
+     summary: Fee distribution outstanding rewards of a single validator
+     tags:
+       - ICS24
+     produces:
+       - application/json
+     responses:
+       200:
+         description: OK
+         schema:
+           type: array
+           items:
+             $ref: "#/definitions/Coin"
+       500:
+         description: Internal Server Error
   /distribution/validators/{validatorAddr}/rewards:
     parameters:
       - in: path
@@ -1566,8 +1588,8 @@ paths:
         required: true
         type: string
     get:
-      summary: Commission and self-delegation rewards of a single a validator
-      description: Query the commission and self-delegation rewards of a validator.
+      summary: Commission and self-delegation rewards of a single validator
+      description: Query the commission and self-delegation rewards of validator.
       tags:
         - ICS24
       produces:
@@ -1630,22 +1652,6 @@ paths:
                 type: string
         500:
           description: Internal Server Error
-  /distribution/outstanding_rewards:
-    get:
-      summary: Fee distribution outstanding rewards
-      tags:
-        - ICS24
-      produces:
-        - application/json
-      responses:
-        200:
-          description: OK
-          schema:
-            type: array
-            items:
-              $ref: "#/definitions/Coin"
-        500:
-          description: Internal Server Error
 definitions:
   CheckTxResult:
     type: object

types/decimal.go

@@ -291,6 +291,21 @@ func (d Dec) QuoTruncate(d2 Dec) Dec {
 	return Dec{chopped}
 }
 
+// quotient, round up
+func (d Dec) QuoRoundUp(d2 Dec) Dec {
+	// multiply precision twice
+	mul := new(big.Int).Mul(d.Int, precisionReuse)
+	mul.Mul(mul, precisionReuse)
+
+	quo := new(big.Int).Quo(mul, d2.Int)
+	chopped := chopPrecisionAndRoundUp(quo)
+
+	if chopped.BitLen() > 255+DecimalPrecisionBits {
+		panic("Int overflow")
+	}
+	return Dec{chopped}
+}
+
 // quotient
 func (d Dec) QuoInt(i Int) Dec {
 	mul := new(big.Int).Quo(d.Int, i.i)
@@ -412,6 +427,29 @@ func chopPrecisionAndRound(d *big.Int) *big.Int {
 	}
 }
 
+func chopPrecisionAndRoundUp(d *big.Int) *big.Int {
+
+	// remove the negative and add it back when returning
+	if d.Sign() == -1 {
+		// make d positive, compute chopped value, and then un-mutate d
+		d = d.Neg(d)
+		// truncate since d is negative...
+		d = chopPrecisionAndTruncate(d)
+		d = d.Neg(d)
+		return d
+	}
+
+	// get the truncated quotient and remainder
+	quo, rem := d, big.NewInt(0)
+	quo, rem = quo.QuoRem(d, precisionReuse, rem)
+
+	if rem.Sign() == 0 { // remainder is zero
+		return quo
+	}
+
+	return quo.Add(quo, oneInt)
+}
+
 func chopPrecisionAndRoundNonMutative(d *big.Int) *big.Int {
 	tmp := new(big.Int).Set(d)
 	return chopPrecisionAndRound(tmp)

types/decimal_test.go

@@ -156,44 +156,61 @@ func TestDecsEqual(t *testing.T) {
 
 func TestArithmetic(t *testing.T) {
 	tests := []struct {
-		d1, d2                         Dec
-		expMul, expQuo, expAdd, expSub Dec
+		d1, d2                                Dec
+		expMul, expMulTruncate                Dec
+		expQuo, expQuoRoundUp, expQuoTruncate Dec
+		expAdd, expSub                        Dec
 	}{
-		// d1       d2         MUL        DIV        ADD        SUB
-		{NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0)},
-		{NewDec(1), NewDec(0), NewDec(0), NewDec(0), NewDec(1), NewDec(1)},
-		{NewDec(0), NewDec(1), NewDec(0), NewDec(0), NewDec(1), NewDec(-1)},
-		{NewDec(0), NewDec(-1), NewDec(0), NewDec(0), NewDec(-1), NewDec(1)},
-		{NewDec(-1), NewDec(0), NewDec(0), NewDec(0), NewDec(-1), NewDec(-1)},
-
-		{NewDec(1), NewDec(1), NewDec(1), NewDec(1), NewDec(2), NewDec(0)},
-		{NewDec(-1), NewDec(-1), NewDec(1), NewDec(1), NewDec(-2), NewDec(0)},
-		{NewDec(1), NewDec(-1), NewDec(-1), NewDec(-1), NewDec(0), NewDec(2)},
-		{NewDec(-1), NewDec(1), NewDec(-1), NewDec(-1), NewDec(0), NewDec(-2)},
-
-		{NewDec(3), NewDec(7), NewDec(21), NewDecWithPrec(428571428571428571, 18), NewDec(10), NewDec(-4)},
-		{NewDec(2), NewDec(4), NewDec(8), NewDecWithPrec(5, 1), NewDec(6), NewDec(-2)},
-		{NewDec(100), NewDec(100), NewDec(10000), NewDec(1), NewDec(200), NewDec(0)},
-
-		{NewDecWithPrec(15, 1), NewDecWithPrec(15, 1), NewDecWithPrec(225, 2),
-			NewDec(1), NewDec(3), NewDec(0)},
-		{NewDecWithPrec(3333, 4), NewDecWithPrec(333, 4), NewDecWithPrec(1109889, 8),
-			MustNewDecFromStr("10.009009009009009009"), NewDecWithPrec(3666, 4), NewDecWithPrec(3, 1)},
+		//  d1         d2         MUL    MulTruncate    QUO    QUORoundUp QUOTrunctate  ADD         SUB
+		{NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0)},
+		{NewDec(1), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(1), NewDec(1)},
+		{NewDec(0), NewDec(1), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(1), NewDec(-1)},
+		{NewDec(0), NewDec(-1), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(-1), NewDec(1)},
+		{NewDec(-1), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(0), NewDec(-1), NewDec(-1)},
+
+		{NewDec(1), NewDec(1), NewDec(1), NewDec(1), NewDec(1), NewDec(1), NewDec(1), NewDec(2), NewDec(0)},
+		{NewDec(-1), NewDec(-1), NewDec(1), NewDec(1), NewDec(1), NewDec(1), NewDec(1), NewDec(-2), NewDec(0)},
+		{NewDec(1), NewDec(-1), NewDec(-1), NewDec(-1), NewDec(-1), NewDec(-1), NewDec(-1), NewDec(0), NewDec(2)},
+		{NewDec(-1), NewDec(1), NewDec(-1), NewDec(-1), NewDec(-1), NewDec(-1), NewDec(-1), NewDec(0), NewDec(-2)},
+
+		{NewDec(3), NewDec(7), NewDec(21), NewDec(21),
+			NewDecWithPrec(428571428571428571, 18), NewDecWithPrec(428571428571428572, 18), NewDecWithPrec(428571428571428571, 18),
+			NewDec(10), NewDec(-4)},
+		{NewDec(2), NewDec(4), NewDec(8), NewDec(8), NewDecWithPrec(5, 1), NewDecWithPrec(5, 1), NewDecWithPrec(5, 1),
+			NewDec(6), NewDec(-2)},
+
+		{NewDec(100), NewDec(100), NewDec(10000), NewDec(10000), NewDec(1), NewDec(1), NewDec(1), NewDec(200), NewDec(0)},
+
+		{NewDecWithPrec(15, 1), NewDecWithPrec(15, 1), NewDecWithPrec(225, 2), NewDecWithPrec(225, 2),
+			NewDec(1), NewDec(1), NewDec(1), NewDec(3), NewDec(0)},
+		{NewDecWithPrec(3333, 4), NewDecWithPrec(333, 4), NewDecWithPrec(1109889, 8), NewDecWithPrec(1109889, 8),
+			MustNewDecFromStr("10.009009009009009009"), MustNewDecFromStr("10.009009009009009010"), MustNewDecFromStr("10.009009009009009009"),
+			NewDecWithPrec(3666, 4), NewDecWithPrec(3, 1)},
 	}
 
 	for tcIndex, tc := range tests {
 		resAdd := tc.d1.Add(tc.d2)
 		resSub := tc.d1.Sub(tc.d2)
 		resMul := tc.d1.Mul(tc.d2)
+		resMulTruncate := tc.d1.MulTruncate(tc.d2)
 		require.True(t, tc.expAdd.Equal(resAdd), "exp %v, res %v, tc %d", tc.expAdd, resAdd, tcIndex)
 		require.True(t, tc.expSub.Equal(resSub), "exp %v, res %v, tc %d", tc.expSub, resSub, tcIndex)
 		require.True(t, tc.expMul.Equal(resMul), "exp %v, res %v, tc %d", tc.expMul, resMul, tcIndex)
+		require.True(t, tc.expMulTruncate.Equal(resMulTruncate), "exp %v, res %v, tc %d", tc.expMulTruncate, resMulTruncate, tcIndex)
 
 		if tc.d2.IsZero() { // panic for divide by zero
 			require.Panics(t, func() { tc.d1.Quo(tc.d2) })
 		} else {
 			resQuo := tc.d1.Quo(tc.d2)
 			require.True(t, tc.expQuo.Equal(resQuo), "exp %v, res %v, tc %d", tc.expQuo.String(), resQuo.String(), tcIndex)
+
+			resQuoRoundUp := tc.d1.QuoRoundUp(tc.d2)
+			require.True(t, tc.expQuoRoundUp.Equal(resQuoRoundUp), "exp %v, res %v, tc %d",
+				tc.expQuoRoundUp.String(), resQuoRoundUp.String(), tcIndex)
+
+			resQuoTruncate := tc.d1.QuoTruncate(tc.d2)
+			require.True(t, tc.expQuoTruncate.Equal(resQuoTruncate), "exp %v, res %v, tc %d",
+				tc.expQuoTruncate.String(), resQuoTruncate.String(), tcIndex)
 		}
 	}
 }

types/staking.go

@@ -61,19 +61,20 @@ func (b BondStatus) Equal(b2 BondStatus) bool {
 
 // validator for a delegated proof of stake system
 type Validator interface {
-	GetJailed() bool              // whether the validator is jailed
-	GetMoniker() string           // moniker of the validator
-	GetStatus() BondStatus        // status of the validator
-	GetOperator() ValAddress      // operator address to receive/return validators coins
-	GetConsPubKey() crypto.PubKey // validation consensus pubkey
-	GetConsAddr() ConsAddress     // validation consensus address
-	GetTokens() Int               // validation tokens
-	GetBondedTokens() Int         // validator bonded tokens
-	GetTendermintPower() int64    // validation power in tendermint
-	GetCommission() Dec           // validator commission rate
-	GetMinSelfDelegation() Int    // validator minimum self delegation
-	GetDelegatorShares() Dec      // total outstanding delegator shares
-	GetDelegatorShareExRate() Dec // tokens per delegator share exchange rate
+	GetJailed() bool                       // whether the validator is jailed
+	GetMoniker() string                    // moniker of the validator
+	GetStatus() BondStatus                 // status of the validator
+	GetOperator() ValAddress               // operator address to receive/return validators coins
+	GetConsPubKey() crypto.PubKey          // validation consensus pubkey
+	GetConsAddr() ConsAddress              // validation consensus address
+	GetTokens() Int                        // validation tokens
+	GetBondedTokens() Int                  // validator bonded tokens
+	GetTendermintPower() int64             // validation power in tendermint
+	GetCommission() Dec                    // validator commission rate
+	GetMinSelfDelegation() Int             // validator minimum self delegation
+	GetDelegatorShares() Dec               // total outstanding delegator shares
+	GetDelegatorShareExRate() Dec          // tokens per delegator share exchange rate
+	GetDelegatorShareExRateTruncated() Dec // tokens per delegator share exchange rate
 }
 
 // validator which fulfills abci validator interface for use in Tendermint

x/distribution/alias.go

@@ -50,14 +50,15 @@ var (
 	NewMsgWithdrawDelegatorReward     = types.NewMsgWithdrawDelegatorReward
 	NewMsgWithdrawValidatorCommission = types.NewMsgWithdrawValidatorCommission
 
-	NewKeeper                           = keeper.NewKeeper
-	NewQuerier                          = keeper.NewQuerier
-	NewQueryValidatorCommissionParams   = keeper.NewQueryValidatorCommissionParams
-	NewQueryValidatorSlashesParams      = keeper.NewQueryValidatorSlashesParams
-	NewQueryDelegationRewardsParams     = keeper.NewQueryDelegationRewardsParams
-	NewQueryDelegatorParams             = keeper.NewQueryDelegatorParams
-	NewQueryDelegatorWithdrawAddrParams = keeper.NewQueryDelegatorWithdrawAddrParams
-	DefaultParamspace                   = keeper.DefaultParamspace
+	NewKeeper                                 = keeper.NewKeeper
+	NewQuerier                                = keeper.NewQuerier
+	NewQueryValidatorOutstandingRewardsParams = keeper.NewQueryValidatorOutstandingRewardsParams
+	NewQueryValidatorCommissionParams         = keeper.NewQueryValidatorCommissionParams
+	NewQueryValidatorSlashesParams            = keeper.NewQueryValidatorSlashesParams
+	NewQueryDelegationRewardsParams           = keeper.NewQueryDelegationRewardsParams
+	NewQueryDelegatorParams                   = keeper.NewQueryDelegatorParams
+	NewQueryDelegatorWithdrawAddrParams       = keeper.NewQueryDelegatorWithdrawAddrParams
+	DefaultParamspace                         = keeper.DefaultParamspace
 
 	RegisterCodec       = types.RegisterCodec
 	DefaultGenesisState = types.DefaultGenesisState

x/distribution/client/cli/query.go

@@ -32,22 +32,22 @@ func GetCmdQueryParams(queryRoute string, cdc *codec.Codec) *cobra.Command {
 	}
 }
 
-// GetCmdQueryOutstandingRewards implements the query outstanding rewards command.
-func GetCmdQueryOutstandingRewards(queryRoute string, cdc *codec.Codec) *cobra.Command {
+// GetCmdQueryValidatorOutstandingRewards implements the query validator outstanding rewards command.
+func GetCmdQueryValidatorOutstandingRewards(queryRoute string, cdc *codec.Codec) *cobra.Command {
 	return &cobra.Command{
-		Use:   "outstanding-rewards",
+		Use:   "validator-outstanding-rewards",
 		Args:  cobra.NoArgs,
-		Short: "Query distribution outstanding (un-withdrawn) rewards",
+		Short: "Query distribution outstanding (un-withdrawn) rewards for a validator and all their delegations",
 		RunE: func(cmd *cobra.Command, args []string) error {
 			cliCtx := context.NewCLIContext().WithCodec(cdc)
 
-			route := fmt.Sprintf("custom/%s/outstanding_rewards", queryRoute)
+			route := fmt.Sprintf("custom/%s/validator_outstanding_rewards", queryRoute)
 			res, err := cliCtx.QueryWithData(route, []byte{})
 			if err != nil {
 				return err
 			}
 
-			var outstandingRewards types.OutstandingRewards
+			var outstandingRewards types.ValidatorOutstandingRewards
 			cdc.MustUnmarshalJSON(res, &outstandingRewards)
 			return cliCtx.PrintOutput(outstandingRewards)
 		},

x/distribution/client/module_client.go

@@ -27,7 +27,7 @@ func (mc ModuleClient) GetQueryCmd() *cobra.Command {
 
 	distQueryCmd.AddCommand(client.GetCommands(
 		distCmds.GetCmdQueryParams(mc.storeKey, mc.cdc),
-		distCmds.GetCmdQueryOutstandingRewards(mc.storeKey, mc.cdc),
+		distCmds.GetCmdQueryValidatorOutstandingRewards(mc.storeKey, mc.cdc),
 		distCmds.GetCmdQueryValidatorCommission(mc.storeKey, mc.cdc),
 		distCmds.GetCmdQueryValidatorSlashes(mc.storeKey, mc.cdc),
 		distCmds.GetCmdQueryDelegatorRewards(mc.storeKey, mc.cdc),

x/distribution/client/rest/query.go

@@ -49,17 +49,18 @@ func registerQueryRoutes(cliCtx context.CLIContext, r *mux.Router,
 		validatorRewardsHandlerFn(cliCtx, cdc, queryRoute),
 	).Methods("GET")
 
+	// Outstanding rewards of a single validator
+	r.HandleFunc(
+		"/distribution/validators/{validatorAddr}/outstanding_rewards",
+		outstandingRewardsHandlerFn(cliCtx, cdc, queryRoute),
+	).Methods("GET")
+
 	// Get the current distribution parameter values
 	r.HandleFunc(
 		"/distribution/parameters",
 		paramsHandlerFn(cliCtx, cdc, queryRoute),
 	).Methods("GET")
 
-	// Get the current distribution pool
-	r.HandleFunc(
-		"/distribution/outstanding_rewards",
-		outstandingRewardsHandlerFn(cliCtx, cdc, queryRoute),
-	).Methods("GET")
 }
 
 // HTTP request handler to query the total rewards balance from all delegations
@@ -211,7 +212,13 @@ func outstandingRewardsHandlerFn(cliCtx context.CLIContext, cdc *codec.Codec,
 	queryRoute string) http.HandlerFunc {
 
 	return func(w http.ResponseWriter, r *http.Request) {
-		res, err := cliCtx.QueryWithData(fmt.Sprintf("custom/%s/outstanding_rewards", queryRoute), []byte{})
+		validatorAddr, ok := checkValidatorAddressVar(w, r)
+		if !ok {
+			return
+		}
+
+		bin := cdc.MustMarshalJSON(distribution.NewQueryValidatorOutstandingRewardsParams(validatorAddr))
+		res, err := cliCtx.QueryWithData(fmt.Sprintf("custom/%s/validator_outstanding_rewards", queryRoute), bin)
 		if err != nil {
 			rest.WriteErrorResponse(w, http.StatusInternalServerError, err.Error())
 			return

x/distribution/genesis.go

@@ -16,7 +16,9 @@ func InitGenesis(ctx sdk.Context, keeper Keeper, data types.GenesisState) {
 		keeper.SetDelegatorWithdrawAddr(ctx, dwi.DelegatorAddress, dwi.WithdrawAddress)
 	}
 	keeper.SetPreviousProposerConsAddr(ctx, data.PreviousProposer)
-	keeper.SetOutstandingRewards(ctx, data.OutstandingRewards)
+	for _, rew := range data.OutstandingRewards {
+		keeper.SetValidatorOutstandingRewards(ctx, rew.ValidatorAddress, rew.OutstandingRewards)
+	}
 	for _, acc := range data.ValidatorAccumulatedCommissions {
 		keeper.SetValidatorAccumulatedCommission(ctx, acc.ValidatorAddress, acc.Accumulated)
 	}
@@ -50,7 +52,16 @@ func ExportGenesis(ctx sdk.Context, keeper Keeper) types.GenesisState {
 		return false
 	})
 	pp := keeper.GetPreviousProposerConsAddr(ctx)
-	outstanding := keeper.GetOutstandingRewards(ctx)
+	outstanding := make([]types.ValidatorOutstandingRewardsRecord, 0)
+	keeper.IterateValidatorOutstandingRewards(ctx,
+		func(addr sdk.ValAddress, rewards types.ValidatorOutstandingRewards) (stop bool) {
+			outstanding = append(outstanding, types.ValidatorOutstandingRewardsRecord{
+				ValidatorAddress:   addr,
+				OutstandingRewards: rewards,
+			})
+			return false
+		},
+	)
 	acc := make([]types.ValidatorAccumulatedCommissionRecord, 0)
 	keeper.IterateValidatorAccumulatedCommissions(ctx,
 		func(addr sdk.ValAddress, commission types.ValidatorAccumulatedCommission) (stop bool) {