feat: add query.GenericFilteredPaginated (backport #12253)

CHANGELOG.md

@@ -43,6 +43,10 @@ Ref: https://keepachangelog.com/en/1.0.0/
 * (x/upgrade) [#12264](https://github.com/cosmos/cosmos-sdk/pull/12264) Fix `GetLastCompleteUpgrade` to properly return the latest upgrade.
 * (x/crisis) [#12208](https://github.com/cosmos/cosmos-sdk/pull/12208) Fix progress index of crisis invariant assertion logs.
 
+### Features
+
+* (query) [#12253](https://github.com/cosmos/cosmos-sdk/pull/12253) Add `GenericFilteredPaginate` to the `query` package to improve UX.
+
 ## [v0.45.5](https://github.com/cosmos/cosmos-sdk/releases/tag/v0.45.5) - 2022-06-09
 
 ### Improvements

go.mod

@@ -1,4 +1,4 @@
-go 1.17
+go 1.18
 
 module github.com/cosmos/cosmos-sdk
 

types/query/filtered_pagination.go

@@ -3,6 +3,7 @@ package query
 import (
 	"fmt"
 
+	"github.com/cosmos/cosmos-sdk/codec"
 	"github.com/cosmos/cosmos-sdk/store/types"
 )
 
@@ -46,8 +47,10 @@ func FilteredPaginate(
 		iterator := getIterator(prefixStore, key, reverse)
 		defer iterator.Close()
 
-		var numHits uint64
-		var nextKey []byte
+		var (
+			numHits uint64
+			nextKey []byte
+		)
 
 		for ; iterator.Valid(); iterator.Next() {
 			if numHits == limit {
@@ -79,8 +82,10 @@ func FilteredPaginate(
 
 	end := offset + limit
 
-	var numHits uint64
-	var nextKey []byte
+	var (
+		numHits uint64
+		nextKey []byte
+	)
 
 	for ; iterator.Valid(); iterator.Next() {
 		if iterator.Error() != nil {
@@ -113,3 +118,137 @@ func FilteredPaginate(
 
 	return res, nil
 }
+
+// GenericFilteredPaginate does pagination of all the results in the PrefixStore based on the
+// provided PageRequest. `onResult` should be used to filter or transform the results.
+// `c` is a constructor function that needs to return a new instance of the type T (this is to
+// workaround some generic pitfalls in which we can't instantiate a T struct inside the function).
+// If key is provided, the pagination uses the optimized querying.
+// If offset is used, the pagination uses lazy filtering i.e., searches through all the records.
+// The resulting slice (of type F) can be of a different type than the one being iterated through
+// (type T), so it's possible to do any necessary transformation inside the onResult function.
+func GenericFilteredPaginate[T codec.ProtoMarshaler, F codec.ProtoMarshaler](
+	cdc codec.BinaryCodec,
+	prefixStore types.KVStore,
+	pageRequest *PageRequest,
+	onResult func(key []byte, value T) (F, error),
+	constructor func() T,
+) ([]F, *PageResponse, error) {
+	// if the PageRequest is nil, use default PageRequest
+	if pageRequest == nil {
+		pageRequest = &PageRequest{}
+	}
+
+	offset := pageRequest.Offset
+	key := pageRequest.Key
+	limit := pageRequest.Limit
+	countTotal := pageRequest.CountTotal
+	reverse := pageRequest.Reverse
+	results := []F{}
+
+	if offset > 0 && key != nil {
+		return results, nil, fmt.Errorf("invalid request, either offset or key is expected, got both")
+	}
+
+	if limit == 0 {
+		limit = DefaultLimit
+
+		// count total results when the limit is zero/not supplied
+		countTotal = true
+	}
+
+	if len(key) != 0 {
+		iterator := getIterator(prefixStore, key, reverse)
+		defer iterator.Close()
+
+		var (
+			numHits uint64
+			nextKey []byte
+		)
+
+		for ; iterator.Valid(); iterator.Next() {
+			if numHits == limit {
+				nextKey = iterator.Key()
+				break
+			}
+
+			if iterator.Error() != nil {
+				return nil, nil, iterator.Error()
+			}
+
+			protoMsg := constructor()
+
+			err := cdc.Unmarshal(iterator.Value(), protoMsg)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			val, err := onResult(iterator.Key(), protoMsg)
+			if err != nil {
+				return nil, nil, err
+			}
+
+			if val.Size() != 0 {
+				results = append(results, val)
+				numHits++
+			}
+		}
+
+		return results, &PageResponse{
+			NextKey: nextKey,
+		}, nil
+	}
+
+	iterator := getIterator(prefixStore, nil, reverse)
+	defer iterator.Close()
+
+	end := offset + limit
+
+	var (
+		numHits uint64
+		nextKey []byte
+	)
+
+	for ; iterator.Valid(); iterator.Next() {
+		if iterator.Error() != nil {
+			return nil, nil, iterator.Error()
+		}
+
+		protoMsg := constructor()
+
+		err := cdc.Unmarshal(iterator.Value(), protoMsg)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		val, err := onResult(iterator.Key(), protoMsg)
+		if err != nil {
+			return nil, nil, err
+		}
+
+		if val.Size() != 0 {
+			// Previously this was the "accumulate" flag
+			if numHits >= offset && numHits < end {
+				results = append(results, val)
+			}
+			numHits++
+		}
+
+		if numHits == end+1 {
+			if nextKey == nil {
+				nextKey = iterator.Key()
+			}
+
+			if !countTotal {
+				break
+			}
+		}
+	}
+
+	res := &PageResponse{NextKey: nextKey}
+	if countTotal {
+		res.Total = numHits
+	}
+
+	return results, res, nil
+}

x/authz/keeper/grpc_query.go

@@ -6,9 +6,6 @@ import (
 	"google.golang.org/grpc/codes"
 	"google.golang.org/grpc/status"
 
-	proto "github.com/gogo/protobuf/proto"
-
-	"github.com/cosmos/cosmos-sdk/codec"
 	codectypes "github.com/cosmos/cosmos-sdk/codec/types"
 	"github.com/cosmos/cosmos-sdk/store/prefix"
 	sdk "github.com/cosmos/cosmos-sdk/types"
@@ -35,10 +32,6 @@ func (k Keeper) Grants(c context.Context, req *authz.QueryGrantsRequest) (*authz
 	}
 	ctx := sdk.UnwrapSDKContext(c)
 
-	store := ctx.KVStore(k.storeKey)
-	key := grantStoreKey(grantee, granter, "")
-	authStore := prefix.NewStore(store, key)
-
 	if req.MsgTypeUrl != "" {
 		authorization, expiration := k.GetCleanAuthorization(ctx, grantee, granter, req.MsgTypeUrl)
 		if authorization == nil {
@@ -56,29 +49,26 @@ func (k Keeper) Grants(c context.Context, req *authz.QueryGrantsRequest) (*authz
 		}, nil
 	}
 
-	var authorizations []*authz.Grant
-	pageRes, err := query.FilteredPaginate(authStore, req.Pagination, func(key []byte, value []byte, accumulate bool) (bool, error) {
-		auth, err := unmarshalAuthorization(k.cdc, value)
+	store := ctx.KVStore(k.storeKey)
+	key := grantStoreKey(grantee, granter, "")
+	grantsStore := prefix.NewStore(store, key)
+
+	authorizations, pageRes, err := query.GenericFilteredPaginate(k.cdc, grantsStore, req.Pagination, func(key []byte, auth *authz.Grant) (*authz.Grant, error) {
+		auth1 := auth.GetAuthorization()
 		if err != nil {
-			return false, err
+			return nil, err
 		}
-		auth1 := auth.GetAuthorization()
-		if accumulate {
-			msg, ok := auth1.(proto.Message)
-			if !ok {
-				return false, status.Errorf(codes.Internal, "can't protomarshal %T", msg)
-			}
-
-			authorizationAny, err := codectypes.NewAnyWithValue(msg)
-			if err != nil {
-				return false, status.Errorf(codes.Internal, err.Error())
-			}
-			authorizations = append(authorizations, &authz.Grant{
-				Authorization: authorizationAny,
-				Expiration:    auth.Expiration,
-			})
+
+		authorizationAny, err := codectypes.NewAnyWithValue(auth1)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, err.Error())
 		}
-		return true, nil
+		return &authz.Grant{
+			Authorization: authorizationAny,
+			Expiration:    auth.Expiration,
+		}, nil
+	}, func() *authz.Grant {
+		return &authz.Grant{}
 	})
 	if err != nil {
 		return nil, err
@@ -105,32 +95,29 @@ func (k Keeper) GranterGrants(c context.Context, req *authz.QueryGranterGrantsRe
 	store := ctx.KVStore(k.storeKey)
 	authzStore := prefix.NewStore(store, grantStoreKey(nil, granter, ""))
 
-	var grants []*authz.GrantAuthorization
-	pageRes, err := query.FilteredPaginate(authzStore, req.Pagination, func(key []byte, value []byte,
-		accumulate bool,
-	) (bool, error) {
-		auth, err := unmarshalAuthorization(k.cdc, value)
+	grants, pageRes, err := query.GenericFilteredPaginate(k.cdc, authzStore, req.Pagination, func(key []byte, auth *authz.Grant) (*authz.GrantAuthorization, error) {
+		auth1 := auth.GetAuthorization()
 		if err != nil {
-			return false, err
+			return nil, err
 		}
 
-		auth1 := auth.GetAuthorization()
-		if accumulate {
-			any, err := codectypes.NewAnyWithValue(auth1)
-			if err != nil {
-				return false, status.Errorf(codes.Internal, err.Error())
-			}
-
-			grantee := firstAddressFromGrantStoreKey(key)
-			grants = append(grants, &authz.GrantAuthorization{
-				Granter:       granter.String(),
-				Grantee:       grantee.String(),
-				Authorization: any,
-				Expiration:    auth.Expiration,
-			})
+		any, err := codectypes.NewAnyWithValue(auth1)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, err.Error())
 		}
-		return true, nil
+
+		grantee := firstAddressFromGrantStoreKey(key)
+		return &authz.GrantAuthorization{
+			Granter:       granter.String(),
+			Grantee:       grantee.String(),
+			Authorization: any,
+			Expiration:    auth.Expiration,
+		}, nil
+
+	}, func() *authz.Grant {
+		return &authz.Grant{}
 	})
+
 	if err != nil {
 		return nil, err
 	}
@@ -155,35 +142,30 @@ func (k Keeper) GranteeGrants(c context.Context, req *authz.QueryGranteeGrantsRe
 	ctx := sdk.UnwrapSDKContext(c)
 	store := prefix.NewStore(ctx.KVStore(k.storeKey), GrantKey)
 
-	var authorizations []*authz.GrantAuthorization
-	pageRes, err := query.FilteredPaginate(store, req.Pagination, func(key []byte, value []byte,
-		accumulate bool,
-	) (bool, error) {
-		auth, err := unmarshalAuthorization(k.cdc, value)
+	authorizations, pageRes, err := query.GenericFilteredPaginate(k.cdc, store, req.Pagination, func(key []byte, auth *authz.Grant) (*authz.GrantAuthorization, error) {
+		auth1 := auth.GetAuthorization()
 		if err != nil {
-			return false, err
+			return nil, err
 		}
 
 		granter, g := addressesFromGrantStoreKey(append(GrantKey, key...))
 		if !g.Equals(grantee) {
-			return false, nil
+			return nil, nil
 		}
 
-		auth1 := auth.GetAuthorization()
-		if accumulate {
-			any, err := codectypes.NewAnyWithValue(auth1)
-			if err != nil {
-				return false, status.Errorf(codes.Internal, err.Error())
-			}
-
-			authorizations = append(authorizations, &authz.GrantAuthorization{
-				Authorization: any,
-				Expiration:    auth.Expiration,
-				Granter:       granter.String(),
-				Grantee:       grantee.String(),
-			})
+		authorizationAny, err := codectypes.NewAnyWithValue(auth1)
+		if err != nil {
+			return nil, status.Errorf(codes.Internal, err.Error())
 		}
-		return true, nil
+
+		return &authz.GrantAuthorization{
+			Authorization: authorizationAny,
+			Expiration:    auth.Expiration,
+			Granter:       granter.String(),
+			Grantee:       grantee.String(),
+		}, nil
+	}, func() *authz.Grant {
+		return &authz.Grant{}
 	})
 	if err != nil {
 		return nil, err
@@ -194,9 +176,3 @@ func (k Keeper) GranteeGrants(c context.Context, req *authz.QueryGranteeGrantsRe
 		Pagination: pageRes,
 	}, nil
 }
-
-// unmarshal an authorization from a store value
-func unmarshalAuthorization(cdc codec.BinaryCodec, value []byte) (v authz.Grant, err error) {
-	err = cdc.Unmarshal(value, &v)
-	return v, err
-}

x/authz/keeper/grpc_query_test.go

@@ -214,7 +214,8 @@ func (suite *TestSuite) TestGRPCQueryGranterGrants() {
 		},
 		{
 			"valid case, pagination",
-			func() {},
+			func() {
+			},
 			false,
 			authz.QueryGranterGrantsRequest{
 				Granter: addrs[0].String(),
@@ -273,6 +274,15 @@ func (suite *TestSuite) TestGRPCQueryGranteeGrants() {
 			},
 			1,
 		},
+		{
+			"valid case, no authorization found",
+			func() {},
+			false,
+			authz.QueryGranteeGrantsRequest{
+				Grantee: addrs[2].String(),
+			},
+			0,
+		},
 		{
 			"valid case, multiple authorization",
 			func() {