feat: support multi gRPC query clients serve with old binary

CHANGELOG.md

@@ -38,6 +38,10 @@ Ref: https://keepachangelog.com/en/1.0.0/
 
 ## Unreleased
 
+### Features
+
+* (rpc) [#1603](https://github.com/evmos/ethermint/pull/1603) Support multi gRPC query clients serve with old binary.
+
 ## [v0.21.0-rc1] - 2022-1-13
 
 ### State Machine Breaking

app/app.go

@@ -412,18 +412,16 @@ func NewEthermintApp(
 	tracer := cast.ToString(appOpts.Get(srvflags.EVMTracer))
 
 	// Create Ethermint keepers
-	feeMarketSs := app.GetSubspace(feemarkettypes.ModuleName)
 	app.FeeMarketKeeper = feemarketkeeper.NewKeeper(
 		appCodec, authtypes.NewModuleAddress(govtypes.ModuleName),
-		keys[feemarkettypes.StoreKey], tkeys[feemarkettypes.TransientKey], feeMarketSs,
+		keys[feemarkettypes.StoreKey], tkeys[feemarkettypes.TransientKey],
 	)
 
 	// Set authority to x/gov module account to only expect the module account to update params
-	evmSs := app.GetSubspace(evmtypes.ModuleName)
 	app.EvmKeeper = evmkeeper.NewKeeper(
 		appCodec, keys[evmtypes.StoreKey], tkeys[evmtypes.TransientKey], authtypes.NewModuleAddress(govtypes.ModuleName),
 		app.AccountKeeper, app.BankKeeper, app.StakingKeeper, app.FeeMarketKeeper,
-		nil, geth.NewEVM, tracer, evmSs,
+		nil, geth.NewEVM, tracer,
 	)
 
 	// Create IBC Keeper
@@ -509,8 +507,8 @@ func NewEthermintApp(
 		ibc.NewAppModule(app.IBCKeeper),
 		transferModule,
 		// Ethermint app modules
-		feemarket.NewAppModule(app.FeeMarketKeeper, feeMarketSs),
-		evm.NewAppModule(app.EvmKeeper, app.AccountKeeper, evmSs),
+		feemarket.NewAppModule(app.FeeMarketKeeper, app.GetSubspace(feemarkettypes.ModuleName)),
+		evm.NewAppModule(app.EvmKeeper, app.AccountKeeper, app.GetSubspace(evmtypes.ModuleName)),
 	)
 
 	// During begin block slashing happens after distr.BeginBlocker so that

go.mod

@@ -38,7 +38,6 @@ require (
 	github.com/tyler-smith/go-bip39 v1.1.0
 	golang.org/x/net v0.5.0
 	golang.org/x/text v0.6.0
-	golang.org/x/text v0.6.0
 	google.golang.org/genproto v0.0.0-20221118155620-16455021b5e6
 	google.golang.org/grpc v1.52.0
 	sigs.k8s.io/yaml v1.3.0

gomod2nix.toml

@@ -358,8 +358,8 @@ schema = 3
     version = "v0.0.5"
     hash = "sha256-/5i70IkH/qSW5KjGzv8aQNKh9tHoz98tqtL0K2DMFn4="
   [mod."github.com/onsi/ginkgo/v2"]
-    version = "v2.6.1"
-    hash = "sha256-OEiWYKCGPCaqL3vzSrHquHGm+Q8URT2anpanAVK5hRo="
+    version = "v2.7.0"
+    hash = "sha256-BKqQKCsPA73FaQwYpAY+QsWFHIncrG5jgRhC2IiNmCk="
   [mod."github.com/onsi/gomega"]
     version = "v1.24.2"
     hash = "sha256-iascSzzBT1Uv/XybezSblIwwrq78BU4a9BVB5MvK6MM="
@@ -496,23 +496,23 @@ schema = 3
     version = "v0.0.0-20220722155223-a9213eeb770e"
     hash = "sha256-kNgzydWRpjm0sZl4uXEs3LX5L0xjJtJRAFf/CTlYUN4="
   [mod."golang.org/x/net"]
-    version = "v0.4.0"
-    hash = "sha256-7IwGZh/xg4mQz88cJio2Ov5d3jGRXKj1itlAja/EAbQ="
+    version = "v0.5.0"
+    hash = "sha256-HpbIAiLs7S1+tVsaSSdbCPw1IK43A0bFFuSzPSyjLbo="
   [mod."golang.org/x/oauth2"]
     version = "v0.0.0-20221014153046-6fdb5e3db783"
     hash = "sha256-IoygidVNqyAZmN+3macDeIefK8hhJToygpcqlwehdYQ="
   [mod."golang.org/x/sync"]
     version = "v0.1.0"
     hash = "sha256-Hygjq9euZ0qz6TvHYQwOZEjNiTbTh1nSLRAWZ6KFGR8="
   [mod."golang.org/x/sys"]
-    version = "v0.3.0"
-    hash = "sha256-TIHhfYbZ99sCU1ZMikxwomXH5AEtD/lA1VMMW+UAhbU="
+    version = "v0.4.0"
+    hash = "sha256-jchMzHCH5dg+IL/F+LqaX/fyAcB/nvHQpfBjqwaRJH0="
   [mod."golang.org/x/term"]
-    version = "v0.3.0"
-    hash = "sha256-NKv2o8wz8DB/2W2h/muGEIHb+S06mBXZxhG254RpQ5s="
+    version = "v0.4.0"
+    hash = "sha256-wQKxHV10TU4vCU8Re2/hFmAbur/jRWEOB8QXBzgTFNY="
   [mod."golang.org/x/text"]
-    version = "v0.5.0"
-    hash = "sha256-ztH+xQyM/clOcQl+y/UEPcfNKbc3xApMbEPDDZ9up0o="
+    version = "v0.6.0"
+    hash = "sha256-+bpeRWR3relKACdal6NPj+eP5dnWCplTViArSN7/qA4="
   [mod."golang.org/x/xerrors"]
     version = "v0.0.0-20220907171357-04be3eba64a2"
     hash = "sha256-6+zueutgefIYmgXinOflz8qGDDDj0Zhv+2OkGhBTKno="

rpc/apis.go

@@ -35,6 +35,7 @@ import (
 	ethermint "github.com/evmos/ethermint/types"
 
 	rpcclient "github.com/tendermint/tendermint/rpc/jsonrpc/client"
+	"google.golang.org/grpc"
 )
 
 // RPC namespaces and API version
@@ -60,6 +61,7 @@ const (
 type APICreator = func(
 	ctx *server.Context,
 	clientCtx client.Context,
+	backupGRPCClientConns map[[2]int]*grpc.ClientConn,
 	tendermintWebsocketClient *rpcclient.WSClient,
 	allowUnprotectedTxs bool,
 	indexer ethermint.EVMTxIndexer,
@@ -72,11 +74,12 @@ func init() {
 	apiCreators = map[string]APICreator{
 		EthNamespace: func(ctx *server.Context,
 			clientCtx client.Context,
+			backupGRPCClientConns map[[2]int]*grpc.ClientConn,
 			tmWSClient *rpcclient.WSClient,
 			allowUnprotectedTxs bool,
 			indexer ethermint.EVMTxIndexer,
 		) []rpc.API {
-			evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs, indexer)
+			evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, backupGRPCClientConns, allowUnprotectedTxs, indexer)
 			return []rpc.API{
 				{
 					Namespace: EthNamespace,
@@ -92,7 +95,14 @@ func init() {
 				},
 			}
 		},
-		Web3Namespace: func(*server.Context, client.Context, *rpcclient.WSClient, bool, ethermint.EVMTxIndexer) []rpc.API {
+		Web3Namespace: func(
+			*server.Context,
+			client.Context,
+			map[[2]int]*grpc.ClientConn,
+			*rpcclient.WSClient,
+			bool,
+			ethermint.EVMTxIndexer,
+		) []rpc.API {
 			return []rpc.API{
 				{
 					Namespace: Web3Namespace,
@@ -102,7 +112,14 @@ func init() {
 				},
 			}
 		},
-		NetNamespace: func(_ *server.Context, clientCtx client.Context, _ *rpcclient.WSClient, _ bool, _ ethermint.EVMTxIndexer) []rpc.API {
+		NetNamespace: func(
+			_ *server.Context,
+			clientCtx client.Context,
+			_ map[[2]int]*grpc.ClientConn,
+			_ *rpcclient.WSClient,
+			_ bool,
+			_ ethermint.EVMTxIndexer,
+		) []rpc.API {
 			return []rpc.API{
 				{
 					Namespace: NetNamespace,
@@ -114,11 +131,12 @@ func init() {
 		},
 		PersonalNamespace: func(ctx *server.Context,
 			clientCtx client.Context,
+			backupGRPCClientConns map[[2]int]*grpc.ClientConn,
 			_ *rpcclient.WSClient,
 			allowUnprotectedTxs bool,
 			indexer ethermint.EVMTxIndexer,
 		) []rpc.API {
-			evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs, indexer)
+			evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, backupGRPCClientConns, allowUnprotectedTxs, indexer)
 			return []rpc.API{
 				{
 					Namespace: PersonalNamespace,
@@ -128,7 +146,14 @@ func init() {
 				},
 			}
 		},
-		TxPoolNamespace: func(ctx *server.Context, _ client.Context, _ *rpcclient.WSClient, _ bool, _ ethermint.EVMTxIndexer) []rpc.API {
+		TxPoolNamespace: func(
+			ctx *server.Context,
+			_ client.Context,
+			_ map[[2]int]*grpc.ClientConn,
+			_ *rpcclient.WSClient,
+			_ bool,
+			_ ethermint.EVMTxIndexer,
+		) []rpc.API {
 			return []rpc.API{
 				{
 					Namespace: TxPoolNamespace,
@@ -140,11 +165,12 @@ func init() {
 		},
 		DebugNamespace: func(ctx *server.Context,
 			clientCtx client.Context,
+			backupGRPCClientConns map[[2]int]*grpc.ClientConn,
 			_ *rpcclient.WSClient,
 			allowUnprotectedTxs bool,
 			indexer ethermint.EVMTxIndexer,
 		) []rpc.API {
-			evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs, indexer)
+			evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, backupGRPCClientConns, allowUnprotectedTxs, indexer)
 			return []rpc.API{
 				{
 					Namespace: DebugNamespace,
@@ -156,11 +182,12 @@ func init() {
 		},
 		MinerNamespace: func(ctx *server.Context,
 			clientCtx client.Context,
+			backupGRPCClientConns map[[2]int]*grpc.ClientConn,
 			_ *rpcclient.WSClient,
 			allowUnprotectedTxs bool,
 			indexer ethermint.EVMTxIndexer,
 		) []rpc.API {
-			evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs, indexer)
+			evmBackend := backend.NewBackend(ctx, ctx.Logger, clientCtx, backupGRPCClientConns, allowUnprotectedTxs, indexer)
 			return []rpc.API{
 				{
 					Namespace: MinerNamespace,
@@ -176,6 +203,7 @@ func init() {
 // GetRPCAPIs returns the list of all APIs
 func GetRPCAPIs(ctx *server.Context,
 	clientCtx client.Context,
+	backupGRPCClientConns map[[2]int]*grpc.ClientConn,
 	tmWSClient *rpcclient.WSClient,
 	allowUnprotectedTxs bool,
 	indexer ethermint.EVMTxIndexer,
@@ -185,7 +213,7 @@ func GetRPCAPIs(ctx *server.Context,
 
 	for _, ns := range selectedAPIs {
 		if creator, ok := apiCreators[ns]; ok {
-			apis = append(apis, creator(ctx, clientCtx, tmWSClient, allowUnprotectedTxs, indexer)...)
+			apis = append(apis, creator(ctx, clientCtx, backupGRPCClientConns, tmWSClient, allowUnprotectedTxs, indexer)...)
 		} else {
 			ctx.Logger.Error("invalid namespace value", "namespace", ns)
 		}

rpc/backend/account_info.go

@@ -40,8 +40,9 @@ func (b *Backend) GetCode(address common.Address, blockNrOrHash rpctypes.BlockNu
 	req := &evmtypes.QueryCodeRequest{
 		Address: address.String(),
 	}
-
-	res, err := b.queryClient.Code(rpctypes.ContextWithHeight(blockNum.Int64()), req)
+	height := blockNum.Int64()
+	queryClient := b.getGrpcClient(height)
+	res, err := queryClient.Code(rpctypes.ContextWithHeight(height), req)
 	if err != nil {
 		return nil, err
 	}
@@ -82,10 +83,10 @@ func (b *Backend) GetProof(address common.Address, storageKeys []string, blockNr
 
 	// query storage proofs
 	storageProofs := make([]rpctypes.StorageResult, len(storageKeys))
-
+	queryClient := b.getGrpcClient(height)
 	for i, key := range storageKeys {
 		hexKey := common.HexToHash(key)
-		valueBz, proof, err := b.queryClient.GetProof(clientCtx, evmtypes.StoreKey, evmtypes.StateKey(address, hexKey.Bytes()))
+		valueBz, proof, err := queryClient.GetProof(clientCtx, evmtypes.StoreKey, evmtypes.StateKey(address, hexKey.Bytes()))
 		if err != nil {
 			return nil, err
 		}
@@ -102,14 +103,14 @@ func (b *Backend) GetProof(address common.Address, storageKeys []string, blockNr
 		Address: address.String(),
 	}
 
-	res, err := b.queryClient.Account(ctx, req)
+	res, err := queryClient.Account(ctx, req)
 	if err != nil {
 		return nil, err
 	}
 
 	// query account proofs
 	accountKey := authtypes.AddressStoreKey(sdk.AccAddress(address.Bytes()))
-	_, proof, err := b.queryClient.GetProof(clientCtx, authtypes.StoreKey, accountKey)
+	_, proof, err := queryClient.GetProof(clientCtx, authtypes.StoreKey, accountKey)
 	if err != nil {
 		return nil, err
 	}
@@ -141,8 +142,9 @@ func (b *Backend) GetStorageAt(address common.Address, key string, blockNrOrHash
 		Address: address.String(),
 		Key:     key,
 	}
-
-	res, err := b.queryClient.Storage(rpctypes.ContextWithHeight(blockNum.Int64()), req)
+	height := blockNum.Int64()
+	queryClient := b.getGrpcClient(height)
+	res, err := queryClient.Storage(rpctypes.ContextWithHeight(height), req)
 	if err != nil {
 		return nil, err
 	}
@@ -167,7 +169,7 @@ func (b *Backend) GetBalance(address common.Address, blockNrOrHash rpctypes.Bloc
 		return nil, err
 	}
 
-	res, err := b.queryClient.Balance(rpctypes.ContextWithHeight(blockNum.Int64()), req)
+	res, err := b.getGrpcClient(blockNum.Int64()).Balance(rpctypes.ContextWithHeight(blockNum.Int64()), req)
 	if err != nil {
 		return nil, err
 	}

rpc/backend/backend.go

@@ -24,6 +24,7 @@ import (
 	"github.com/cosmos/cosmos-sdk/crypto/keyring"
 	"github.com/cosmos/cosmos-sdk/server"
 	sdk "github.com/cosmos/cosmos-sdk/types"
+	"github.com/cosmos/cosmos-sdk/types/tx"
 	"github.com/ethereum/go-ethereum/common"
 	"github.com/ethereum/go-ethereum/common/hexutil"
 	ethtypes "github.com/ethereum/go-ethereum/core/types"
@@ -34,8 +35,10 @@ import (
 	"github.com/evmos/ethermint/server/config"
 	ethermint "github.com/evmos/ethermint/types"
 	evmtypes "github.com/evmos/ethermint/x/evm/types"
+	feemarkettypes "github.com/evmos/ethermint/x/feemarket/types"
 	"github.com/tendermint/tendermint/libs/log"
 	tmrpctypes "github.com/tendermint/tendermint/rpc/core/types"
+	"google.golang.org/grpc"
 )
 
 // BackendI implements the Cosmos and EVM backend.
@@ -149,6 +152,7 @@ type Backend struct {
 	ctx                 context.Context
 	clientCtx           client.Context
 	queryClient         *rpctypes.QueryClient // gRPC query client
+	backupQueryClients  map[[2]int]*rpctypes.QueryClient
 	logger              log.Logger
 	chainID             *big.Int
 	cfg                 config.Config
@@ -161,6 +165,7 @@ func NewBackend(
 	ctx *server.Context,
 	logger log.Logger,
 	clientCtx client.Context,
+	backupGRPCClientConns map[[2]int]*grpc.ClientConn,
 	allowUnprotectedTxs bool,
 	indexer ethermint.EVMTxIndexer,
 ) *Backend {
@@ -174,14 +179,23 @@ func NewBackend(
 		panic(err)
 	}
 
-	return &Backend{
+	backend := &Backend{
 		ctx:                 context.Background(),
 		clientCtx:           clientCtx,
 		queryClient:         rpctypes.NewQueryClient(clientCtx),
+		backupQueryClients:  make(map[[2]int]*rpctypes.QueryClient),
 		logger:              logger.With("module", "backend"),
 		chainID:             chainID,
 		cfg:                 appConf,
 		allowUnprotectedTxs: allowUnprotectedTxs,
 		indexer:             indexer,
 	}
+	for key, conn := range backupGRPCClientConns {
+		backend.backupQueryClients[key] = &rpctypes.QueryClient{
+			ServiceClient: tx.NewServiceClient(conn),
+			QueryClient:   evmtypes.NewQueryClient(conn),
+			FeeMarket:     feemarkettypes.NewQueryClient(conn),
+		}
+	}
+	return backend
 }

rpc/backend/backend_suite_test.go

@@ -27,6 +27,7 @@ import (
 	rpctypes "github.com/evmos/ethermint/rpc/types"
 	"github.com/evmos/ethermint/tests"
 	evmtypes "github.com/evmos/ethermint/x/evm/types"
+	"google.golang.org/grpc"
 )
 
 type BackendTestSuite struct {
@@ -79,7 +80,7 @@ func (suite *BackendTestSuite) SetupTest() {
 	allowUnprotectedTxs := false
 	idxer := indexer.NewKVIndexer(dbm.NewMemDB(), ctx.Logger, clientCtx)
 
-	suite.backend = NewBackend(ctx, ctx.Logger, clientCtx, allowUnprotectedTxs, idxer)
+	suite.backend = NewBackend(ctx, ctx.Logger, clientCtx, make(map[[2]int]*grpc.ClientConn), allowUnprotectedTxs, idxer)
 	suite.backend.queryClient.QueryClient = mocks.NewEVMQueryClient(suite.T())
 	suite.backend.clientCtx.Client = mocks.NewClient(suite.T())
 	suite.backend.queryClient.FeeMarket = mocks.NewFeeMarketQueryClient(suite.T())

rpc/backend/blocks.go

@@ -42,6 +42,7 @@ import (
 func (b *Backend) BlockNumber() (hexutil.Uint64, error) {
 	// do any grpc query, ignore the response and use the returned block height
 	var header metadata.MD
+	// use latest queryClient to get block height
 	_, err := b.queryClient.Params(b.ctx, &evmtypes.QueryParamsRequest{}, grpc.Header(&header))
 	if err != nil {
 		return hexutil.Uint64(0), err
@@ -412,7 +413,7 @@ func (b *Backend) RPCBlockFromTendermintBlock(
 	var validatorAccAddr sdk.AccAddress
 
 	ctx := rpctypes.ContextWithHeight(block.Height)
-	res, err := b.queryClient.ValidatorAccount(ctx, req)
+	res, err := b.getGrpcClient(block.Height).ValidatorAccount(ctx, req)
 	if err != nil {
 		b.logger.Debug(
 			"failed to query validator operator address",