Merge v2022.04.05

.github/workflows/ci.yml

@@ -71,6 +71,8 @@ jobs:
   docker:
     runs-on: ubuntu-18.04
     steps:
-      - uses: actions/checkout@v2
-      - run: git submodule update --init --recursive --force
-      - run: docker build .
+      - uses: actions/checkout@v3
+        with:
+          submodules: recursive
+          fetch-depth: 0 # fetch git tags for "git describe"
+      - run: make docker

.gitmodules

@@ -3,4 +3,4 @@
 	url = https://github.com/ethereum/tests
 [submodule "libmdbx"]
 	path = libmdbx
-	url = https://github.com/erthink/libmdbx
+	url = https://github.com/torquem-ch/libmdbx.git

Dockerfile

@@ -26,6 +26,7 @@ EXPOSE 8545 8546 30303 30303/udp 30304 30304/udp 8080 9090 6060
 # https://github.com/opencontainers/image-spec/blob/main/annotations.md
 ARG BUILD_DATE
 ARG VCS_REF
+ARG VERSION
 LABEL org.label-schema.build-date=$BUILD_DATE \
       org.label-schema.name="Erigon" \
       org.label-schema.description="Erigon Ethereum Client" \

Makefile

@@ -4,7 +4,7 @@ GOTEST = GODEBUG=cgocheck=0 $(GO) test -tags nosqlite -trimpath ./... -p 2
 
 GIT_COMMIT ?= $(shell git rev-list -1 HEAD)
 GIT_BRANCH ?= $(shell git rev-parse --abbrev-ref HEAD)
-GIT_TAG    ?= $(shell git describe --tags `git rev-list --tags="v*" --max-count=1`)
+GIT_TAG    ?= $(shell git describe --tags '--match=v*')
 
 CGO_CFLAGS := $(shell $(GO) env CGO_CFLAGS) # don't loose default
 CGO_CFLAGS += -DMDBX_FORCE_ASSERTIONS=0 # Enable MDBX's asserts by default in 'devel' branch and disable in 'stable'
@@ -26,7 +26,12 @@ go-version:
 	fi
 
 docker:
-	DOCKER_BUILDKIT=1 docker build -t erigon:latest --build-arg git_commit='${GIT_COMMIT}' --build-arg git_branch='${GIT_BRANCH}' --build-arg git_tag='${GIT_TAG}' .
+	DOCKER_BUILDKIT=1 docker build \
+		--build-arg "BUILD_DATE=$(shell date -Iseconds)" \
+		--build-arg VCS_REF=${GIT_COMMIT} \
+		--build-arg VERSION=${GIT_TAG} \
+		${DOCKER_FLAGS} \
+		.
 
 xdg_data_home :=  ~/.local/share
 ifdef XDG_DATA_HOME

README.md

@@ -36,6 +36,7 @@ NB! <code>In-depth links are marked by the microscope sign (🔬) </code>
 **Disclaimer: this software is currently a tech preview. We will do our best to keep it stable and make no breaking
 changes but we don't guarantee anything. Things can and will break.**
 
+<code>🔬 Alpha/Beta versions difference: [here](https://erigon.substack.com/p/erigon-2-three-upgrades?s=r)</code>
 
 System Requirements
 ===================
@@ -45,7 +46,7 @@ folder `<datadir>/etl-tmp` to another disk).
 
 RAM: 16GB, 64-bit architecture, [Golang version >= 1.16](https://golang.org/doc/install)
 
-<code>🔬 more info on disk storage is [here](https://ledgerwatch.github.io/turbo_geth_release.html#Disk-space)) </code>
+<code>🔬 more details on disk storage [here](https://erigon.substack.com/p/disk-footprint-changes-in-new-erigon?s=r) and [here](https://ledgerwatch.github.io/turbo_geth_release.html#Disk-space).</code>
 
 Usage
 =====

cmd/erigon/main.go

@@ -4,7 +4,7 @@ import (
 	"fmt"
 	"os"
 
-	"github.com/ledgerwatch/erigon/common/debug"
+	"github.com/ledgerwatch/erigon-lib/common/dbg"
 	"github.com/ledgerwatch/erigon/params"
 	erigoncli "github.com/ledgerwatch/erigon/turbo/cli"
 	"github.com/ledgerwatch/erigon/turbo/node"
@@ -13,7 +13,16 @@ import (
 )
 
 func main() {
-	defer debug.LogPanic()
+	defer func() {
+		panicResult := recover()
+		if panicResult == nil {
+			return
+		}
+
+		log.Error("catch panic", "err", panicResult, "stack", dbg.Stack())
+		os.Exit(1)
+	}()
+
 	app := erigoncli.MakeApp(runErigon, erigoncli.DefaultFlags)
 	if err := app.Run(os.Args); err != nil {
 		fmt.Fprintln(os.Stderr, err)

cmd/integration/commands/state_stages.go

@@ -179,7 +179,7 @@ func syncBySmallSteps(db kv.RwDB, miningConfig params.MiningConfig, ctx context.
 
 	stateStages.DisableStages(stages.Headers, stages.BlockHashes, stages.Bodies, stages.Senders,
 		stages.TxPool, // TODO: enable TxPoolDB stage
-		stages.Finish)
+	)
 
 	execCfg := stagedsync.StageExecuteBlocksCfg(db, pm, batchSize, changeSetHook, chainConfig, engine, vmConfig, nil, false, tmpDir)
 

cmd/rpcdaemon/commands/eth_api.go

@@ -75,7 +75,7 @@ type EthAPI interface {
 
 	// Sending related (see ./eth_call.go)
 	Call(ctx context.Context, args ethapi.CallArgs, blockNrOrHash rpc.BlockNumberOrHash, overrides *ethapi.StateOverrides) (hexutil.Bytes, error)
-	EstimateGas(ctx context.Context, args ethapi.CallArgs, blockNrOrHash *rpc.BlockNumberOrHash) (hexutil.Uint64, error)
+	EstimateGas(ctx context.Context, argsOrNil *ethapi.CallArgs, blockNrOrHash *rpc.BlockNumberOrHash) (hexutil.Uint64, error)
 	SendRawTransaction(ctx context.Context, encodedTx hexutil.Bytes) (common.Hash, error)
 	SendTransaction(_ context.Context, txObject interface{}) (common.Hash, error)
 	Sign(ctx context.Context, _ common.Address, _ hexutil.Bytes) (hexutil.Bytes, error)

cmd/rpcdaemon/commands/eth_call.go

@@ -113,7 +113,13 @@ func HeaderByNumberOrHash(ctx context.Context, tx kv.Tx, blockNrOrHash rpc.Block
 }
 
 // EstimateGas implements eth_estimateGas. Returns an estimate of how much gas is necessary to allow the transaction to complete. The transaction will not be added to the blockchain.
-func (api *APIImpl) EstimateGas(ctx context.Context, args ethapi.CallArgs, blockNrOrHash *rpc.BlockNumberOrHash) (hexutil.Uint64, error) {
+func (api *APIImpl) EstimateGas(ctx context.Context, argsOrNil *ethapi.CallArgs, blockNrOrHash *rpc.BlockNumberOrHash) (hexutil.Uint64, error) {
+	var args ethapi.CallArgs
+	// if we actually get CallArgs here, we use them
+	if argsOrNil != nil {
+		args = *argsOrNil
+	}
+
 	bNrOrHash := rpc.BlockNumberOrHashWithNumber(rpc.PendingBlockNumber)
 	if blockNrOrHash != nil {
 		bNrOrHash = *blockNrOrHash

cmd/rpcdaemon/commands/eth_call_test.go

@@ -19,7 +19,7 @@ func TestEstimateGas(t *testing.T) {
 	api := NewEthAPI(NewBaseApi(nil, stateCache, false), db, nil, nil, nil, 5000000)
 	var from = common.HexToAddress("0x71562b71999873db5b286df957af199ec94617f7")
 	var to = common.HexToAddress("0x0d3ab14bbad3d99f4203bd7a11acb94882050e7e")
-	if _, err := api.EstimateGas(context.Background(), ethapi.CallArgs{
+	if _, err := api.EstimateGas(context.Background(), &ethapi.CallArgs{
 		From: &from,
 		To:   &to,
 	}, nil); err != nil {

cmd/rpcdaemon/commands/eth_filters.go

@@ -7,6 +7,7 @@ import (
 	"github.com/ledgerwatch/erigon/common/debug"
 	"github.com/ledgerwatch/erigon/common/hexutil"
 	"github.com/ledgerwatch/erigon/core/types"
+	"github.com/ledgerwatch/erigon/eth/filters"
 	"github.com/ledgerwatch/erigon/rpc"
 	"github.com/ledgerwatch/log/v3"
 )
@@ -110,3 +111,38 @@ func (api *APIImpl) NewPendingTransactions(ctx context.Context) (*rpc.Subscripti
 
 	return rpcSub, nil
 }
+
+// SubscribeLogs send a notification each time a new log appears.
+func (api *APIImpl) Logs(ctx context.Context, crit filters.FilterCriteria) (*rpc.Subscription, error) {
+	if api.filters == nil {
+		return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported
+	}
+	notifier, supported := rpc.NotifierFromContext(ctx)
+	if !supported {
+		return &rpc.Subscription{}, rpc.ErrNotificationsUnsupported
+	}
+
+	rpcSub := notifier.CreateSubscription()
+
+	go func() {
+		defer debug.LogPanic()
+		logs := make(chan *types.Log, 1)
+		defer close(logs)
+		id := api.filters.SubscribeLogs(logs, crit)
+		defer api.filters.UnsubscribeLogs(id)
+
+		for {
+			select {
+			case h := <-logs:
+				err := notifier.Notify(rpcSub.ID, h)
+				if err != nil {
+					log.Warn("error while notifying subscription", "err", err)
+				}
+			case <-rpcSub.Err():
+				return
+			}
+		}
+	}()
+
+	return rpcSub, nil
+}

cmd/rpcdaemon/commands/tracing.go

@@ -11,6 +11,7 @@ import (
 	"github.com/ledgerwatch/erigon/core/rawdb"
 	"github.com/ledgerwatch/erigon/core/state"
 	"github.com/ledgerwatch/erigon/core/types"
+	"github.com/ledgerwatch/erigon/core/vm"
 	"github.com/ledgerwatch/erigon/eth/tracers"
 	"github.com/ledgerwatch/erigon/ethdb"
 	"github.com/ledgerwatch/erigon/internal/ethapi"
@@ -65,7 +66,7 @@ func (api *PrivateDebugAPIImpl) traceBlock(ctx context.Context, blockNrOrHash rp
 		return rawdb.ReadHeader(tx, hash, number)
 	}
 
-	_, blockCtx, txCtx, ibs, reader, err := transactions.ComputeTxEnv(ctx, block, chainConfig, getHeader, contractHasTEVM, ethash.NewFaker(), tx, block.Hash(), 0)
+	_, blockCtx, _, ibs, reader, err := transactions.ComputeTxEnv(ctx, block, chainConfig, getHeader, contractHasTEVM, ethash.NewFaker(), tx, block.Hash(), 0)
 	if err != nil {
 		stream.WriteNil()
 		return err
@@ -82,6 +83,11 @@ func (api *PrivateDebugAPIImpl) traceBlock(ctx context.Context, blockNrOrHash rp
 		}
 		ibs.Prepare(tx.Hash(), block.Hash(), idx)
 		msg, _ := tx.AsMessage(*signer, block.BaseFee())
+		txCtx := vm.TxContext{
+			TxHash:   tx.Hash(),
+			Origin:   msg.From(),
+			GasPrice: msg.GasPrice().ToBig(),
+		}
 
 		transactions.TraceTx(ctx, msg, blockCtx, txCtx, ibs, config, chainConfig, stream)
 		_ = ibs.FinalizeTx(chainConfig.Rules(blockCtx.BlockNumber), reader)

cmd/rpcdaemon/filters/filters.go

@@ -9,8 +9,14 @@ import (
 	"io"
 	"reflect"
 	"sync"
+	"sync/atomic"
 	"time"
 
+	"github.com/ledgerwatch/erigon-lib/gointerfaces"
+	"github.com/ledgerwatch/erigon/common"
+	"github.com/ledgerwatch/erigon/eth/filters"
+
+	"github.com/ledgerwatch/erigon-lib/gointerfaces/grpcutil"
 	"github.com/ledgerwatch/erigon-lib/gointerfaces/remote"
 	"github.com/ledgerwatch/erigon-lib/gointerfaces/txpool"
 	"github.com/ledgerwatch/erigon/cmd/rpcdaemon/services"
@@ -28,6 +34,7 @@ type (
 	PendingLogsSubID  SubscriptionID
 	PendingBlockSubID SubscriptionID
 	PendingTxsSubID   SubscriptionID
+	LogsSubID         uint64
 )
 
 type Filters struct {
@@ -39,6 +46,8 @@ type Filters struct {
 	pendingLogsSubs  map[PendingLogsSubID]chan types.Logs
 	pendingBlockSubs map[PendingBlockSubID]chan *types.Block
 	pendingTxsSubs   map[PendingTxsSubID]chan []types.Transaction
+	logsSubs         *LogsFilterAggregator
+	logsRequestor    atomic.Value
 }
 
 func New(ctx context.Context, ethBackend services.ApiBackend, txPool txpool.TxpoolClient, mining txpool.MiningClient) *Filters {
@@ -49,6 +58,7 @@ func New(ctx context.Context, ethBackend services.ApiBackend, txPool txpool.Txpo
 		pendingTxsSubs:   make(map[PendingTxsSubID]chan []types.Transaction),
 		pendingLogsSubs:  make(map[PendingLogsSubID]chan types.Logs),
 		pendingBlockSubs: make(map[PendingBlockSubID]chan *types.Block),
+		logsSubs:         NewLogsFilterAggregator(),
 	}
 
 	go func() {
@@ -83,6 +93,31 @@ func New(ctx context.Context, ethBackend services.ApiBackend, txPool txpool.Txpo
 		}
 	}()
 
+	go func() {
+		if ethBackend == nil {
+			return
+		}
+		for {
+			select {
+			case <-ctx.Done():
+				return
+			default:
+			}
+			if err := ethBackend.SubscribeLogs(ctx, ff.OnNewLogs, &ff.logsRequestor); err != nil {
+				select {
+				case <-ctx.Done():
+					return
+				default:
+				}
+				if grpcutil.IsEndOfStream(err) || grpcutil.IsRetryLater(err) {
+					time.Sleep(3 * time.Second)
+					continue
+				}
+				log.Warn("rpc filters: error subscribing to logs", "err", err)
+			}
+		}
+	}()
+
 	if txPool != nil {
 		go func() {
 			for {
@@ -337,6 +372,73 @@ func (ff *Filters) UnsubscribePendingTxs(id PendingTxsSubID) {
 	delete(ff.pendingTxsSubs, id)
 }
 
+func (ff *Filters) SubscribeLogs(out chan *types.Log, crit filters.FilterCriteria) LogsSubID {
+	id, f := ff.logsSubs.insertLogsFilter(out)
+	f.addrs = map[common.Address]int{}
+	if len(crit.Addresses) == 0 {
+		f.allAddrs = 1
+	} else {
+		for _, addr := range crit.Addresses {
+			f.addrs[addr] = 1
+		}
+	}
+	f.topics = map[common.Hash]int{}
+	if len(crit.Topics) == 0 {
+		f.allTopics = 1
+	} else {
+		for _, topics := range crit.Topics {
+			for _, topic := range topics {
+				f.topics[topic] = 1
+			}
+		}
+	}
+	f.topicsOriginal = crit.Topics
+	ff.logsSubs.addLogsFilters(f)
+	lfr := &remote.LogsFilterRequest{
+		AllAddresses: ff.logsSubs.aggLogsFilter.allAddrs == 1,
+		AllTopics:    ff.logsSubs.aggLogsFilter.allTopics == 1,
+	}
+	for addr := range ff.logsSubs.aggLogsFilter.addrs {
+		lfr.Addresses = append(lfr.Addresses, gointerfaces.ConvertAddressToH160(addr))
+	}
+	for topic := range ff.logsSubs.aggLogsFilter.topics {
+		lfr.Topics = append(lfr.Topics, gointerfaces.ConvertHashToH256(topic))
+	}
+	ff.mu.Lock()
+	defer ff.mu.Unlock()
+	loaded := ff.logsRequestor.Load()
+	if loaded != nil {
+		if err := loaded.(func(*remote.LogsFilterRequest) error)(lfr); err != nil {
+			log.Warn("Could not update remote logs filter", "err", err)
+			ff.logsSubs.removeLogsFilter(id)
+		}
+	}
+	return id
+}
+
+func (ff *Filters) UnsubscribeLogs(id LogsSubID) {
+	ff.logsSubs.removeLogsFilter(id)
+	lfr := &remote.LogsFilterRequest{
+		AllAddresses: ff.logsSubs.aggLogsFilter.allAddrs == 1,
+		AllTopics:    ff.logsSubs.aggLogsFilter.allTopics == 1,
+	}
+	for addr := range ff.logsSubs.aggLogsFilter.addrs {
+		lfr.Addresses = append(lfr.Addresses, gointerfaces.ConvertAddressToH160(addr))
+	}
+	for topic := range ff.logsSubs.aggLogsFilter.topics {
+		lfr.Topics = append(lfr.Topics, gointerfaces.ConvertHashToH256(topic))
+	}
+	ff.mu.Lock()
+	defer ff.mu.Unlock()
+	loaded := ff.logsRequestor.Load()
+	if loaded != nil {
+		if err := loaded.(func(*remote.LogsFilterRequest) error)(lfr); err != nil {
+			log.Warn("Could not update remote logs filter", "err", err)
+			ff.logsSubs.removeLogsFilter(id)
+		}
+	}
+}
+
 func (ff *Filters) OnNewEvent(event *remote.SubscribeReply) {
 	ff.mu.RLock()
 	defer ff.mu.RUnlock()
@@ -411,6 +513,25 @@ func (ff *Filters) OnNewTx(reply *txpool.OnAddReply) {
 	}
 }
 
+func (ff *Filters) OnNewLogs(reply *remote.SubscribeLogsReply) {
+	lg := &types.Log{
+		Address:     gointerfaces.ConvertH160toAddress(reply.Address),
+		Data:        reply.Data,
+		BlockNumber: reply.BlockNumber,
+		TxHash:      gointerfaces.ConvertH256ToHash(reply.TransactionHash),
+		TxIndex:     uint(reply.TransactionIndex),
+		BlockHash:   gointerfaces.ConvertH256ToHash(reply.BlockHash),
+		Index:       uint(reply.LogIndex),
+		Removed:     reply.Removed,
+	}
+	t := make([]common.Hash, 0)
+	for _, v := range reply.Topics {
+		t = append(t, gointerfaces.ConvertH256ToHash(v))
+	}
+	lg.Topics = t
+	ff.logsSubs.distributeLog(reply)
+}
+
 func generateSubscriptionID() SubscriptionID {
 	var id [32]byte