Merge upstream 20231030

.github/workflows/test-integration.yml

@@ -34,6 +34,11 @@ jobs:
       - name: test-integration
         run: make test-integration
 
+      - name: Test erigon as a library
+        env:
+          GIT_COMMIT: ${{ github.event.pull_request.head.sha || github.sha }}
+        run: make test-erigon-ext GIT_COMMIT=$GIT_COMMIT
+
   #        name: history-v3-test-integration
   #        run: make test3-integration
 

.golangci.yml

@@ -81,7 +81,7 @@ linters-settings:
         rules: "rules.go"
       hugeParam:
         # size in bytes that makes the warning trigger (default 80)
-        sizeThreshold: 1000
+        sizeThreshold: 1100
       rangeExprCopy:
         # size in bytes that makes the warning trigger (default 512)
         sizeThreshold: 512

Makefile

@@ -25,8 +25,7 @@ CGO_CFLAGS += -DMDBX_FORCE_ASSERTIONS=0 # Enable MDBX's asserts by default in 'd
 #CGO_CFLAGS += -DMDBX_ENV_CHECKPID=0 # Erigon doesn't do fork() syscall
 CGO_CFLAGS += -O
 CGO_CFLAGS += -D__BLST_PORTABLE__
-CGO_CFLAGS += -Wno-unknown-warning-option -Wno-enum-int-mismatch -Wno-strict-prototypes
-#CGO_CFLAGS += -Wno-error=strict-prototypes # for Clang15, remove it when can https://github.com/ledgerwatch/erigon/issues/6113#issuecomment-1359526277
+CGO_CFLAGS += -Wno-unknown-warning-option -Wno-enum-int-mismatch -Wno-strict-prototypes -Wno-unused-but-set-variable
 
 # about netgo see: https://github.com/golang/go/issues/30310#issuecomment-471669125 and https://github.com/golang/go/issues/57757
 BUILD_TAGS = nosqlite,noboltdb
@@ -142,22 +141,24 @@ db-tools:
 	rm -rf vendor
 	@echo "Run \"$(GOBIN)/mdbx_stat -h\" to get info about mdbx db file."
 
-## test:                              run unit tests with a 100s timeout
-test:
+test-erigon-lib:
 	@cd erigon-lib && $(MAKE) test
+
+test-erigon-ext:
+	@cd tests/erigon-ext-test && ./test.sh $(GIT_COMMIT)
+
+## test:                              run unit tests with a 100s timeout
+test: test-erigon-lib
 	$(GOTEST) --timeout 10m
 
-test3:
-	@cd erigon-lib && $(MAKE) test
+test3: test-erigon-lib
 	$(GOTEST) --timeout 10m -tags $(BUILD_TAGS),e3
 
 ## test-integration:                  run integration tests with a 30m timeout
-test-integration:
-	@cd erigon-lib && $(MAKE) test
+test-integration: test-erigon-lib
 	$(GOTEST) --timeout 240m -tags $(BUILD_TAGS),integration
 
-test3-integration:
-	@cd erigon-lib && $(MAKE) test
+test3-integration: test-erigon-lib
 	$(GOTEST) --timeout 240m -tags $(BUILD_TAGS),integration,e3
 
 ## lint-deps:                         install lint dependencies

README.md

@@ -55,9 +55,10 @@ System Requirements
 
 * Goerli Full node (see `--prune*` flags): 189GB on Beta, 114GB on Alpha (April 2022).
 
-* Gnosis Chain Archive: 370GB (January 2023).
+* Gnosis Chain Archive: 600GB (October 2023).
 
-* Polygon Mainnet Archive: 5TB. (April 2022). `--prune.*.older 15768000`: 5.1Tb (Sept 2023). Polygon Mumbai Archive: 1TB. (April 2022).
+* Polygon Mainnet Archive: 5TB. (April 2022). `--prune.*.older 15768000`: 5.1Tb (Sept 2023). Polygon Mumbai Archive:
+  1TB. (April 2022).
 
 SSD or NVMe. Do not recommend HDD - on HDD Erigon will always stay N blocks behind chain tip, but not fall behind.
 Bear in mind that SSD performance deteriorates when close to capacity.
@@ -111,7 +112,8 @@ Running `make help` will list and describe the convenience commands available in
 ### Datadir structure
 
 - chaindata: recent blocks, state, recent state history. low-latency disk recommended.
-- snapshots: old blocks, old state history. can symlink/mount it to cheaper disk. mostly immutable. must have ~100gb free space (for merge recent files to bigger one).
+- snapshots: old blocks, old state history. can symlink/mount it to cheaper disk. mostly immutable. must have ~100gb
+  free space (for merge recent files to bigger one).
 - temp: can grow to ~100gb, but usually empty. can symlink/mount it to cheaper disk.
 - txpool: pending transactions. safe to remove.
 - nodes:  p2p peers. safe to remove.
@@ -141,7 +143,9 @@ int value specifying the highest output log level:
   LvlTrace = 5
 ```
 
-To set an output dir for logs to be collected on disk, please set `--log.dir.path` If you want to change the filename produced from `erigon` you should also set the `--log.dir.prefix` flag to an alternate name. The flag `--log.dir.verbosity` is
+To set an output dir for logs to be collected on disk, please set `--log.dir.path` If you want to change the filename
+produced from `erigon` you should also set the `--log.dir.prefix` flag to an alternate name. The
+flag `--log.dir.verbosity` is
 also available to control the verbosity of this logging, with the same int value as above, or the string value e.g. '
 debug' or 'info'. Default verbosity is 'debug' (4), for disk logging.
 
@@ -530,13 +534,10 @@ Detailed explanation: [./docs/programmers_guide/db_faq.md](./docs/programmers_gu
 
 ### Default Ports and Firewalls
 
-
-
 #### `erigon` ports
 
-
 | Component | Port  | Protocol  | Purpose                     | Should Expose |
-| --------- | ----- | --------- | --------------------------- | ------------- |
+|-----------|-------|-----------|-----------------------------|---------------|
 | engine    | 9090  | TCP       | gRPC Server                 | Private       |
 | engine    | 42069 | TCP & UDP | Snap sync (Bittorrent)      | Public        |
 | engine    | 8551  | TCP       | Engine API (JWT auth)       | Private       |
@@ -545,40 +546,31 @@ Detailed explanation: [./docs/programmers_guide/db_faq.md](./docs/programmers_gu
 | sentry    | 9091  | TCP       | incoming gRPC Connections   | Private       |
 | rpcdaemon | 8545  | TCP       | HTTP & WebSockets & GraphQL | Private       |
 
-
-
-
 Typically, 30303 and 30304 are exposed to the internet to allow incoming peering connections. 9090 is exposed only
 internally for rpcdaemon or other connections, (e.g. rpcdaemon -> erigon).
 Port 8551 (JWT authenticated) is exposed only internally for [Engine API] JSON-RPC queries from the Consensus Layer
 node.
 
-
-
-
-
 #### `caplin` ports
+
 | Component | Port | Protocol | Purpose          | Should Expose |
-| --------- | ---- | -------- | ---------------- | ------------- |
+|-----------|------|----------|------------------|---------------|
 | sentinel  | 4000 | UDP      | Peering          | Public        |
 | sentinel  | 4001 | TCP      | Peering          | Public        |
 
 If you are using `--internalcl` aka `caplin` as your consensus client, then also look at the chart above
 
-
 #### `shared` ports
 
-| Component | Port  | Protocol  | Purpose                     | Should Expose |
-| --------- | ----- | --------- | --------------------------- | ------------- |
-| all       | 6060 | TCP      | pprof            | Private       |
-| all       | 6060 | TCP      | metrics          | Private       |
-
+| Component | Port | Protocol | Purpose | Should Expose |
+|-----------|------|----------|---------|---------------|
+| all       | 6060 | TCP      | pprof   | Private       |
+| all       | 6060 | TCP      | metrics | Private       |
 
 Optional flags can be enabled that enable pprof or metrics (or both) - however, they both run on 6060 by default, so
 
 you'll have to change one if you want to run both at the same time. use `--help` with the binary for more info.
 
-
 #### `other` ports
 
 Reserved for future use: **gRPC ports**: `9092` consensus engine, `9093` snapshot downloader, `9094` TxPool
@@ -606,15 +598,16 @@ Device Benchmark Testing   RFC 2544
 RFC 922, Section 7
 ```
 
-Same in [IpTables syntax](https://ethereum.stackexchange.com/questions/6386/how-to-prevent-being-blacklisted-for-running-an-ethereum-client/13068#13068)
+Same
+in [IpTables syntax](https://ethereum.stackexchange.com/questions/6386/how-to-prevent-being-blacklisted-for-running-an-ethereum-client/13068#13068)
 
 ### How to get diagnostic for bug report?
 
 - Get stack trace: `kill -SIGUSR1 <pid>`, get trace and stop: `kill -6 <pid>`
 - Get CPU profiling: add `--pprof flag`
-    run `go tool pprof -png  http://127.0.0.1:6060/debug/pprof/profile\?seconds\=20 > cpu.png`
+  run `go tool pprof -png  http://127.0.0.1:6060/debug/pprof/profile\?seconds\=20 > cpu.png`
 - Get RAM profiling: add `--pprof flag`
-    run `go tool pprof -inuse_space -png  http://127.0.0.1:6060/debug/pprof/heap > mem.png`
+  run `go tool pprof -inuse_space -png  http://127.0.0.1:6060/debug/pprof/heap > mem.png`
 
 ### How to run local devnet?
 
@@ -628,6 +621,13 @@ UID/GID (1000).
 More details
 in [post](https://www.fullstaq.com/knowledge-hub/blogs/docker-and-the-host-filesystem-owner-matching-problem)
 
+### How to run public RPC api
+
+- `--txpool.nolocals=true`
+- don't add `admin` in `--http.api` list
+- to increase throughput may need
+  increase/decrease: `--db.read.concurrency`, `--rpc.batch.concurrency`, `--rpc.batch.limit`
+
 ### Run RaspberyPI
 
 https://github.com/mathMakesArt/Erigon-on-RPi-4
@@ -642,7 +642,8 @@ Getting in touch
 
 ### Erigon Discord Server
 
-The main discussions are happening on our Discord server. To get an invite, send an email to `bloxster [at] proton.me` with
+The main discussions are happening on our Discord server. To get an invite, send an email to `bloxster [at] proton.me`
+with
 your name, occupation, a brief explanation of why you want to join the Discord, and how you heard about Erigon.
 
 ### Reporting security issues/concerns
@@ -669,20 +670,22 @@ Next tools show correct memory usage of Erigon:
 
 - `vmmap -summary PID | grep -i "Physical footprint"`. Without `grep` you can see details
     - `section MALLOC ZONE column Resident Size` shows App memory usage, `section REGION TYPE column Resident Size`
-        shows OS pages cache size.
+      shows OS pages cache size.
 - `Prometheus` dashboard shows memory of Go app without OS pages cache (`make prometheus`, open in
-    browser `localhost:3000`, credentials `admin/admin`)
+  browser `localhost:3000`, credentials `admin/admin`)
 - `cat /proc/<PID>/smaps`
 
-    Erigon uses ~4Gb of RAM during genesis sync and ~1Gb during normal work. OS pages cache can utilize unlimited amount of
-    memory.
-
-    **Warning:** Multiple instances of Erigon on same machine will touch Disk concurrently, it impacts performance - one of
-    main Erigon optimisations: "reduce Disk random access".
-    "Blocks Execution stage" still does many random reads - this is reason why it's slowest stage. We do not recommend
-    running
-    multiple genesis syncs on same Disk. If genesis sync passed, then it's fine to run multiple Erigon instances on same
-    Disk.
+  Erigon uses ~4Gb of RAM during genesis sync and ~1Gb during normal work. OS pages cache can utilize unlimited amount
+  of
+  memory.
+
+  **Warning:** Multiple instances of Erigon on same machine will touch Disk concurrently, it impacts performance - one
+  of
+  main Erigon optimisations: "reduce Disk random access".
+  "Blocks Execution stage" still does many random reads - this is reason why it's slowest stage. We do not recommend
+  running
+  multiple genesis syncs on same Disk. If genesis sync passed, then it's fine to run multiple Erigon instances on same
+  Disk.
 
 ### Blocks Execution is slow on cloud-network-drives
 

cl/cltypes/beacon_block.go

@@ -94,7 +94,9 @@ func NewBeaconBlock(beaconCfg *clparams.BeaconChainConfig) *BeaconBlock {
 }
 
 func NewBeaconBody(beaconCfg *clparams.BeaconChainConfig) *BeaconBody {
-	return &BeaconBody{beaconCfg: beaconCfg}
+	return &BeaconBody{
+		beaconCfg: beaconCfg,
+	}
 }
 
 // Version returns beacon block version.
@@ -172,6 +174,8 @@ func (b *BeaconBody) DecodeSSZ(buf []byte, version int) error {
 		return fmt.Errorf("[BeaconBody] err: %s", ssz.ErrLowBufferSize)
 	}
 
+	b.ExecutionPayload = NewEth1Block(b.Version, b.beaconCfg)
+
 	err := ssz2.UnmarshalSSZ(buf, version, b.getSchema(false)...)
 	return err
 }

cl/persistence/format/chunk_encoding/chunks.go

@@ -28,15 +28,40 @@ func WriteChunk(w io.Writer, buf []byte, t DataType) error {
 	return nil
 }
 
-func ReadChunk(r io.Reader) (buf []byte, t DataType, err error) {
+func ReadChunk(r io.Reader, out io.Writer) (t DataType, err error) {
+	prefix := make([]byte, 8)
+	if _, err := r.Read(prefix); err != nil {
+		return DataType(0), err
+	}
+	t = DataType(prefix[0])
+	prefix[0] = 0
+
+	bufLen := binary.BigEndian.Uint64(prefix)
+	if bufLen == 0 {
+		return
+	}
+
+	if _, err = io.CopyN(out, r, int64(bufLen)); err != nil {
+		return
+	}
+	return
+}
+
+func ReadChunkToBytes(r io.Reader) (b []byte, t DataType, err error) {
 	prefix := make([]byte, 8)
 	if _, err := r.Read(prefix); err != nil {
 		return nil, DataType(0), err
 	}
 	t = DataType(prefix[0])
 	prefix[0] = 0
-	buf = make([]byte, binary.BigEndian.Uint64(prefix))
-	if _, err = r.Read(buf); err != nil {
+
+	bufLen := binary.BigEndian.Uint64(prefix)
+	if bufLen == 0 {
+		return
+	}
+	b = make([]byte, bufLen)
+
+	if _, err = r.Read(b); err != nil {
 		return
 	}
 	return