Merge pre v2.56

.github/workflows/ci.yml

@@ -27,7 +27,7 @@ jobs:
     if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
     strategy:
       matrix:
-        os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments
+        os: [ ubuntu-22.04 ] # Disabling macos as getting inexplicable "this job failed" messages with no logs, macos-13-xlarge ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -56,7 +56,7 @@ jobs:
         if: runner.os == 'Linux'
         uses: golangci/golangci-lint-action@v3
         with:
-          version: v1.54.2
+          version: v1.55.2
           skip-build-cache: true
           args: --help
 
@@ -96,8 +96,8 @@ jobs:
           key: chocolatey-${{ matrix.os }}
       - name: Install dependencies
         run: |
-          choco upgrade mingw -y --no-progress --version 11.2.0.07112021
-          choco install cmake -y --no-progress --version 3.23.1
+          choco upgrade mingw -y --no-progress --version 13.2.0
+          choco install cmake -y --no-progress --version 3.27.8
 
       - name: Build
         run: .\wmake.ps1 all
@@ -111,7 +111,7 @@ jobs:
   docker-build-check:
     # don't run this on devel - the PR must have run it to be merged and it misleads that this pushes the docker image
     if: (${{ github.event_name == 'push' || !github.event.pull_request.draft }}) && ${{ github.ref != 'refs/heads/devel' }}
-    runs-on: ubuntu-20.04
+    runs-on: ubuntu-22.04
     steps:
       - uses: AutoModality/action-clean@v1
       - uses: actions/checkout@v3
@@ -127,7 +127,7 @@ jobs:
 
 #  automated-tests:
 #    runs-on:
-#      ubuntu-20.04
+#      ubuntu-22.04
 #    if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
 #    steps:
 #      - uses: actions/checkout@v3

.github/workflows/qa-clean-exit.yml

@@ -0,0 +1,89 @@
+name: QA - Clean exit on Ctrl-C
+
+on:
+  push:
+    branches:
+      - devel
+      - alpha
+      - 'release/**'
+  pull_request:
+    branches:
+      - devel
+      - alpha
+      - 'release/**'
+    types:
+      - opened
+      - reopened
+      - synchronize
+      - ready_for_review
+
+jobs:
+  long-running-test:
+    #if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
+    #strategy:
+    #  matrix:
+    #    os: [ ubuntu-22.04, macos-13-xlarge ]
+    #runs-on: ${{ matrix.os }}
+    runs-on: self-hosted
+    env:
+      ERIGON_DATA_DIR: ${{ github.workspace }}/erigon_data
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.20'
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+
+    #- name: Install dependencies
+    #  run: |
+    #    sudo apt-get update
+    #    sudo apt-get install -y build-essential make gcc
+
+    - name: Clean Erigon Build Directory
+      run: |
+        make clean
+
+    - name: Build Erigon
+      run: |
+        make erigon
+      working-directory: ${{ github.workspace }}
+
+    #- name: Download Python Script for Logs Checking
+    #  run: |
+    #    curl -o check_erigon_exit.py 'https://gist.githubusercontent.com/mriccobene/8db4030a745de34d527f136f2caa104f/raw/3c1a860cb87d61075e78ce399e17f0ab157cacc6/check_erigon_exit.py'
+
+    - name: Run Erigon, send ctrl-c and check for clean exiting
+      run: |
+        # Run Erigon, send ctrl-c and check logs
+        python3 ${{ github.workspace }}/../../../../erigon-qa/test_system/qa-tests/clean-exit/run_and_check_clean_exit.py ${{ github.workspace }}/build/bin $ERIGON_DATA_DIR
+
+        # Capture monitoring script exit status
+        monitoring_exit_status=$?
+
+        # Clean up Erigon process if it's still running
+        if kill -0 $ERIGON_PID 2> /dev/null; then
+          echo "Terminating Erigon"
+          kill $ERIGON_PID
+          wait $ERIGON_PID
+        else
+          echo "Erigon has already terminated"
+        fi
+
+        # Clean up Erigon build and data directories
+        rm -rf $ERIGON_DATA_DIR
+
+        # Check monitoring script exit status
+        if [ $monitoring_exit_status -eq 0 ]; then
+          echo "Monitoring completed successfully"
+        else
+          echo "Error detected in Erigon logs or monitoring script exited unexpectedly"
+          exit 1
+        fi

.github/workflows/test-integration-caplin.yml

@@ -20,7 +20,8 @@ jobs:
   tests:
     strategy:
       matrix:
-        os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments
+#        disable macos-11 until https://github.com/ledgerwatch/erigon/issues/8789
+        os: [ ubuntu-22.04 ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -55,8 +56,8 @@ jobs:
           key: chocolatey-${{ matrix.os }}
       - name: Install dependencies
         run: |
-          choco upgrade mingw -y --no-progress --version 11.2.0.07112021
-          choco install cmake -y --no-progress --version 3.23.1
+          choco upgrade mingw -y --no-progress --version 13.2.0
+          choco install cmake -y --no-progress --version 3.27.8
 
       - name: test-integration-caplin
         run: cd ./cl/spectest/ && .\wmake.ps1 Tests Mainnet

.github/workflows/test-integration.yml

@@ -18,7 +18,7 @@ jobs:
   tests:
     strategy:
       matrix:
-        os: [ ubuntu-20.04, macos-11 ] # list of os: https://github.com/actions/virtual-environments
+        os: [ ubuntu-22.04 ] # Disable macos as it is giving us "This Job Failed" errors with no logs, macos-13-xlarge ] # list of os: https://github.com/actions/virtual-environments
     runs-on: ${{ matrix.os }}
 
     steps:
@@ -63,8 +63,8 @@ jobs:
           key: chocolatey-${{ matrix.os }}
       - name: Install dependencies
         run: |
-          choco upgrade mingw -y --no-progress --version 11.2.0.07112021
-          choco install cmake -y --no-progress --version 3.23.1
+          choco upgrade mingw -y --no-progress --version 13.2.0
+          choco install cmake -y --no-progress --version 3.27.8
 
       - name: test-integration
         run: .\wmake.ps1 test-integration

.gitignore

@@ -94,3 +94,7 @@ jwt.hex
 *__debug_bin*
 yarn.lock
 node_modules
+
+/config.toml
+/config.yaml
+/config.yml

.golangci.yml

@@ -25,7 +25,10 @@ linters:
     - errorlint #TODO: enable me
     - errchkjson #TODO: enable me
     - unused #TODO: enable me
+    - testifylint #TODO: enable me
+    - perfsprint #TODO: enable me
     - gocheckcompilerdirectives
+    - protogetter
   enable:
     - unconvert
     #    - predeclared #TODO: enable me
@@ -81,7 +84,7 @@ linters-settings:
         rules: "rules.go"
       hugeParam:
         # size in bytes that makes the warning trigger (default 80)
-        sizeThreshold: 1100
+        sizeThreshold: 1000
       rangeExprCopy:
         # size in bytes that makes the warning trigger (default 512)
         sizeThreshold: 512
@@ -124,6 +127,7 @@ issues:
         - unused
         - deadcode
         - gocritic
+        - perfsprint
     - path: hack\.go
       linters:
         - gosec

.goreleaser.yml

@@ -14,7 +14,7 @@ builds:
     env:
       - CC=o64-clang
       - CXX=o64-clang++
-    tags: [ nosqlite, noboltdb ]
+    tags: [ nosqlite, noboltdb, nosilkworm ]
     ldflags: -s -w
 
   - id: darwin-arm64
@@ -25,7 +25,7 @@ builds:
     env:
       - CC=oa64-clang
       - CXX=oa64-clang++
-    tags: [ nosqlite, noboltdb ]
+    tags: [ nosqlite, noboltdb, nosilkworm ]
     ldflags: -s -w
 
   - id: linux-amd64
@@ -36,7 +36,7 @@ builds:
     env:
       - CC=x86_64-linux-gnu-gcc
       - CXX=x86_64-linux-gnu-g++
-    tags: [ nosqlite, noboltdb ]
+    tags: [ nosqlite, noboltdb, nosilkworm ]
     ldflags: -s -w -extldflags "-static" # We need to build a static binary because we are building in a glibc based system and running in a musl container
 
   - id: linux-arm64
@@ -47,7 +47,7 @@ builds:
     env:
       - CC=aarch64-linux-gnu-gcc
       - CXX=aarch64-linux-gnu-g++
-    tags: [ nosqlite, noboltdb ]
+    tags: [ nosqlite, noboltdb, nosilkworm ]
     ldflags: -s -w -extldflags "-static" # We need to build a static binary because we are building in a glibc based system and running in a musl container
 
   - id: windows-amd64
@@ -58,7 +58,7 @@ builds:
     env:
       - CC=x86_64-w64-mingw32-gcc
       - CXX=x86_64-w64-mingw32-g++
-    tags: [ nosqlite, noboltdb ]
+    tags: [ nosqlite, noboltdb, nosilkworm ]
     ldflags: -s -w
 
 

Dockerfile

@@ -15,7 +15,7 @@ ADD . .
 RUN --mount=type=cache,target=/root/.cache \
     --mount=type=cache,target=/tmp/go-build \
     --mount=type=cache,target=/go/pkg/mod \
-    make all
+    make BUILD_TAGS=nosqlite,noboltdb,nosilkworm all
 
 
 FROM docker.io/library/golang:1.20-alpine3.17 AS tools-builder

Makefile

@@ -27,16 +27,28 @@ CGO_CFLAGS += -O
 CGO_CFLAGS += -D__BLST_PORTABLE__
 CGO_CFLAGS += -Wno-unknown-warning-option -Wno-enum-int-mismatch -Wno-strict-prototypes -Wno-unused-but-set-variable
 
+CGO_LDFLAGS := $(shell $(GO) env CGO_LDFLAGS 2> /dev/null)
+ifeq ($(shell uname -s), Darwin)
+	ifeq ($(filter-out 13.%,$(shell sw_vers --productVersion)),)
+		CGO_LDFLAGS += -mmacosx-version-min=13.3
+	endif
+endif
+
 # about netgo see: https://github.com/golang/go/issues/30310#issuecomment-471669125 and https://github.com/golang/go/issues/57757
 BUILD_TAGS = nosqlite,noboltdb
+
+ifneq ($(shell "$(CURDIR)/turbo/silkworm/silkworm_compat_check.sh"),)
+	BUILD_TAGS := $(BUILD_TAGS),nosilkworm
+endif
+
 PACKAGE = github.com/ledgerwatch/erigon
 
 GO_FLAGS += -trimpath -tags $(BUILD_TAGS) -buildvcs=false
 GO_FLAGS += -ldflags "-X ${PACKAGE}/params.GitCommit=${GIT_COMMIT} -X ${PACKAGE}/params.GitBranch=${GIT_BRANCH} -X ${PACKAGE}/params.GitTag=${GIT_TAG}"
 
-GOBUILD = CGO_CFLAGS="$(CGO_CFLAGS)" $(GO) build $(GO_FLAGS)
-GO_DBG_BUILD = CGO_CFLAGS="$(CGO_CFLAGS) -DMDBX_DEBUG=1" $(GO) build -tags $(BUILD_TAGS),debug -gcflags=all="-N -l"  # see delve docs
-GOTEST = CGO_CFLAGS="$(CGO_CFLAGS)" GODEBUG=cgocheck=0 $(GO) test $(GO_FLAGS) ./... -p 2
+GOBUILD = CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) build $(GO_FLAGS)
+GO_DBG_BUILD = CGO_CFLAGS="$(CGO_CFLAGS) -DMDBX_DEBUG=1" CGO_LDFLAGS="$(CGO_LDFLAGS)" $(GO) build -tags $(BUILD_TAGS),debug -gcflags=all="-N -l"  # see delve docs
+GOTEST = CGO_CFLAGS="$(CGO_CFLAGS)" CGO_LDFLAGS="$(CGO_LDFLAGS)" GODEBUG=cgocheck=0 $(GO) test $(GO_FLAGS) ./... -p 2
 
 default: all
 
@@ -122,6 +134,8 @@ COMMANDS += evm
 COMMANDS += sentinel
 COMMANDS += caplin
 COMMANDS += caplin-regression
+COMMANDS += tooling
+
 
 
 # build each command using %.cmd rule
@@ -224,6 +238,16 @@ git-submodules:
 	@git submodule sync --quiet --recursive || true
 	@git submodule update --quiet --init --recursive --force || true
 
+## install:                            copies binaries and libraries to DIST
+DIST ?= $(CURDIR)/build/dist
+.PHONY: install
+install:
+	mkdir -p "$(DIST)"
+	cp -f "$$($(CURDIR)/turbo/silkworm/silkworm_lib_path.sh)" "$(DIST)"
+	cp -f "$(GOBIN)/"* "$(DIST)"
+	@echo "Copied files to $(DIST):"
+	@ls -al "$(DIST)"
+
 PACKAGE_NAME          := github.com/ledgerwatch/erigon
 GOLANG_CROSS_VERSION  ?= v1.20.7