Merge pull request #81 from bobanetwork/jyellick/merge-pre-v2.57

Merge pre v2.57

.github/workflows/ci.yml

@@ -27,7 +27,10 @@ jobs:
     if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
     strategy:
       matrix:
-        os: [ ubuntu-22.04 ] # Disabling macos as getting inexplicable "this job failed" messages with no logs, macos-13-xlarge ] # list of os: https://github.com/actions/virtual-environments
+        # list of os: https://github.com/actions/virtual-environments
+        os:
+          - ubuntu-22.04
+          - macos-13
     runs-on: ${{ matrix.os }}
 
     steps:

.github/workflows/qa-clean-exit-block-downloading.yml

@@ -0,0 +1,81 @@
+name: QA - Clean exit (block downloading)
+
+on:
+  push:
+    branches:
+      - 'release/**'
+  pull_request:
+    branches:
+      - devel
+      - 'release/**'
+    types:
+      - ready_for_review
+
+jobs:
+  long-running-test:
+    runs-on: self-hosted
+    env:
+      ERIGON_REFERENCE_DATA_DIR: /opt/erigon-release/datadir
+      ERIGON_TESTBED_DATA_DIR: /opt/erigon-testbed/datadir
+      WORKING_TIME_SECONDS: 600
+
+    steps:
+    - name: Check out repository
+      uses: actions/checkout@v2
+
+    - name: Set up Go
+      uses: actions/setup-go@v4
+      with:
+        go-version: '1.21'
+
+    - name: Set up Python
+      uses: actions/setup-python@v4
+      with:
+        python-version: '3.x'
+
+    #- name: Install dependencies
+    #  run: |
+    #    sudo apt-get update
+    #    sudo apt-get install -y build-essential make gcc
+
+    - name: Restore Erigon Testbed Data Directory
+      run: |
+        rsync -av --delete $ERIGON_REFERENCE_DATA_DIR/ $ERIGON_TESTBED_DATA_DIR/
+
+    - name: Clean Erigon Build Directory
+      run: |
+        make clean
+
+    - name: Build Erigon
+      run: |
+        make erigon
+      working-directory: ${{ github.workspace }}
+
+    #- name: Download Python Script for Logs Checking
+    #  run: |
+    #    curl -o check_erigon_exit.py 'https://gist.githubusercontent.com/mriccobene/8db4030a745de34d527f136f2caa104f/raw/3c1a860cb87d61075e78ce399e17f0ab157cacc6/check_erigon_exit.py'
+
+    - name: Run Erigon, send ctrl-c and check for clean exiting
+      run: |
+        # Run Erigon, send ctrl-c and check logs
+        python3 ${{ github.workspace }}/../../../../erigon-qa/test_system/qa-tests/clean-exit/run_and_check_clean_exit.py ${{ github.workspace }}/build/bin $ERIGON_TESTBED_DATA_DIR $WORKING_TIME_SECONDS
+  
+        # Capture monitoring script exit status
+        monitoring_exit_status=$?
+
+        # Clean up Erigon process if it's still running
+        if kill -0 $ERIGON_PID 2> /dev/null; then
+          echo "Terminating Erigon"
+          kill $ERIGON_PID
+          wait $ERIGON_PID
+        else
+          echo "Erigon has already terminated"
+        fi
+
+        # Check monitoring script exit status
+        if [ $monitoring_exit_status -eq 0 ]; then
+          echo "Monitoring completed successfully"
+        else
+          echo "Error detected in Erigon logs or monitoring script exited unexpectedly"
+          exit 1
+        fi

.github/workflows/qa-clean-exit.yml → .github/workflows/qa-clean-exit-snapshot-downloading.yml

@@ -1,4 +1,4 @@
-name: QA - Clean exit on Ctrl-C
+name: QA - Clean exit (snapshot downloading)
 
 on:
   push:
@@ -19,14 +19,10 @@ on:
 
 jobs:
   long-running-test:
-    #if: ${{ github.event_name == 'push' || !github.event.pull_request.draft }}
-    #strategy:
-    #  matrix:
-    #    os: [ ubuntu-22.04, macos-13-xlarge ]
-    #runs-on: ${{ matrix.os }}
     runs-on: self-hosted
     env:
       ERIGON_DATA_DIR: ${{ github.workspace }}/erigon_data
+      WORKING_TIME_SECONDS: 600
 
     steps:
     - name: Check out repository
@@ -35,7 +31,7 @@ jobs:
     - name: Set up Go
       uses: actions/setup-go@v4
       with:
-        go-version: '1.20'
+        go-version: '1.21'
 
     - name: Set up Python
       uses: actions/setup-python@v4
@@ -63,7 +59,7 @@ jobs:
     - name: Run Erigon, send ctrl-c and check for clean exiting
       run: |
         # Run Erigon, send ctrl-c and check logs
-        python3 ${{ github.workspace }}/../../../../erigon-qa/test_system/qa-tests/clean-exit/run_and_check_clean_exit.py ${{ github.workspace }}/build/bin $ERIGON_DATA_DIR
+        python3 ${{ github.workspace }}/../../../../erigon-qa/test_system/qa-tests/clean-exit/run_and_check_clean_exit.py ${{ github.workspace }}/build/bin $ERIGON_DATA_DIR $WORKING_TIME_SECONDS
   
         # Capture monitoring script exit status
         monitoring_exit_status=$?

.github/workflows/release.yml

@@ -47,7 +47,7 @@ jobs:
         id: prepare
         run: |
           TAG=${GITHUB_REF#refs/tags/}
-          echo ::set-output name=tag_name::${TAG}
+          echo "tag_name=${TAG}" >> $GITHUB_OUTPUT
 
       - name: Set up QEMU
         uses: docker/setup-qemu-action@v2

.github/workflows/test-integration.yml

@@ -18,7 +18,10 @@ jobs:
   tests:
     strategy:
       matrix:
-        os: [ ubuntu-22.04 ] # Disable macos as it is giving us "This Job Failed" errors with no logs, macos-13-xlarge ] # list of os: https://github.com/actions/virtual-environments
+        # list of os: https://github.com/actions/virtual-environments
+        os:
+          - ubuntu-22.04
+          - macos-13
     runs-on: ${{ matrix.os }}
 
     steps:

.gitignore

@@ -98,3 +98,5 @@ node_modules
 /config.toml
 /config.yaml
 /config.yml
+
+vendor

.golangci.yml

@@ -84,7 +84,7 @@ linters-settings:
         rules: "rules.go"
       hugeParam:
         # size in bytes that makes the warning trigger (default 80)
-        sizeThreshold: 1000
+        sizeThreshold: 1100
       rangeExprCopy:
         # size in bytes that makes the warning trigger (default 512)
         sizeThreshold: 512

Makefile

@@ -135,6 +135,8 @@ COMMANDS += sentinel
 COMMANDS += caplin
 COMMANDS += caplin-regression
 COMMANDS += tooling
+COMMANDS += snapshots
+
 
 
 

README.md

@@ -57,7 +57,7 @@ System Requirements
 
 * Gnosis Chain Archive: 600GB (October 2023).
 
-* Polygon Mainnet Archive: 5TB. (April 2022). `--prune.*.older 15768000`: 5.1Tb (Sept 2023). Polygon Mumbai Archive:
+* Polygon Mainnet Archive: 8.5TiB (December 2023). `--prune.*.older 15768000`: 5.1Tb (September 2023). Polygon Mumbai Archive:
   1TB. (April 2022).
 
 SSD or NVMe. Do not recommend HDD - on HDD Erigon will always stay N blocks behind chain tip, but not fall behind.

cl/abstract/beacon_state.go

@@ -22,7 +22,7 @@ type BeaconStateUpgradable interface {
 }
 
 type BeaconStateExtension interface {
-	SlashValidator(slashedInd uint64, whistleblowerInd *uint64) error
+	SlashValidator(slashedInd uint64, whistleblowerInd *uint64) (uint64, error)
 	InitiateValidatorExit(index uint64) error
 	GetActiveValidatorsIndices(epoch uint64) (indicies []uint64)
 	GetTotalActiveBalance() uint64
@@ -40,6 +40,7 @@ type BeaconStateExtension interface {
 	ValidatorIndexByPubkey(key [48]byte) (uint64, bool)
 	PreviousStateRoot() common.Hash
 	SetPreviousStateRoot(root common.Hash)
+	GetValidatorActivationChurnLimit() uint64
 }
 
 type BeaconStateBasic interface {
@@ -162,6 +163,8 @@ type BeaconStateMinimal interface {
 	Eth1Data() *cltypes.Eth1Data
 	Eth1DataVotes() *solid.ListSSZ[*cltypes.Eth1Data]
 	Eth1DepositIndex() uint64
+	ValidatorSet() *solid.ValidatorSet
+	PreviousEpochParticipation() *solid.BitList
 
 	ForEachValidator(fn func(v solid.Validator, idx int, total int) bool)
 	ValidatorForValidatorIndex(index int) (solid.Validator, error)

cl/antiquary/antiquary.go

@@ -34,7 +34,7 @@ type Antiquary struct {
 	beaconDB        persistence.BlockSource
 	backfilled      *atomic.Bool
 	cfg             *clparams.BeaconChainConfig
-	states          bool
+	states, blocks  bool
 	fs              afero.Fs
 	validatorsTable *state_accessors.StaticValidatorTable
 	genesisState    *state.CachingBeaconState
@@ -43,7 +43,7 @@ type Antiquary struct {
 	balances32   []byte
 }
 
-func NewAntiquary(ctx context.Context, genesisState *state.CachingBeaconState, validatorsTable *state_accessors.StaticValidatorTable, cfg *clparams.BeaconChainConfig, dirs datadir.Dirs, downloader proto_downloader.DownloaderClient, mainDB kv.RwDB, sn *freezeblocks.CaplinSnapshots, reader freezeblocks.BeaconSnapshotReader, beaconDB persistence.BlockSource, logger log.Logger, states bool, fs afero.Fs) *Antiquary {
+func NewAntiquary(ctx context.Context, genesisState *state.CachingBeaconState, validatorsTable *state_accessors.StaticValidatorTable, cfg *clparams.BeaconChainConfig, dirs datadir.Dirs, downloader proto_downloader.DownloaderClient, mainDB kv.RwDB, sn *freezeblocks.CaplinSnapshots, reader freezeblocks.BeaconSnapshotReader, beaconDB persistence.BlockSource, logger log.Logger, states, blocks bool, fs afero.Fs) *Antiquary {
 	backfilled := &atomic.Bool{}
 	backfilled.Store(false)
 	return &Antiquary{
@@ -61,12 +61,13 @@ func NewAntiquary(ctx context.Context, genesisState *state.CachingBeaconState, v
 		fs:              fs,
 		validatorsTable: validatorsTable,
 		genesisState:    genesisState,
+		blocks:          blocks,
 	}
 }
 
 // Antiquate is the function that starts transactions seeding and shit, very cool but very shit too as a name.
 func (a *Antiquary) Loop() error {
-	if a.downloader == nil {
+	if a.downloader == nil || !a.blocks {
 		return nil // Just skip if we don't have a downloader
 	}
 	// Skip if we dont support backfilling for the current network
@@ -94,7 +95,6 @@ func (a *Antiquary) Loop() error {
 		return err
 	}
 	// Here we need to start mdbx transaction and lock the thread
-	log.Info("[Antiquary]: Stopping Caplin to process historical indicies")
 	tx, err := a.mainDB.BeginRw(a.ctx)
 	if err != nil {
 		return err
@@ -110,6 +110,7 @@ func (a *Antiquary) Loop() error {
 		return err
 	}
 	defer logInterval.Stop()
+	log.Info("[Antiquary]: Stopping Caplin to process historical indicies", "from", from, "to", a.sn.BlocksAvailable())
 
 	// Now write the snapshots as indicies
 	for i := from; i < a.sn.BlocksAvailable(); i++ {
@@ -208,7 +209,7 @@ func (a *Antiquary) Loop() error {
 			if to-from < snaptype.Erigon2MergeLimit {
 				continue
 			}
-			if err := a.antiquate(from, to); err != nil {
+			if err := a.antiquate(a.sn.Version(), from, to); err != nil {
 				return err
 			}
 		case <-a.ctx.Done():
@@ -217,12 +218,12 @@ func (a *Antiquary) Loop() error {
 }
 
 // Antiquate will antiquate a specific block range (aka. retire snapshots), this should be ran in the background.
-func (a *Antiquary) antiquate(from, to uint64) error {
+func (a *Antiquary) antiquate(version uint8, from, to uint64) error {
 	if a.downloader == nil {
 		return nil // Just skip if we don't have a downloader
 	}
 	log.Info("[Antiquary]: Antiquating", "from", from, "to", to)
-	if err := freezeblocks.DumpBeaconBlocks(a.ctx, a.mainDB, a.beaconDB, from, to, snaptype.Erigon2MergeLimit, a.dirs.Tmp, a.dirs.Snap, 1, log.LvlDebug, a.logger); err != nil {
+	if err := freezeblocks.DumpBeaconBlocks(a.ctx, a.mainDB, a.beaconDB, version, from, to, snaptype.Erigon2MergeLimit, a.dirs.Tmp, a.dirs.Snap, 1, log.LvlDebug, a.logger); err != nil {
 		return err
 	}
 
@@ -248,7 +249,7 @@ func (a *Antiquary) antiquate(from, to uint64) error {
 	}
 	// Notify bittorent to seed the new snapshots
 	if _, err := a.downloader.Add(a.ctx, &proto_downloader.AddRequest{Items: downloadItems}); err != nil {
-		return err
+		log.Warn("[Antiquary]: Failed to add items to bittorent", "err", err)
 	}
 
 	tx, err := a.mainDB.BeginRw(a.ctx)