From 98ec9c17f1755d19990d181da6209317ad672bd5 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 10:08:18 +0200
Subject: [PATCH 01/63] save

---
 turbo/snapshotsync/freezeblocks/block_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go
index cb1bf9f5af7..6dabe3a929e 100644
--- a/turbo/snapshotsync/freezeblocks/block_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go
@@ -2021,7 +2021,7 @@ func BeaconBlocksIdx(ctx context.Context, segmentFilePath string, blockFrom, blo
 		}
 	}()
 	// Calculate how many records there will be in the index
-	d, err := compress.NewDecompressor(segmentFilePath)
+	d, err := compress.NewDecompressor(join.Path(snapDir, segmentFilePath))
 	if err != nil {
 		return err
 	}

From 92735f3d06cb90fd578a814cb11fda83999a75f3 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 10:09:17 +0200
Subject: [PATCH 02/63] save

---
 turbo/snapshotsync/freezeblocks/block_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go
index 6dabe3a929e..01dd94f7e55 100644
--- a/turbo/snapshotsync/freezeblocks/block_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go
@@ -2021,7 +2021,7 @@ func BeaconBlocksIdx(ctx context.Context, segmentFilePath string, blockFrom, blo
 		}
 	}()
 	// Calculate how many records there will be in the index
-	d, err := compress.NewDecompressor(join.Path(snapDir, segmentFilePath))
+	d, err := compress.NewDecompressor(path.Join(snapDir, segmentFilePath))
 	if err != nil {
 		return err
 	}

From 624410c6815c2b99832659cfb68cc37264f4c2e1 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 10:57:27 +0200
Subject: [PATCH 03/63] save

---
 cl/persistence/format/snapshot_format/snapshots.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/cl/persistence/format/snapshot_format/snapshots.go b/cl/persistence/format/snapshot_format/snapshots.go
index 8cecbc8f1e0..dab2aea7ac3 100644
--- a/cl/persistence/format/snapshot_format/snapshots.go
+++ b/cl/persistence/format/snapshot_format/snapshots.go
@@ -77,6 +77,9 @@ func DumpBeaconBlocks(ctx context.Context, db kv.RoDB, b persistence.BlockSource
 	}
 
 	for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, blocksPerFile) {
+		if toSlot-i < blocksPerFile {
+			break
+		}
 		to := chooseSegmentEnd(i, toSlot, blocksPerFile)
 		logger.Log(lvl, "Dumping beacon blocks", "from", i, "to", to)
 		if err := dumpBeaconBlocksRange(ctx, db, b, i, to, tmpDir, snapDir, workers, lvl, logger); err != nil {

From 9e1f73437e45ce8c6185e0850e815e1e662bdec6 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 18:10:37 +0200
Subject: [PATCH 04/63] save

---
 .../format/snapshot_format/blocks.go          |  71 +++
 .../format/snapshot_format/blocks_test.go     |  12 +-
 .../format/snapshot_format/snapshots.go       |  90 ----
 .../format/snapshot_format/test_util.go       |  11 +
 .../freezeblocks/beacon_block_reader.go       |  66 +++
 .../freezeblocks/block_snapshots.go           |  86 +---
 .../freezeblocks/caplin_snapshots.go          | 421 ++++++++++++++++++
 7 files changed, 592 insertions(+), 165 deletions(-)
 delete mode 100644 cl/persistence/format/snapshot_format/snapshots.go
 create mode 100644 cl/persistence/format/snapshot_format/test_util.go
 create mode 100644 turbo/snapshotsync/freezeblocks/beacon_block_reader.go
 create mode 100644 turbo/snapshotsync/freezeblocks/caplin_snapshots.go

diff --git a/cl/persistence/format/snapshot_format/blocks.go b/cl/persistence/format/snapshot_format/blocks.go
index e1f7abac312..e9ed846f866 100644
--- a/cl/persistence/format/snapshot_format/blocks.go
+++ b/cl/persistence/format/snapshot_format/blocks.go
@@ -201,3 +201,74 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 
 	return block, block.DecodeSSZ(plainSSZ, int(v))
 }
+
+func ReadRawBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNumber, cfg *clparams.BeaconChainConfig) ([]byte, error) {
+	plainSSZ := []byte{}
+
+	// Metadata section is just the current hardfork of the block. TODO(give it a useful purpose)
+	v, err := readMetadataForBlock(r)
+	if err != nil {
+		return nil, err
+	}
+
+	// Read the first chunk
+	chunk1, dT1, err := chunk_encoding.ReadChunk(r)
+	if err != nil {
+		return nil, err
+	}
+	if dT1 != chunk_encoding.ChunkDataType {
+		return nil, fmt.Errorf("malformed beacon block, invalid chunk 1 type %d, expected: %d", dT1, chunk_encoding.ChunkDataType)
+	}
+	plainSSZ = append(plainSSZ, chunk1...)
+	// Read the attestation chunk (2nd chunk)
+	chunk2, dT2, err := chunk_encoding.ReadChunk(snappy.NewReader(r))
+	if err != nil {
+		return nil, err
+	}
+	if dT2 != chunk_encoding.ChunkDataType {
+		return nil, fmt.Errorf("malformed beacon block, invalid chunk 2 type %d, expected: %d", dT2, chunk_encoding.ChunkDataType)
+	}
+	plainSSZ = append(plainSSZ, chunk2...)
+	// Read the 3rd chunk
+	chunk3, dT3, err := chunk_encoding.ReadChunk(r)
+	if err != nil {
+		return nil, err
+	}
+	if dT3 != chunk_encoding.ChunkDataType {
+		return nil, fmt.Errorf("malformed beacon block, invalid chunk 3 type %d, expected: %d", dT3, chunk_encoding.ChunkDataType)
+	}
+	plainSSZ = append(plainSSZ, chunk3...)
+	if v <= clparams.AltairVersion {
+		return plainSSZ, nil
+	}
+	// Read the block pointer and retrieve chunk4 from the execution reader
+	blockPointer, err := readExecutionBlockPtr(r)
+	if err != nil {
+		return nil, err
+	}
+	executionBlock, err := executionReader.BlockByNumber(blockPointer)
+	if err != nil {
+		return nil, err
+	}
+	// Read the 4th chunk
+	chunk4, err := executionBlock.EncodeSSZ(nil)
+	if err != nil {
+		return nil, err
+	}
+	plainSSZ = append(plainSSZ, chunk4...)
+	if v <= clparams.BellatrixVersion {
+		return plainSSZ, nil
+	}
+
+	// Read the 5h chunk
+	chunk5, dT5, err := chunk_encoding.ReadChunk(r)
+	if err != nil {
+		return nil, err
+	}
+	if dT5 != chunk_encoding.ChunkDataType {
+		return nil, fmt.Errorf("malformed beacon block, invalid chunk 5 type %d, expected: %d", dT5, chunk_encoding.ChunkDataType)
+	}
+	plainSSZ = append(plainSSZ, chunk5...)
+
+	return plainSSZ, nil
+}
diff --git a/cl/persistence/format/snapshot_format/blocks_test.go b/cl/persistence/format/snapshot_format/blocks_test.go
index 5efc2963fe6..c0e9b46bee6 100644
--- a/cl/persistence/format/snapshot_format/blocks_test.go
+++ b/cl/persistence/format/snapshot_format/blocks_test.go
@@ -43,19 +43,11 @@ func getTestBlocks(t *testing.T) []*cltypes.SignedBeaconBlock {
 	return []*cltypes.SignedBeaconBlock{phase0Block, altairBlock, bellatrixBlock, capellaBlock, denebBlock}
 }
 
-type TestBlockReader struct {
-	Block *cltypes.Eth1Block
-}
-
-func (t *TestBlockReader) BlockByNumber(number uint64) (*cltypes.Eth1Block, error) {
-	return t.Block, nil
-}
-
 func TestBlockSnapshotEncoding(t *testing.T) {
 	for _, blk := range getTestBlocks(t) {
-		var br TestBlockReader
+		var br snapshot_format.MockBlockReader
 		if blk.Version() >= clparams.BellatrixVersion {
-			br = TestBlockReader{Block: blk.Block.Body.ExecutionPayload}
+			br = snapshot_format.MockBlockReader{Block: blk.Block.Body.ExecutionPayload}
 		}
 		var b bytes.Buffer
 		require.NoError(t, snapshot_format.WriteBlockForSnapshot(blk, &b))
diff --git a/cl/persistence/format/snapshot_format/snapshots.go b/cl/persistence/format/snapshot_format/snapshots.go
deleted file mode 100644
index dab2aea7ac3..00000000000
--- a/cl/persistence/format/snapshot_format/snapshots.go
+++ /dev/null
@@ -1,90 +0,0 @@
-package snapshot_format
-
-import (
-	"bytes"
-	"context"
-	"fmt"
-
-	"github.com/ledgerwatch/erigon-lib/common/background"
-	"github.com/ledgerwatch/erigon-lib/common/cmp"
-	"github.com/ledgerwatch/erigon-lib/compress"
-	"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
-	"github.com/ledgerwatch/erigon-lib/kv"
-	"github.com/ledgerwatch/erigon/cl/persistence"
-	"github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks"
-	"github.com/ledgerwatch/log/v3"
-)
-
-func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockSource, fromSlot uint64, toSlot uint64, tmpDir, snapDir string, workers int, lvl log.Lvl, logger log.Logger) error {
-	segName := snaptype.SegmentFileName(fromSlot, toSlot, snaptype.BeaconBlocks)
-	f, _ := snaptype.ParseFileName(snapDir, segName)
-
-	sn, err := compress.NewCompressor(ctx, "Snapshot BeaconBlocks", f.Path, tmpDir, compress.MinPatternScore, workers, lvl, logger)
-	if err != nil {
-		return err
-	}
-	defer sn.Close()
-
-	tx, err := db.BeginRo(ctx)
-	if err != nil {
-		return err
-	}
-	defer tx.Rollback()
-	// Generate .seg file, which is just the list of beacon blocks.
-	var buf bytes.Buffer
-	for i := fromSlot; i <= toSlot; i++ {
-		obj, err := b.GetBlock(ctx, tx, i)
-		if err != nil {
-			return err
-		}
-		if obj == nil {
-			if err := sn.AddWord(nil); err != nil {
-				return err
-			}
-			continue
-		}
-		if err := WriteBlockForSnapshot(obj.Data, &buf); err != nil {
-			return err
-		}
-		if err := sn.AddWord(buf.Bytes()); err != nil {
-			return err
-		}
-		buf.Reset()
-	}
-	if err := sn.Compress(); err != nil {
-		return fmt.Errorf("compress: %w", err)
-	}
-	// Generate .idx file, which is the slot => offset mapping.
-	p := &background.Progress{}
-
-	return freezeblocks.BeaconBlocksIdx(ctx, segName, fromSlot, toSlot, snapDir, tmpDir, p, lvl, logger)
-}
-
-func chooseSegmentEnd(from, to, blocksPerFile uint64) uint64 {
-	next := (from/blocksPerFile + 1) * blocksPerFile
-	to = cmp.Min(next, to)
-
-	if to < snaptype.Erigon2MinSegmentSize {
-		return to
-	}
-
-	return to - (to % snaptype.Erigon2MinSegmentSize) // round down to the nearest 1k
-}
-
-func DumpBeaconBlocks(ctx context.Context, db kv.RoDB, b persistence.BlockSource, fromSlot, toSlot, blocksPerFile uint64, tmpDir, snapDir string, workers int, lvl log.Lvl, logger log.Logger) error {
-	if blocksPerFile == 0 {
-		return nil
-	}
-
-	for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, blocksPerFile) {
-		if toSlot-i < blocksPerFile {
-			break
-		}
-		to := chooseSegmentEnd(i, toSlot, blocksPerFile)
-		logger.Log(lvl, "Dumping beacon blocks", "from", i, "to", to)
-		if err := dumpBeaconBlocksRange(ctx, db, b, i, to, tmpDir, snapDir, workers, lvl, logger); err != nil {
-			return err
-		}
-	}
-	return nil
-}
diff --git a/cl/persistence/format/snapshot_format/test_util.go b/cl/persistence/format/snapshot_format/test_util.go
new file mode 100644
index 00000000000..3993c1648b7
--- /dev/null
+++ b/cl/persistence/format/snapshot_format/test_util.go
@@ -0,0 +1,11 @@
+package snapshot_format
+
+import "github.com/ledgerwatch/erigon/cl/cltypes"
+
+type MockBlockReader struct {
+	Block *cltypes.Eth1Block
+}
+
+func (t *MockBlockReader) BlockByNumber(number uint64) (*cltypes.Eth1Block, error) {
+	return t.Block, nil
+}
diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
new file mode 100644
index 00000000000..8134f23416b
--- /dev/null
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -0,0 +1,66 @@
+package freezeblocks
+
+import (
+	"bytes"
+
+	"github.com/ledgerwatch/erigon/cl/clparams"
+	"github.com/ledgerwatch/erigon/cl/cltypes"
+	"github.com/ledgerwatch/erigon/cl/persistence/format/snapshot_format"
+)
+
+type BeaconSnapshotReader interface {
+	// ReadBlock reads the block at the given slot.
+	// If the block is not present, it returns nil.
+	ReadBlock(slot uint64) (*cltypes.SignedBeaconBlock, error)
+	RawBlockSSZ(slot uint64) ([]byte, error)
+
+	FrozenSlots() uint64
+}
+
+type beaconSnapshotReader struct {
+	sn *CaplinSnapshots
+
+	eth1Getter snapshot_format.ExecutionBlockReaderByNumber
+	cfg        *clparams.BeaconChainConfig
+}
+
+func NewBeaconSnapshotReader(snapshots *CaplinSnapshots, eth1Getter snapshot_format.ExecutionBlockReaderByNumber) BeaconSnapshotReader {
+	return &beaconSnapshotReader{sn: snapshots, eth1Getter: eth1Getter}
+}
+
+func (r *beaconSnapshotReader) FrozenSlots() uint64 {
+	return r.sn.BlocksAvailable()
+}
+
+func (r *beaconSnapshotReader) ReadBlock(slot uint64) (*cltypes.SignedBeaconBlock, error) {
+	buf, err := r.RawBlockSSZ(slot)
+	if err != nil {
+		return nil, err
+	}
+	return snapshot_format.ReadBlockFromSnapshot(bytes.NewBuffer(buf), r.eth1Getter, r.cfg)
+}
+
+func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
+	view := r.sn.View()
+	defer view.Close()
+
+	var buf []byte
+	seg, ok := view.BeaconBlocksSegment(slot)
+	if !ok {
+		return nil, nil
+	}
+
+	if seg.idxSlot == nil {
+		return nil, nil
+	}
+	blockOffset := seg.idxSlot.OrdinalLookup(slot - seg.idxSlot.BaseDataID())
+
+	gg := seg.seg.MakeGetter()
+	gg.Reset(blockOffset)
+	if !gg.HasNext() {
+		return nil, nil
+	}
+	gg.Next(buf)
+	buf, _ = gg.Next(buf[:0])
+	return buf, nil
+}
diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go
index 01dd94f7e55..69f13b55bc8 100644
--- a/turbo/snapshotsync/freezeblocks/block_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go
@@ -1118,6 +1118,27 @@ func noOverlaps(in []snaptype.FileInfo) (res []snaptype.FileInfo) {
 	return res
 }
 
+func SegmentsCaplin(dir string) (res []snaptype.FileInfo, missingSnapshots []Range, err error) {
+	list, err := snaptype.Segments(dir)
+	if err != nil {
+		return nil, missingSnapshots, err
+	}
+	{
+		var l []snaptype.FileInfo
+		var m []Range
+		for _, f := range list {
+			if f.T != snaptype.BeaconBlocks {
+				continue
+			}
+			l = append(l, f)
+		}
+		l, m = noGaps(noOverlaps(allTypeOfSegmentsMustExist(dir, l)))
+		res = append(res, l...)
+		missingSnapshots = append(missingSnapshots, m...)
+	}
+	return res, missingSnapshots, nil
+}
+
 func Segments(dir string) (res []snaptype.FileInfo, missingSnapshots []Range, err error) {
 	list, err := snaptype.Segments(dir)
 	if err != nil {
@@ -2014,71 +2035,6 @@ RETRY:
 	return nil
 }
 
-func BeaconBlocksIdx(ctx context.Context, segmentFilePath string, blockFrom, blockTo uint64, snapDir string, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) {
-	defer func() {
-		if rec := recover(); rec != nil {
-			err = fmt.Errorf("BeaconBlocksIdx: at=%d-%d, %v, %s", blockFrom, blockTo, rec, dbg.Stack())
-		}
-	}()
-	// Calculate how many records there will be in the index
-	d, err := compress.NewDecompressor(path.Join(snapDir, segmentFilePath))
-	if err != nil {
-		return err
-	}
-	defer d.Close()
-	g := d.MakeGetter()
-	var idxFilePath = filepath.Join(snapDir, snaptype.IdxFileName(blockFrom, blockTo, snaptype.BeaconBlocks.String()))
-
-	var baseSpanId uint64
-	if blockFrom > zerothSpanEnd {
-		baseSpanId = 1 + (blockFrom-zerothSpanEnd-1)/spanLength
-	}
-
-	rs, err := recsplit.NewRecSplit(recsplit.RecSplitArgs{
-		KeyCount:   d.Count(),
-		Enums:      d.Count() > 0,
-		BucketSize: 2000,
-		LeafSize:   8,
-		TmpDir:     tmpDir,
-		IndexFile:  idxFilePath,
-		BaseDataID: baseSpanId,
-	}, logger)
-	if err != nil {
-		return err
-	}
-	rs.LogLvl(log.LvlDebug)
-
-	defer d.EnableMadvNormal().DisableReadAhead()
-RETRY:
-	g.Reset(0)
-	var i, offset, nextPos uint64
-	var key [8]byte
-	for g.HasNext() {
-		nextPos, _ = g.Skip()
-		binary.BigEndian.PutUint64(key[:], i)
-		i++
-		if err = rs.AddKey(key[:], offset); err != nil {
-			return err
-		}
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
-		}
-		offset = nextPos
-	}
-	if err = rs.Build(ctx); err != nil {
-		if errors.Is(err, recsplit.ErrCollision) {
-			logger.Info("Building recsplit. Collision happened. It's ok. Restarting with another salt...", "err", err)
-			rs.ResetNextSalt()
-			goto RETRY
-		}
-		return err
-	}
-
-	return nil
-}
-
 // HeadersIdx - headerHash -> offset (analog of kv.HeaderNumber)
 func HeadersIdx(ctx context.Context, chainConfig *chain.Config, segmentFilePath string, firstBlockNumInSegment uint64, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) {
 	defer func() {
diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
new file mode 100644
index 00000000000..50df22606ad
--- /dev/null
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -0,0 +1,421 @@
+package freezeblocks
+
+import (
+	"bytes"
+	"context"
+	"encoding/binary"
+	"errors"
+	"fmt"
+	"os"
+	"path"
+	"path/filepath"
+	"sync"
+	"sync/atomic"
+
+	"github.com/ledgerwatch/erigon-lib/common/background"
+	"github.com/ledgerwatch/erigon-lib/common/cmp"
+	"github.com/ledgerwatch/erigon-lib/common/dbg"
+	"github.com/ledgerwatch/erigon-lib/compress"
+	"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
+	"github.com/ledgerwatch/erigon-lib/kv"
+	"github.com/ledgerwatch/erigon-lib/recsplit"
+	"github.com/ledgerwatch/erigon/cl/persistence"
+	"github.com/ledgerwatch/erigon/cl/persistence/format/snapshot_format"
+	"github.com/ledgerwatch/erigon/eth/ethconfig"
+	"github.com/ledgerwatch/log/v3"
+)
+
+type BeaconBlockSegment struct {
+	seg     *compress.Decompressor // value: chunked(ssz(SignedBeaconBlocks))
+	idxSlot *recsplit.Index        // slot       -> beacon_slot_segment_offset
+	ranges  Range
+}
+
+func (sn *BeaconBlockSegment) closeIdx() {
+	if sn.idxSlot != nil {
+		sn.idxSlot.Close()
+		sn.idxSlot = nil
+	}
+}
+func (sn *BeaconBlockSegment) closeSeg() {
+	if sn.seg != nil {
+		sn.seg.Close()
+		sn.seg = nil
+	}
+}
+func (sn *BeaconBlockSegment) close() {
+	sn.closeSeg()
+	sn.closeIdx()
+}
+func (sn *BeaconBlockSegment) reopenSeg(dir string) (err error) {
+	sn.closeSeg()
+	fileName := snaptype.SegmentFileName(sn.ranges.from, sn.ranges.to, snaptype.BeaconBlocks)
+	sn.seg, err = compress.NewDecompressor(path.Join(dir, fileName))
+	if err != nil {
+		return fmt.Errorf("%w, fileName: %s", err, fileName)
+	}
+	return nil
+}
+func (sn *BeaconBlockSegment) reopenIdxIfNeed(dir string, optimistic bool) (err error) {
+	if sn.idxSlot != nil {
+		return nil
+	}
+	err = sn.reopenIdx(dir)
+	if err != nil {
+		if !errors.Is(err, os.ErrNotExist) {
+			if optimistic {
+				log.Warn("[snapshots] open index", "err", err)
+			} else {
+				return err
+			}
+		}
+	}
+	return nil
+}
+
+func (sn *BeaconBlockSegment) reopenIdx(dir string) (err error) {
+	sn.closeIdx()
+	if sn.seg == nil {
+		return nil
+	}
+	fileName := snaptype.IdxFileName(sn.ranges.from, sn.ranges.to, snaptype.BeaconBlocks.String())
+	sn.idxSlot, err = recsplit.OpenIndex(path.Join(dir, fileName))
+	if err != nil {
+		return fmt.Errorf("%w, fileName: %s", err, fileName)
+	}
+	if sn.idxSlot.ModTime().Before(sn.seg.ModTime()) {
+		// Index has been created before the segment file, needs to be ignored (and rebuilt) as inconsistent
+		sn.idxSlot.Close()
+		sn.idxSlot = nil
+	}
+	return nil
+}
+
+type beaconBlockSegments struct {
+	lock     sync.RWMutex
+	segments []*BeaconBlockSegment
+}
+
+func (s *beaconBlockSegments) View(f func(segments []*BeaconBlockSegment) error) error {
+	s.lock.RLock()
+	defer s.lock.RUnlock()
+	return f(s.segments)
+}
+
+func BeaconBlocksIdx(ctx context.Context, segmentFilePath string, blockFrom, blockTo uint64, snapDir string, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) {
+	defer func() {
+		if rec := recover(); rec != nil {
+			err = fmt.Errorf("BeaconBlocksIdx: at=%d-%d, %v, %s", blockFrom, blockTo, rec, dbg.Stack())
+		}
+	}()
+	// Calculate how many records there will be in the index
+	d, err := compress.NewDecompressor(path.Join(snapDir, segmentFilePath))
+	if err != nil {
+		return err
+	}
+	defer d.Close()
+	g := d.MakeGetter()
+	var idxFilePath = filepath.Join(snapDir, snaptype.IdxFileName(blockFrom, blockTo, snaptype.BeaconBlocks.String()))
+
+	var baseSpanId uint64
+	if blockFrom > zerothSpanEnd {
+		baseSpanId = 1 + (blockFrom-zerothSpanEnd-1)/spanLength
+	}
+
+	rs, err := recsplit.NewRecSplit(recsplit.RecSplitArgs{
+		KeyCount:   d.Count(),
+		Enums:      d.Count() > 0,
+		BucketSize: 2000,
+		LeafSize:   8,
+		TmpDir:     tmpDir,
+		IndexFile:  idxFilePath,
+		BaseDataID: baseSpanId,
+	}, logger)
+	if err != nil {
+		return err
+	}
+	rs.LogLvl(log.LvlDebug)
+
+	defer d.EnableMadvNormal().DisableReadAhead()
+RETRY:
+	g.Reset(0)
+	var i, offset, nextPos uint64
+	var key [8]byte
+	for g.HasNext() {
+		nextPos, _ = g.Skip()
+		binary.BigEndian.PutUint64(key[:], i)
+		i++
+		if err = rs.AddKey(key[:], offset); err != nil {
+			return err
+		}
+		select {
+		case <-ctx.Done():
+			return ctx.Err()
+		default:
+		}
+		offset = nextPos
+	}
+	if err = rs.Build(ctx); err != nil {
+		if errors.Is(err, recsplit.ErrCollision) {
+			logger.Info("Building recsplit. Collision happened. It's ok. Restarting with another salt...", "err", err)
+			rs.ResetNextSalt()
+			goto RETRY
+		}
+		return err
+	}
+
+	return nil
+}
+
+type CaplinSnapshots struct {
+	indicesReady  atomic.Bool
+	segmentsReady atomic.Bool
+
+	BeaconBlocks *beaconBlockSegments
+
+	dir         string
+	segmentsMax atomic.Uint64 // all types of .seg files are available - up to this number
+	idxMax      atomic.Uint64 // all types of .idx files are available - up to this number
+	cfg         ethconfig.BlocksFreezing
+	logger      log.Logger
+}
+
+// NewCaplinSnapshots - opens all snapshots. But to simplify everything:
+//   - it opens snapshots only on App start and immutable after
+//   - all snapshots of given blocks range must exist - to make this blocks range available
+//   - gaps are not allowed
+//   - segment have [from:to) semantic
+func NewCaplinSnapshots(cfg ethconfig.BlocksFreezing, snapDir string, logger log.Logger) *CaplinSnapshots {
+	return &CaplinSnapshots{dir: snapDir, cfg: cfg, BeaconBlocks: &beaconBlockSegments{}, logger: logger}
+}
+
+func (s *CaplinSnapshots) IndicesMax() uint64  { return s.idxMax.Load() }
+func (s *CaplinSnapshots) SegmentsMax() uint64 { return s.segmentsMax.Load() }
+func (s *CaplinSnapshots) BlocksAvailable() uint64 {
+	return cmp.Min(s.segmentsMax.Load(), s.idxMax.Load())
+}
+
+// ReopenList stops on optimistic=false, continue opening files on optimistic=true
+func (s *CaplinSnapshots) ReopenList(fileNames []string, optimistic bool) error {
+	s.BeaconBlocks.lock.Lock()
+	defer s.BeaconBlocks.lock.Unlock()
+
+	s.closeWhatNotInList(fileNames)
+	var segmentsMax uint64
+	var segmentsMaxSet bool
+Loop:
+	for _, fName := range fileNames {
+		f, ok := snaptype.ParseFileName(s.dir, fName)
+		if !ok {
+			continue
+		}
+		var processed bool = true
+
+		switch f.T {
+		case snaptype.BeaconBlocks:
+			var sn *BeaconBlockSegment
+			var exists bool
+			for _, sn2 := range s.BeaconBlocks.segments {
+				if sn2.seg == nil { // it's ok if some segment was not able to open
+					continue
+				}
+				if fName == sn2.seg.FileName() {
+					sn = sn2
+					exists = true
+					break
+				}
+			}
+			if !exists {
+				sn = &BeaconBlockSegment{ranges: Range{f.From, f.To}}
+			}
+			if err := sn.reopenSeg(s.dir); err != nil {
+				if errors.Is(err, os.ErrNotExist) {
+					if optimistic {
+						continue Loop
+					} else {
+						break Loop
+					}
+				}
+				if optimistic {
+					s.logger.Warn("[snapshots] open segment", "err", err)
+					continue Loop
+				} else {
+					return err
+				}
+			}
+
+			if !exists {
+				// it's possible to iterate over .seg file even if you don't have index
+				// then make segment available even if index open may fail
+				s.BeaconBlocks.segments = append(s.BeaconBlocks.segments, sn)
+			}
+			if err := sn.reopenIdxIfNeed(s.dir, optimistic); err != nil {
+				return err
+			}
+		}
+
+		if processed {
+			if f.To > 0 {
+				segmentsMax = f.To - 1
+			} else {
+				segmentsMax = 0
+			}
+			segmentsMaxSet = true
+		}
+	}
+	if segmentsMaxSet {
+		s.segmentsMax.Store(segmentsMax)
+	}
+	s.segmentsReady.Store(true)
+	s.idxMax.Store(s.idxAvailability())
+	s.indicesReady.Store(true)
+
+	return nil
+}
+
+func (s *CaplinSnapshots) idxAvailability() uint64 {
+	var beaconBlocks uint64
+	for _, seg := range s.BeaconBlocks.segments {
+		if seg.idxSlot == nil {
+			break
+		}
+		beaconBlocks = seg.ranges.to - 1
+	}
+	return beaconBlocks
+}
+
+func (s *CaplinSnapshots) ReopenFolder() error {
+	files, _, err := SegmentsCaplin(s.dir)
+	if err != nil {
+		return err
+	}
+	list := make([]string, 0, len(files))
+	for _, f := range files {
+		_, fName := filepath.Split(f.Path)
+		list = append(list, fName)
+	}
+	return s.ReopenList(list, false)
+}
+
+func (s *CaplinSnapshots) closeWhatNotInList(l []string) {
+Loop1:
+	for i, sn := range s.BeaconBlocks.segments {
+		if sn.seg == nil {
+			continue Loop1
+		}
+		_, name := filepath.Split(sn.seg.FilePath())
+		for _, fName := range l {
+			if fName == name {
+				continue Loop1
+			}
+		}
+		sn.close()
+		s.BeaconBlocks.segments[i] = nil
+	}
+	var i int
+	for i = 0; i < len(s.BeaconBlocks.segments) && s.BeaconBlocks.segments[i] != nil && s.BeaconBlocks.segments[i].seg != nil; i++ {
+	}
+	tail := s.BeaconBlocks.segments[i:]
+	s.BeaconBlocks.segments = s.BeaconBlocks.segments[:i]
+	for i = 0; i < len(tail); i++ {
+		if tail[i] != nil {
+			tail[i].close()
+			tail[i] = nil
+		}
+	}
+}
+
+type CaplinView struct {
+	s      *CaplinSnapshots
+	closed bool
+}
+
+func (s *CaplinSnapshots) View() *CaplinView {
+	v := &CaplinView{s: s}
+	v.s.BeaconBlocks.lock.RLock()
+	return v
+}
+
+func (v *CaplinView) Close() {
+	if v.closed {
+		return
+	}
+	v.closed = true
+	v.s.BeaconBlocks.lock.RUnlock()
+
+}
+
+func (v *CaplinView) BeaconBlocks() []*BeaconBlockSegment { return v.s.BeaconBlocks.segments }
+
+func (v *CaplinView) BeaconBlocksSegment(slot uint64) (*BeaconBlockSegment, bool) {
+	for _, seg := range v.BeaconBlocks() {
+		if !(slot >= seg.ranges.from && slot < seg.ranges.to) {
+			continue
+		}
+		return seg, true
+	}
+	return nil, false
+}
+
+func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockSource, fromSlot uint64, toSlot uint64, tmpDir, snapDir string, workers int, lvl log.Lvl, logger log.Logger) error {
+	segName := snaptype.SegmentFileName(fromSlot, toSlot, snaptype.BeaconBlocks)
+	f, _ := snaptype.ParseFileName(snapDir, segName)
+
+	sn, err := compress.NewCompressor(ctx, "Snapshot BeaconBlocks", f.Path, tmpDir, compress.MinPatternScore, workers, lvl, logger)
+	if err != nil {
+		return err
+	}
+	defer sn.Close()
+
+	tx, err := db.BeginRo(ctx)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+	// Generate .seg file, which is just the list of beacon blocks.
+	var buf bytes.Buffer
+	for i := fromSlot; i <= toSlot; i++ {
+		obj, err := b.GetBlock(ctx, tx, i)
+		if err != nil {
+			return err
+		}
+		if obj == nil {
+			if err := sn.AddWord(nil); err != nil {
+				return err
+			}
+			continue
+		}
+		if err := snapshot_format.WriteBlockForSnapshot(obj.Data, &buf); err != nil {
+			return err
+		}
+		if err := sn.AddWord(buf.Bytes()); err != nil {
+			return err
+		}
+		buf.Reset()
+	}
+	if err := sn.Compress(); err != nil {
+		return fmt.Errorf("compress: %w", err)
+	}
+	// Generate .idx file, which is the slot => offset mapping.
+	p := &background.Progress{}
+
+	return BeaconBlocksIdx(ctx, segName, fromSlot, toSlot, snapDir, tmpDir, p, lvl, logger)
+}
+
+func DumpBeaconBlocks(ctx context.Context, db kv.RoDB, b persistence.BlockSource, fromSlot, toSlot, blocksPerFile uint64, tmpDir, snapDir string, workers int, lvl log.Lvl, logger log.Logger) error {
+	if blocksPerFile == 0 {
+		return nil
+	}
+
+	for i := fromSlot; i < toSlot; i = chooseSegmentEnd(i, toSlot, blocksPerFile) {
+		if toSlot-i < blocksPerFile {
+			break
+		}
+		to := chooseSegmentEnd(i, toSlot, blocksPerFile)
+		logger.Log(lvl, "Dumping beacon blocks", "from", i, "to", to)
+		if err := dumpBeaconBlocksRange(ctx, db, b, i, to, tmpDir, snapDir, workers, lvl, logger); err != nil {
+			return err
+		}
+	}
+	return nil
+}

From aaefb416d24ca06b31768d8b37a241372b6a9fc8 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 18:12:41 +0200
Subject: [PATCH 05/63] ops

---
 cmd/capcli/cli.go | 76 +++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 73 insertions(+), 3 deletions(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index e9c70f0629e..3e72f039805 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -12,6 +12,8 @@ import (
 	"github.com/ledgerwatch/erigon/cl/cltypes"
 	persistence2 "github.com/ledgerwatch/erigon/cl/persistence"
 	"github.com/ledgerwatch/erigon/cmd/caplin/caplin1"
+	"github.com/ledgerwatch/erigon/eth/ethconfig"
+	"github.com/ledgerwatch/erigon/turbo/snapshotsync/freezeblocks"
 
 	"github.com/ledgerwatch/erigon-lib/common/datadir"
 	"github.com/ledgerwatch/erigon-lib/downloader/snaptype"
@@ -45,8 +47,9 @@ var CLI struct {
 	Blocks Blocks `cmd:"" help:"download blocks from reqresp network"`
 	Epochs Epochs `cmd:"" help:"download epochs from reqresp network"`
 
-	Chain         Chain         `cmd:"" help:"download the entire chain from reqresp network"`
-	DumpSnapshots DumpSnapshots `cmd:"" help:"generate caplin snapshots"`
+	Chain          Chain         `cmd:"" help:"download the entire chain from reqresp network"`
+	DumpSnapshots  DumpSnapshots `cmd:"" help:"generate caplin snapshots"`
+	CheckSnapshots DumpSnapshots `cmd:"" help:"check snapshot folder against content of chain data"`
 }
 
 type chainCfg struct {
@@ -425,5 +428,72 @@ func (c *DumpSnapshots) Run(ctx *Context) error {
 		return
 	})
 
-	return snapshot_format.DumpBeaconBlocks(ctx, db, beaconDB, 0, to, snaptype.Erigon2SegmentSize, dirs.Tmp, dirs.Snap, 8, log.LvlInfo, log.Root())
+	return freezeblocks.DumpBeaconBlocks(ctx, db, beaconDB, 0, to, snaptype.Erigon2SegmentSize, dirs.Tmp, dirs.Snap, 8, log.LvlInfo, log.Root())
+}
+
+type CheckSnapshots struct {
+	chainCfg
+	outputFolder
+}
+
+func (c *CheckSnapshots) Run(ctx *Context) error {
+	_, _, beaconConfig, _, err := clparams.GetConfigsByNetworkName(c.Chain)
+	if err != nil {
+		return err
+	}
+	log.Root().SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StderrHandler))
+	log.Info("Started chain download", "chain", c.Chain)
+
+	dirs := datadir.New(c.Datadir)
+	log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StderrHandler))
+
+	rawDB := persistence.AferoRawBeaconBlockChainFromOsPath(beaconConfig, dirs.CaplinHistory)
+	beaconDB, db, err := caplin1.OpenCaplinDatabase(ctx, db_config.DatabaseConfiguration{PruneDepth: math.MaxUint64}, beaconConfig, rawDB, dirs.CaplinIndexing, nil, false)
+	if err != nil {
+		return err
+	}
+	var to uint64
+	tx, err := db.BeginRo(ctx)
+	if err != nil {
+		return err
+	}
+	defer tx.Rollback()
+
+	to = (to / snaptype.Erigon2SegmentSize) * snaptype.Erigon2SegmentSize
+
+	csn := freezeblocks.NewCaplinSnapshots(ethconfig.BlocksFreezing{}, dirs.Snap, log.Root())
+	_ = csn.ReopenFolder()
+
+	br := &snapshot_format.MockBlockReader{}
+	snReader := freezeblocks.NewBeaconSnapshotReader(csn, br)
+	for i := uint64(0); i < to; i++ {
+		// Read the original canonical slot
+		data, err := beaconDB.GetBlock(ctx, tx, i)
+		if err != nil {
+			return err
+		}
+		if data == nil {
+			continue
+		}
+		blk := data.Data
+		if blk == nil {
+			continue
+		}
+		// first thing if the block is bellatrix update the mock block reader
+		if blk.Version() >= clparams.BellatrixVersion {
+			br.Block = blk.Block.Body.ExecutionPayload
+		}
+		blk2, err := snReader.ReadBlock(i)
+		if err != nil {
+			log.Error("Error detected", "err", err, "slot", i)
+			return nil
+		}
+		hash1, _ := blk.HashSSZ()
+		hash2, _ := blk2.HashSSZ()
+		if hash1 != hash2 {
+			log.Error("Mismatching blocks", "slot", i, "datadir", hash1, "snapshot", hash2)
+		}
+		log.Info("Successfully checked", "slot", i)
+	}
+	return nil
 }

From c91f9fc8637035fef7dd481c3ce06c8903f5f8ed Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 18:20:42 +0200
Subject: [PATCH 06/63] save

---
 cmd/capcli/cli.go | 6 +++---
 1 file changed, 3 insertions(+), 3 deletions(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 3e72f039805..8b1fbbf94a1 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -47,9 +47,9 @@ var CLI struct {
 	Blocks Blocks `cmd:"" help:"download blocks from reqresp network"`
 	Epochs Epochs `cmd:"" help:"download epochs from reqresp network"`
 
-	Chain          Chain         `cmd:"" help:"download the entire chain from reqresp network"`
-	DumpSnapshots  DumpSnapshots `cmd:"" help:"generate caplin snapshots"`
-	CheckSnapshots DumpSnapshots `cmd:"" help:"check snapshot folder against content of chain data"`
+	Chain          Chain          `cmd:"" help:"download the entire chain from reqresp network"`
+	DumpSnapshots  DumpSnapshots  `cmd:"" help:"generate caplin snapshots"`
+	CheckSnapshots CheckSnapshots `cmd:"" help:"check snapshot folder against content of chain data"`
 }
 
 type chainCfg struct {

From bbbf0e1995bfa66ce742183b0ca00134b3592a75 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 18:43:36 +0200
Subject: [PATCH 07/63] save

---
 cmd/capcli/cli.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 8b1fbbf94a1..a633e48f836 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -442,7 +442,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 		return err
 	}
 	log.Root().SetHandler(log.LvlFilterHandler(log.LvlDebug, log.StderrHandler))
-	log.Info("Started chain download", "chain", c.Chain)
+	log.Info("Started the checking process", "chain", c.Chain)
 
 	dirs := datadir.New(c.Datadir)
 	log.Root().SetHandler(log.LvlFilterHandler(log.LvlInfo, log.StderrHandler))
@@ -473,10 +473,12 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 			return err
 		}
 		if data == nil {
+			log.Info("Block not found", i)
 			continue
 		}
 		blk := data.Data
 		if blk == nil {
+			log.Info("Block not found", i)
 			continue
 		}
 		// first thing if the block is bellatrix update the mock block reader

From 1589693ae1d5705cc4c8665850f546a612e51ded Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 18:45:52 +0200
Subject: [PATCH 08/63] save

---
 cmd/capcli/cli.go | 5 +++++
 1 file changed, 5 insertions(+)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index a633e48f836..a4ac9454003 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -459,6 +459,11 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 	}
 	defer tx.Rollback()
 
+	to, err = beacon_indicies.ReadHighestFinalized(tx)
+	if err != nil {
+		return err
+	}
+
 	to = (to / snaptype.Erigon2SegmentSize) * snaptype.Erigon2SegmentSize
 
 	csn := freezeblocks.NewCaplinSnapshots(ethconfig.BlocksFreezing{}, dirs.Snap, log.Root())

From 84f49dcfc36e1de0b6c7c37ef1864ab011fd4863 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 18:48:38 +0200
Subject: [PATCH 09/63] save

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index a4ac9454003..9b915efdc55 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -492,7 +492,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 		}
 		blk2, err := snReader.ReadBlock(i)
 		if err != nil {
-			log.Error("Error detected", "err", err, "slot", i)
+			log.Error("Error detected in decoding snapshots", "err", err, "slot", i)
 			return nil
 		}
 		hash1, _ := blk.HashSSZ()

From f2e35551f8faec54e04eba28d13b2a7f7b403cd9 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 18:50:16 +0200
Subject: [PATCH 10/63] save

---
 cl/persistence/format/snapshot_format/blocks.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cl/persistence/format/snapshot_format/blocks.go b/cl/persistence/format/snapshot_format/blocks.go
index e9ed846f866..ba44abf37f7 100644
--- a/cl/persistence/format/snapshot_format/blocks.go
+++ b/cl/persistence/format/snapshot_format/blocks.go
@@ -139,6 +139,7 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	if err != nil {
 		return nil, err
 	}
+	fmt.Println(v)
 
 	// Read the first chunk
 	chunk1, dT1, err := chunk_encoding.ReadChunk(r)
@@ -148,6 +149,7 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	if dT1 != chunk_encoding.ChunkDataType {
 		return nil, fmt.Errorf("malformed beacon block, invalid chunk 1 type %d, expected: %d", dT1, chunk_encoding.ChunkDataType)
 	}
+	fmt.Println("A")
 	plainSSZ = append(plainSSZ, chunk1...)
 	// Read the attestation chunk (2nd chunk)
 	chunk2, dT2, err := chunk_encoding.ReadChunk(snappy.NewReader(r))

From d142f5b73c8d3374800a7ecece0933f5e259a7c0 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 18:53:20 +0200
Subject: [PATCH 11/63] save

---
 cl/persistence/format/snapshot_format/blocks.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/cl/persistence/format/snapshot_format/blocks.go b/cl/persistence/format/snapshot_format/blocks.go
index ba44abf37f7..e9ed846f866 100644
--- a/cl/persistence/format/snapshot_format/blocks.go
+++ b/cl/persistence/format/snapshot_format/blocks.go
@@ -139,7 +139,6 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	if err != nil {
 		return nil, err
 	}
-	fmt.Println(v)
 
 	// Read the first chunk
 	chunk1, dT1, err := chunk_encoding.ReadChunk(r)
@@ -149,7 +148,6 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	if dT1 != chunk_encoding.ChunkDataType {
 		return nil, fmt.Errorf("malformed beacon block, invalid chunk 1 type %d, expected: %d", dT1, chunk_encoding.ChunkDataType)
 	}
-	fmt.Println("A")
 	plainSSZ = append(plainSSZ, chunk1...)
 	// Read the attestation chunk (2nd chunk)
 	chunk2, dT2, err := chunk_encoding.ReadChunk(snappy.NewReader(r))

From bbc33be61bffb293774fc0f18f637a804444eb52 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 18:53:58 +0200
Subject: [PATCH 12/63] save

---
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 7 ++++++-
 1 file changed, 6 insertions(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index 8134f23416b..bb8652ffceb 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -2,6 +2,7 @@ package freezeblocks
 
 import (
 	"bytes"
+	"fmt"
 
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -37,7 +38,11 @@ func (r *beaconSnapshotReader) ReadBlock(slot uint64) (*cltypes.SignedBeaconBloc
 	if err != nil {
 		return nil, err
 	}
-	return snapshot_format.ReadBlockFromSnapshot(bytes.NewBuffer(buf), r.eth1Getter, r.cfg)
+	if buf == nil {
+		return nil, nil
+	}
+	fmt.Println(len(buf))
+	return snapshot_format.ReadBlockFromSnapshot(bytes.NewReader(buf), r.eth1Getter, r.cfg)
 }
 
 func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {

From 6909830a0a580406201b739d1658c85a71c53d5f Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 18:56:32 +0200
Subject: [PATCH 13/63] save

---
 cmd/capcli/cli.go | 8 +++++++-
 1 file changed, 7 insertions(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 9b915efdc55..6311e536b51 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -467,7 +467,9 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 	to = (to / snaptype.Erigon2SegmentSize) * snaptype.Erigon2SegmentSize
 
 	csn := freezeblocks.NewCaplinSnapshots(ethconfig.BlocksFreezing{}, dirs.Snap, log.Root())
-	_ = csn.ReopenFolder()
+	if err := csn.ReopenFolder(); err != nil {
+		return err
+	}
 
 	br := &snapshot_format.MockBlockReader{}
 	snReader := freezeblocks.NewBeaconSnapshotReader(csn, br)
@@ -495,6 +497,10 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 			log.Error("Error detected in decoding snapshots", "err", err, "slot", i)
 			return nil
 		}
+		if blk2 == nil {
+			log.Error("Block not found in snapshot", "slot", i)
+			return nil
+		}
 		hash1, _ := blk.HashSSZ()
 		hash2, _ := blk2.HashSSZ()
 		if hash1 != hash2 {

From fc4a31c3185804528a9da759f394cab4aeb0e637 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 18:57:29 +0200
Subject: [PATCH 14/63] save

---
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index 50df22606ad..a9200dc77d7 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -289,6 +289,7 @@ func (s *CaplinSnapshots) ReopenFolder() error {
 	if err != nil {
 		return err
 	}
+	fmt.Println(files)
 	list := make([]string, 0, len(files))
 	for _, f := range files {
 		_, fName := filepath.Split(f.Path)

From f9f62d1eae2ab8838a852519e729f3b1016990f0 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 19:04:06 +0200
Subject: [PATCH 15/63] save

---
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 2 --
 turbo/snapshotsync/freezeblocks/block_snapshots.go     | 2 ++
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go    | 1 -
 3 files changed, 2 insertions(+), 3 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index bb8652ffceb..cbcb76ce427 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -2,7 +2,6 @@ package freezeblocks
 
 import (
 	"bytes"
-	"fmt"
 
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -41,7 +40,6 @@ func (r *beaconSnapshotReader) ReadBlock(slot uint64) (*cltypes.SignedBeaconBloc
 	if buf == nil {
 		return nil, nil
 	}
-	fmt.Println(len(buf))
 	return snapshot_format.ReadBlockFromSnapshot(bytes.NewReader(buf), r.eth1Getter, r.cfg)
 }
 
diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go
index 69f13b55bc8..df779d759eb 100644
--- a/turbo/snapshotsync/freezeblocks/block_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go
@@ -1123,6 +1123,8 @@ func SegmentsCaplin(dir string) (res []snaptype.FileInfo, missingSnapshots []Ran
 	if err != nil {
 		return nil, missingSnapshots, err
 	}
+	fmt.Println(list)
+
 	{
 		var l []snaptype.FileInfo
 		var m []Range
diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index a9200dc77d7..50df22606ad 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -289,7 +289,6 @@ func (s *CaplinSnapshots) ReopenFolder() error {
 	if err != nil {
 		return err
 	}
-	fmt.Println(files)
 	list := make([]string, 0, len(files))
 	for _, f := range files {
 		_, fName := filepath.Split(f.Path)

From bd4316792c3a262713766035196985732baf8ce0 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 19:10:07 +0200
Subject: [PATCH 16/63] save

---
 turbo/snapshotsync/freezeblocks/block_snapshots.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go
index df779d759eb..2fc31cc8caa 100644
--- a/turbo/snapshotsync/freezeblocks/block_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go
@@ -1123,7 +1123,6 @@ func SegmentsCaplin(dir string) (res []snaptype.FileInfo, missingSnapshots []Ran
 	if err != nil {
 		return nil, missingSnapshots, err
 	}
-	fmt.Println(list)
 
 	{
 		var l []snaptype.FileInfo
@@ -1134,7 +1133,8 @@ func SegmentsCaplin(dir string) (res []snaptype.FileInfo, missingSnapshots []Ran
 			}
 			l = append(l, f)
 		}
-		l, m = noGaps(noOverlaps(allTypeOfSegmentsMustExist(dir, l)))
+		fmt.Println(l)
+		l, m = noGaps(noOverlaps(l))
 		res = append(res, l...)
 		missingSnapshots = append(missingSnapshots, m...)
 	}

From a6fbe3a536f5ae3125b8c78a3f797ff9bc53cea0 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 19:12:29 +0200
Subject: [PATCH 17/63] save

---
 cl/persistence/format/snapshot_format/blocks.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cl/persistence/format/snapshot_format/blocks.go b/cl/persistence/format/snapshot_format/blocks.go
index e9ed846f866..00d2aba5793 100644
--- a/cl/persistence/format/snapshot_format/blocks.go
+++ b/cl/persistence/format/snapshot_format/blocks.go
@@ -127,6 +127,7 @@ func readMetadataForBlock(r io.Reader) (clparams.StateVersion, error) {
 	if _, err := r.Read(b); err != nil {
 		return 0, err
 	}
+	fmt.Println(b)
 	return clparams.StateVersion(b[0]), nil
 }
 

From 87136e109e6cd77cedb4342734e694461cf0612c Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 19:16:06 +0200
Subject: [PATCH 18/63] save

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 6311e536b51..022b12d88df 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -473,7 +473,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 
 	br := &snapshot_format.MockBlockReader{}
 	snReader := freezeblocks.NewBeaconSnapshotReader(csn, br)
-	for i := uint64(0); i < to; i++ {
+	for i := uint64(1); i < to; i++ {
 		// Read the original canonical slot
 		data, err := beaconDB.GetBlock(ctx, tx, i)
 		if err != nil {

From 622624599f80bcb611f1a6833a39920786a0d80d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 19:19:27 +0200
Subject: [PATCH 19/63] save

---
 turbo/snapshotsync/freezeblocks/block_snapshots.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/block_snapshots.go b/turbo/snapshotsync/freezeblocks/block_snapshots.go
index 2fc31cc8caa..97e791128bb 100644
--- a/turbo/snapshotsync/freezeblocks/block_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/block_snapshots.go
@@ -1133,7 +1133,6 @@ func SegmentsCaplin(dir string) (res []snaptype.FileInfo, missingSnapshots []Ran
 			}
 			l = append(l, f)
 		}
-		fmt.Println(l)
 		l, m = noGaps(noOverlaps(l))
 		res = append(res, l...)
 		missingSnapshots = append(missingSnapshots, m...)

From fc21ce9623a233ccb713679d6298ac1edf47af38 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 21:13:10 +0200
Subject: [PATCH 20/63] better

---
 cl/cltypes/beacon_block.go                    |  4 +-
 .../format/snapshot_format/blocks.go          | 47 ++++---------------
 .../format/snapshot_format/blocks_test.go     |  5 +-
 3 files changed, 15 insertions(+), 41 deletions(-)

diff --git a/cl/cltypes/beacon_block.go b/cl/cltypes/beacon_block.go
index d09f404f9e5..18a7173e365 100644
--- a/cl/cltypes/beacon_block.go
+++ b/cl/cltypes/beacon_block.go
@@ -94,7 +94,9 @@ func NewBeaconBlock(beaconCfg *clparams.BeaconChainConfig) *BeaconBlock {
 }
 
 func NewBeaconBody(beaconCfg *clparams.BeaconChainConfig) *BeaconBody {
-	return &BeaconBody{beaconCfg: beaconCfg}
+	return &BeaconBody{
+		beaconCfg: beaconCfg,
+	}
 }
 
 // Version returns beacon block version.
diff --git a/cl/persistence/format/snapshot_format/blocks.go b/cl/persistence/format/snapshot_format/blocks.go
index 00d2aba5793..4497280980c 100644
--- a/cl/persistence/format/snapshot_format/blocks.go
+++ b/cl/persistence/format/snapshot_format/blocks.go
@@ -86,25 +86,10 @@ func WriteBlockForSnapshot(block *cltypes.SignedBeaconBlock, w io.Writer) error
 	// count in body for phase0 fields
 	currentChunkLength += uint64(body.ProposerSlashings.EncodingSizeSSZ())
 	currentChunkLength += uint64(body.AttesterSlashings.EncodingSizeSSZ())
-
-	// Write the chunk and chunk attestations
-	if err := chunk_encoding.WriteChunk(w, encoded[:currentChunkLength], chunk_encoding.ChunkDataType); err != nil {
-		return err
-	}
-	encoded = encoded[currentChunkLength:]
-	snappyWriter := snappy.NewBufferedWriter(w)
-	if err := chunk_encoding.WriteChunk(snappyWriter, encoded[:uint64(body.Attestations.EncodingSizeSSZ())], chunk_encoding.ChunkDataType); err != nil {
-		return err
-	}
-	if err := snappyWriter.Close(); err != nil {
-		return err
-	}
-	encoded = encoded[body.Attestations.EncodingSizeSSZ():]
-	currentChunkLength = 0
-
+	currentChunkLength += uint64(body.Attestations.EncodingSizeSSZ())
 	currentChunkLength += uint64(body.Deposits.EncodingSizeSSZ())
 	currentChunkLength += uint64(body.VoluntaryExits.EncodingSizeSSZ())
-
+	// Write the chunk and chunk attestations
 	if err := chunk_encoding.WriteChunk(w, encoded[:currentChunkLength], chunk_encoding.ChunkDataType); err != nil {
 		return err
 	}
@@ -146,28 +131,12 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	if err != nil {
 		return nil, err
 	}
+
 	if dT1 != chunk_encoding.ChunkDataType {
 		return nil, fmt.Errorf("malformed beacon block, invalid chunk 1 type %d, expected: %d", dT1, chunk_encoding.ChunkDataType)
 	}
 	plainSSZ = append(plainSSZ, chunk1...)
-	// Read the attestation chunk (2nd chunk)
-	chunk2, dT2, err := chunk_encoding.ReadChunk(snappy.NewReader(r))
-	if err != nil {
-		return nil, err
-	}
-	if dT2 != chunk_encoding.ChunkDataType {
-		return nil, fmt.Errorf("malformed beacon block, invalid chunk 2 type %d, expected: %d", dT2, chunk_encoding.ChunkDataType)
-	}
-	plainSSZ = append(plainSSZ, chunk2...)
-	// Read the 3rd chunk
-	chunk3, dT3, err := chunk_encoding.ReadChunk(r)
-	if err != nil {
-		return nil, err
-	}
-	if dT3 != chunk_encoding.ChunkDataType {
-		return nil, fmt.Errorf("malformed beacon block, invalid chunk 3 type %d, expected: %d", dT3, chunk_encoding.ChunkDataType)
-	}
-	plainSSZ = append(plainSSZ, chunk3...)
+
 	if v <= clparams.AltairVersion {
 		return block, block.DecodeSSZ(plainSSZ, int(v))
 	}
@@ -181,24 +150,24 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 		return nil, err
 	}
 	// Read the 4th chunk
-	chunk4, err := executionBlock.EncodeSSZ(nil)
+	chunk2, err := executionBlock.EncodeSSZ(nil)
 	if err != nil {
 		return nil, err
 	}
-	plainSSZ = append(plainSSZ, chunk4...)
+	plainSSZ = append(plainSSZ, chunk2...)
 	if v <= clparams.BellatrixVersion {
 		return block, block.DecodeSSZ(plainSSZ, int(v))
 	}
 
 	// Read the 5h chunk
-	chunk5, dT5, err := chunk_encoding.ReadChunk(r)
+	chunk3, dT5, err := chunk_encoding.ReadChunk(r)
 	if err != nil {
 		return nil, err
 	}
 	if dT5 != chunk_encoding.ChunkDataType {
 		return nil, fmt.Errorf("malformed beacon block, invalid chunk 5 type %d, expected: %d", dT5, chunk_encoding.ChunkDataType)
 	}
-	plainSSZ = append(plainSSZ, chunk5...)
+	plainSSZ = append(plainSSZ, chunk3...)
 
 	return block, block.DecodeSSZ(plainSSZ, int(v))
 }
diff --git a/cl/persistence/format/snapshot_format/blocks_test.go b/cl/persistence/format/snapshot_format/blocks_test.go
index c0e9b46bee6..571280b61d6 100644
--- a/cl/persistence/format/snapshot_format/blocks_test.go
+++ b/cl/persistence/format/snapshot_format/blocks_test.go
@@ -27,8 +27,11 @@ var capellaBlockSSZSnappy []byte
 //go:embed test_data/deneb.ssz_snappy
 var denebBlockSSZSnappy []byte
 
+var emptyBlock = cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
+
 // obtain the test blocks
 func getTestBlocks(t *testing.T) []*cltypes.SignedBeaconBlock {
+	emptyBlock.EncodingSizeSSZ()
 	denebBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
 	capellaBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
 	bellatrixBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
@@ -40,7 +43,7 @@ func getTestBlocks(t *testing.T) []*cltypes.SignedBeaconBlock {
 	require.NoError(t, utils.DecodeSSZSnappy(bellatrixBlock, bellatrixBlockSSZSnappy, int(clparams.BellatrixVersion)))
 	require.NoError(t, utils.DecodeSSZSnappy(altairBlock, altairBlockSSZSnappy, int(clparams.AltairVersion)))
 	require.NoError(t, utils.DecodeSSZSnappy(phase0Block, phase0BlockSSZSnappy, int(clparams.Phase0Version)))
-	return []*cltypes.SignedBeaconBlock{phase0Block, altairBlock, bellatrixBlock, capellaBlock, denebBlock}
+	return []*cltypes.SignedBeaconBlock{phase0Block, altairBlock, bellatrixBlock, capellaBlock, denebBlock, emptyBlock}
 }
 
 func TestBlockSnapshotEncoding(t *testing.T) {

From 5c1fe96a4d8e873a93f2076696d9b395dea1f9b9 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 21:52:05 +0200
Subject: [PATCH 21/63] save

---
 cmd/capcli/cli.go | 10 +++++++++-
 1 file changed, 9 insertions(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 022b12d88df..e01ddb45897 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -7,6 +7,8 @@ import (
 	"strings"
 	"time"
 
+	libcommon "github.com/ledgerwatch/erigon-lib/common"
+
 	"github.com/ledgerwatch/erigon/cl/abstract"
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -501,10 +503,16 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 			log.Error("Block not found in snapshot", "slot", i)
 			return nil
 		}
+
+		d1, _ := blk.EncodeSSZ(nil)
+		d2, _ := blk2.EncodeSSZ(nil)
+		fmt.Println(fmt.Println(d1[:140]))
+		fmt.Println(fmt.Println(d2[:140]))
 		hash1, _ := blk.HashSSZ()
 		hash2, _ := blk2.HashSSZ()
 		if hash1 != hash2 {
-			log.Error("Mismatching blocks", "slot", i, "datadir", hash1, "snapshot", hash2)
+			log.Error("Mismatching blocks", "slot", i, "datadir", libcommon.Hash(hash1), "snapshot", libcommon.Hash(hash2))
+			return nil
 		}
 		log.Info("Successfully checked", "slot", i)
 	}

From f42b7ee39ac191b4ee4dd4d49d767b0ec7a17386 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 21:57:37 +0200
Subject: [PATCH 22/63] save

---
 cmd/capcli/cli.go | 10 +++-------
 1 file changed, 3 insertions(+), 7 deletions(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index e01ddb45897..70576920bfb 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -494,7 +494,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 		if blk.Version() >= clparams.BellatrixVersion {
 			br.Block = blk.Block.Body.ExecutionPayload
 		}
-		blk2, err := snReader.ReadBlock(i)
+		blk2, err := snReader.ReadBlock(i + 1)
 		if err != nil {
 			log.Error("Error detected in decoding snapshots", "err", err, "slot", i)
 			return nil
@@ -504,12 +504,8 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 			return nil
 		}
 
-		d1, _ := blk.EncodeSSZ(nil)
-		d2, _ := blk2.EncodeSSZ(nil)
-		fmt.Println(fmt.Println(d1[:140]))
-		fmt.Println(fmt.Println(d2[:140]))
-		hash1, _ := blk.HashSSZ()
-		hash2, _ := blk2.HashSSZ()
+		hash1, _ := blk.Block.HashSSZ()
+		hash2, _ := blk2.Block.HashSSZ()
 		if hash1 != hash2 {
 			log.Error("Mismatching blocks", "slot", i, "datadir", libcommon.Hash(hash1), "snapshot", libcommon.Hash(hash2))
 			return nil

From 76791d2432924a206e134217cd9d1f691ee50729 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 21:59:42 +0200
Subject: [PATCH 23/63] save

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 70576920bfb..6860066e94d 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -494,7 +494,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 		if blk.Version() >= clparams.BellatrixVersion {
 			br.Block = blk.Block.Body.ExecutionPayload
 		}
-		blk2, err := snReader.ReadBlock(i + 1)
+		blk2, err := snReader.ReadBlock(i)
 		if err != nil {
 			log.Error("Error detected in decoding snapshots", "err", err, "slot", i)
 			return nil

From c39b2a22dbad753148742a0160976b33ed6e842e Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 22:00:34 +0200
Subject: [PATCH 24/63] save

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 6860066e94d..769ac9d9171 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -508,7 +508,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 		hash2, _ := blk2.Block.HashSSZ()
 		if hash1 != hash2 {
 			log.Error("Mismatching blocks", "slot", i, "datadir", libcommon.Hash(hash1), "snapshot", libcommon.Hash(hash2))
-			return nil
+			//return nil
 		}
 		log.Info("Successfully checked", "slot", i)
 	}

From 9a7e94d7e35fe54c2aa69bd7d7baec7cdbccd6b4 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 22:00:55 +0200
Subject: [PATCH 25/63] save

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 769ac9d9171..63b0be7f660 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -507,7 +507,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 		hash1, _ := blk.Block.HashSSZ()
 		hash2, _ := blk2.Block.HashSSZ()
 		if hash1 != hash2 {
-			log.Error("Mismatching blocks", "slot", i, "datadir", libcommon.Hash(hash1), "snapshot", libcommon.Hash(hash2))
+			log.Error("Mismatching blocks", "slot", i, "gotSlot", blk2.Block.Slot, "datadir", libcommon.Hash(hash1), "snapshot", libcommon.Hash(hash2))
 			//return nil
 		}
 		log.Info("Successfully checked", "slot", i)

From 00e295cf43cbba38c6fa1096ad905d8ec735ed24 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 22:02:42 +0200
Subject: [PATCH 26/63] save

---
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index 50df22606ad..4cb279c4342 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -380,7 +380,7 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 			return err
 		}
 		if obj == nil {
-			if err := sn.AddWord(nil); err != nil {
+			if err := sn.AddWord([]byte{0xff}); err != nil {
 				return err
 			}
 			continue

From 47f51c27ad40bce6bbb5c1496fa36984c60bc947 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 22:27:35 +0200
Subject: [PATCH 27/63] fixed?

---
 .../freezeblocks/caplin_snapshots.go          | 56 ++++---------------
 1 file changed, 12 insertions(+), 44 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index 4cb279c4342..19bb5b1bbe4 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -108,60 +108,28 @@ func BeaconBlocksIdx(ctx context.Context, segmentFilePath string, blockFrom, blo
 			err = fmt.Errorf("BeaconBlocksIdx: at=%d-%d, %v, %s", blockFrom, blockTo, rec, dbg.Stack())
 		}
 	}()
+	num := make([]byte, 8)
+
 	// Calculate how many records there will be in the index
 	d, err := compress.NewDecompressor(path.Join(snapDir, segmentFilePath))
 	if err != nil {
 		return err
 	}
 	defer d.Close()
-	g := d.MakeGetter()
-	var idxFilePath = filepath.Join(snapDir, snaptype.IdxFileName(blockFrom, blockTo, snaptype.BeaconBlocks.String()))
 
-	var baseSpanId uint64
-	if blockFrom > zerothSpanEnd {
-		baseSpanId = 1 + (blockFrom-zerothSpanEnd-1)/spanLength
-	}
+	_, fname := filepath.Split(segmentFilePath)
+	p.Name.Store(&fname)
+	p.Total.Store(uint64(d.Count()))
 
-	rs, err := recsplit.NewRecSplit(recsplit.RecSplitArgs{
-		KeyCount:   d.Count(),
-		Enums:      d.Count() > 0,
-		BucketSize: 2000,
-		LeafSize:   8,
-		TmpDir:     tmpDir,
-		IndexFile:  idxFilePath,
-		BaseDataID: baseSpanId,
-	}, logger)
-	if err != nil {
-		return err
-	}
-	rs.LogLvl(log.LvlDebug)
-
-	defer d.EnableMadvNormal().DisableReadAhead()
-RETRY:
-	g.Reset(0)
-	var i, offset, nextPos uint64
-	var key [8]byte
-	for g.HasNext() {
-		nextPos, _ = g.Skip()
-		binary.BigEndian.PutUint64(key[:], i)
-		i++
-		if err = rs.AddKey(key[:], offset); err != nil {
+	if err := Idx(ctx, d, blockFrom, tmpDir, log.LvlDebug, func(idx *recsplit.RecSplit, i, offset uint64, word []byte) error {
+		p.Processed.Add(1)
+		n := binary.PutUvarint(num, i)
+		if err := idx.AddKey(num[:n], offset); err != nil {
 			return err
 		}
-		select {
-		case <-ctx.Done():
-			return ctx.Err()
-		default:
-		}
-		offset = nextPos
-	}
-	if err = rs.Build(ctx); err != nil {
-		if errors.Is(err, recsplit.ErrCollision) {
-			logger.Info("Building recsplit. Collision happened. It's ok. Restarting with another salt...", "err", err)
-			rs.ResetNextSalt()
-			goto RETRY
-		}
-		return err
+		return nil
+	}, logger); err != nil {
+		return fmt.Errorf("BodyNumberIdx: %w", err)
 	}
 
 	return nil

From b8ea1177a76cf38e72662fec37e8576441ffe4b8 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 22:49:21 +0200
Subject: [PATCH 28/63] whoops

---
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 8 ++++----
 1 file changed, 4 insertions(+), 4 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index 19bb5b1bbe4..03a5e6fac68 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -102,7 +102,7 @@ func (s *beaconBlockSegments) View(f func(segments []*BeaconBlockSegment) error)
 	return f(s.segments)
 }
 
-func BeaconBlocksIdx(ctx context.Context, segmentFilePath string, blockFrom, blockTo uint64, snapDir string, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) {
+func BeaconBlocksIdx(ctx context.Context, sn snaptype.FileInfo, segmentFilePath string, blockFrom, blockTo uint64, snapDir string, tmpDir string, p *background.Progress, lvl log.Lvl, logger log.Logger) (err error) {
 	defer func() {
 		if rec := recover(); rec != nil {
 			err = fmt.Errorf("BeaconBlocksIdx: at=%d-%d, %v, %s", blockFrom, blockTo, rec, dbg.Stack())
@@ -121,7 +121,7 @@ func BeaconBlocksIdx(ctx context.Context, segmentFilePath string, blockFrom, blo
 	p.Name.Store(&fname)
 	p.Total.Store(uint64(d.Count()))
 
-	if err := Idx(ctx, d, blockFrom, tmpDir, log.LvlDebug, func(idx *recsplit.RecSplit, i, offset uint64, word []byte) error {
+	if err := Idx(ctx, d, sn.From, tmpDir, log.LvlDebug, func(idx *recsplit.RecSplit, i, offset uint64, word []byte) error {
 		p.Processed.Add(1)
 		n := binary.PutUvarint(num, i)
 		if err := idx.AddKey(num[:n], offset); err != nil {
@@ -348,7 +348,7 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 			return err
 		}
 		if obj == nil {
-			if err := sn.AddWord([]byte{0xff}); err != nil {
+			if err := sn.AddWord(nil); err != nil {
 				return err
 			}
 			continue
@@ -367,7 +367,7 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 	// Generate .idx file, which is the slot => offset mapping.
 	p := &background.Progress{}
 
-	return BeaconBlocksIdx(ctx, segName, fromSlot, toSlot, snapDir, tmpDir, p, lvl, logger)
+	return BeaconBlocksIdx(ctx, f, segName, fromSlot, toSlot, snapDir, tmpDir, p, lvl, logger)
 }
 
 func DumpBeaconBlocks(ctx context.Context, db kv.RoDB, b persistence.BlockSource, fromSlot, toSlot, blocksPerFile uint64, tmpDir, snapDir string, workers int, lvl log.Lvl, logger log.Logger) error {

From d7a9fb637443e096e5a2636d1d16db3703f24eb6 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Mon, 23 Oct 2023 23:03:35 +0200
Subject: [PATCH 29/63] log

---
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index cbcb76ce427..99273814409 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -2,6 +2,7 @@ package freezeblocks
 
 import (
 	"bytes"
+	"fmt"
 
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -56,6 +57,7 @@ func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
 	if seg.idxSlot == nil {
 		return nil, nil
 	}
+	fmt.Println(seg.idxSlot.BaseDataID())
 	blockOffset := seg.idxSlot.OrdinalLookup(slot - seg.idxSlot.BaseDataID())
 
 	gg := seg.seg.MakeGetter()

From 0c627565d85e5e448ee8c906bb0fbfaf1a708037 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 12:26:31 +0200
Subject: [PATCH 30/63] save

---
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index 99273814409..ac01377c33f 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -57,7 +57,7 @@ func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
 	if seg.idxSlot == nil {
 		return nil, nil
 	}
-	fmt.Println(seg.idxSlot.BaseDataID())
+	fmt.Println(seg.seg.Count())
 	blockOffset := seg.idxSlot.OrdinalLookup(slot - seg.idxSlot.BaseDataID())
 
 	gg := seg.seg.MakeGetter()

From f24e736926ec8d1168b175f35209966f28b641ae Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 12:30:49 +0200
Subject: [PATCH 31/63] save

---
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 2 --
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go    | 2 +-
 2 files changed, 1 insertion(+), 3 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index ac01377c33f..cbcb76ce427 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -2,7 +2,6 @@ package freezeblocks
 
 import (
 	"bytes"
-	"fmt"
 
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -57,7 +56,6 @@ func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
 	if seg.idxSlot == nil {
 		return nil, nil
 	}
-	fmt.Println(seg.seg.Count())
 	blockOffset := seg.idxSlot.OrdinalLookup(slot - seg.idxSlot.BaseDataID())
 
 	gg := seg.seg.MakeGetter()
diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index 03a5e6fac68..51f771a4c45 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -342,7 +342,7 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 	defer tx.Rollback()
 	// Generate .seg file, which is just the list of beacon blocks.
 	var buf bytes.Buffer
-	for i := fromSlot; i <= toSlot; i++ {
+	for i := fromSlot; i < toSlot; i++ {
 		obj, err := b.GetBlock(ctx, tx, i)
 		if err != nil {
 			return err

From bdfb1fb608b113037dd64cba0313e1d7a47e0f83 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 12:50:26 +0200
Subject: [PATCH 32/63] save

---
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index cbcb76ce427..524b91b7205 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -2,6 +2,7 @@ package freezeblocks
 
 import (
 	"bytes"
+	"fmt"
 
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -56,6 +57,7 @@ func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
 	if seg.idxSlot == nil {
 		return nil, nil
 	}
+	fmt.Println(seg.idxSlot.BaseDataID(), seg.seg.Count())
 	blockOffset := seg.idxSlot.OrdinalLookup(slot - seg.idxSlot.BaseDataID())
 
 	gg := seg.seg.MakeGetter()

From d4630ca6389aaf9ad80517231eaa486e94571970 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 12:55:10 +0200
Subject: [PATCH 33/63] save

---
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 3 +++
 1 file changed, 3 insertions(+)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index 51f771a4c45..f1571dbe804 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -359,6 +359,9 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 		if err := sn.AddWord(buf.Bytes()); err != nil {
 			return err
 		}
+		if i%20_000 == 0 {
+			logger.Log(lvl, "Dumping beacon blocks", "progress", i)
+		}
 		buf.Reset()
 	}
 	if err := sn.Compress(); err != nil {

From 1733224ed01c4b90ca03f058054b4acdeb6e1beb Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 13:00:02 +0200
Subject: [PATCH 34/63] save

---
 cl/persistence/format/snapshot_format/blocks.go        |  1 -
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go |  2 +-
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go    | 10 +++++++---
 3 files changed, 8 insertions(+), 5 deletions(-)

diff --git a/cl/persistence/format/snapshot_format/blocks.go b/cl/persistence/format/snapshot_format/blocks.go
index 4497280980c..f62443cbb94 100644
--- a/cl/persistence/format/snapshot_format/blocks.go
+++ b/cl/persistence/format/snapshot_format/blocks.go
@@ -112,7 +112,6 @@ func readMetadataForBlock(r io.Reader) (clparams.StateVersion, error) {
 	if _, err := r.Read(b); err != nil {
 		return 0, err
 	}
-	fmt.Println(b)
 	return clparams.StateVersion(b[0]), nil
 }
 
diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index 524b91b7205..2a6036148e3 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -57,7 +57,7 @@ func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
 	if seg.idxSlot == nil {
 		return nil, nil
 	}
-	fmt.Println(seg.idxSlot.BaseDataID(), seg.seg.Count())
+	fmt.Println(seg.idxSlot.BaseDataID(), seg.seg.Count(), seg.idxSlot.KeyCount())
 	blockOffset := seg.idxSlot.OrdinalLookup(slot - seg.idxSlot.BaseDataID())
 
 	gg := seg.seg.MakeGetter()
diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index f1571dbe804..715de529292 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -122,6 +122,9 @@ func BeaconBlocksIdx(ctx context.Context, sn snaptype.FileInfo, segmentFilePath
 	p.Total.Store(uint64(d.Count()))
 
 	if err := Idx(ctx, d, sn.From, tmpDir, log.LvlDebug, func(idx *recsplit.RecSplit, i, offset uint64, word []byte) error {
+		if i%20_000 == 0 {
+			logger.Log(lvl, "Dumping beacon blocks", "progress", i)
+		}
 		p.Processed.Add(1)
 		n := binary.PutUvarint(num, i)
 		if err := idx.AddKey(num[:n], offset); err != nil {
@@ -347,7 +350,11 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 		if err != nil {
 			return err
 		}
+		if i%20_000 == 0 {
+			logger.Log(lvl, "Dumping beacon blocks", "progress", i)
+		}
 		if obj == nil {
+
 			if err := sn.AddWord(nil); err != nil {
 				return err
 			}
@@ -359,9 +366,6 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 		if err := sn.AddWord(buf.Bytes()); err != nil {
 			return err
 		}
-		if i%20_000 == 0 {
-			logger.Log(lvl, "Dumping beacon blocks", "progress", i)
-		}
 		buf.Reset()
 	}
 	if err := sn.Compress(); err != nil {

From 03866ad57831afa19c53b79a21d66794faca9204 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 13:12:05 +0200
Subject: [PATCH 35/63] save

---
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 2 +-
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go    | 4 ++--
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index 2a6036148e3..74d6ed8f73e 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -57,7 +57,7 @@ func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
 	if seg.idxSlot == nil {
 		return nil, nil
 	}
-	fmt.Println(seg.idxSlot.BaseDataID(), seg.seg.Count(), seg.idxSlot.KeyCount())
+	fmt.Println(slot, seg.idxSlot.BaseDataID(), seg.seg.Count(), seg.idxSlot.KeyCount())
 	blockOffset := seg.idxSlot.OrdinalLookup(slot - seg.idxSlot.BaseDataID())
 
 	gg := seg.seg.MakeGetter()
diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index 715de529292..d0bdecc0687 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -122,8 +122,8 @@ func BeaconBlocksIdx(ctx context.Context, sn snaptype.FileInfo, segmentFilePath
 	p.Total.Store(uint64(d.Count()))
 
 	if err := Idx(ctx, d, sn.From, tmpDir, log.LvlDebug, func(idx *recsplit.RecSplit, i, offset uint64, word []byte) error {
-		if i%20_000 == 0 {
-			logger.Log(lvl, "Dumping beacon blocks", "progress", i)
+		if i%100_000 == 0 {
+			logger.Log(lvl, "Compressing beacon blocks", "progress", i)
 		}
 		p.Processed.Add(1)
 		n := binary.PutUvarint(num, i)

From 611ab679d40332af954c5c846a577ad69143cb8e Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 13:16:28 +0200
Subject: [PATCH 36/63] save

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 63b0be7f660..60de85a8f3f 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -477,7 +477,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 	snReader := freezeblocks.NewBeaconSnapshotReader(csn, br)
 	for i := uint64(1); i < to; i++ {
 		// Read the original canonical slot
-		data, err := beaconDB.GetBlock(ctx, tx, i)
+		data, err := beaconDB.GetBlock(ctx, tx, i+1)
 		if err != nil {
 			return err
 		}

From 709d065cf029e6f5b86dbad204f5f6b66de2e077 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 13:23:09 +0200
Subject: [PATCH 37/63] save

---
 cmd/capcli/cli.go                                      | 2 +-
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 1 +
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go    | 3 +++
 3 files changed, 5 insertions(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 60de85a8f3f..63b0be7f660 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -477,7 +477,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 	snReader := freezeblocks.NewBeaconSnapshotReader(csn, br)
 	for i := uint64(1); i < to; i++ {
 		// Read the original canonical slot
-		data, err := beaconDB.GetBlock(ctx, tx, i+1)
+		data, err := beaconDB.GetBlock(ctx, tx, i)
 		if err != nil {
 			return err
 		}
diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index 74d6ed8f73e..1ebae6ef710 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -61,6 +61,7 @@ func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
 	blockOffset := seg.idxSlot.OrdinalLookup(slot - seg.idxSlot.BaseDataID())
 
 	gg := seg.seg.MakeGetter()
+	fmt.Println(blockOffset, slot-seg.idxSlot.BaseDataID())
 	gg.Reset(blockOffset)
 	if !gg.HasNext() {
 		return nil, nil
diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index d0bdecc0687..9864f4fd5eb 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -125,6 +125,9 @@ func BeaconBlocksIdx(ctx context.Context, sn snaptype.FileInfo, segmentFilePath
 		if i%100_000 == 0 {
 			logger.Log(lvl, "Compressing beacon blocks", "progress", i)
 		}
+		if i == 0 {
+			fmt.Println(offset, i)
+		}
 		p.Processed.Add(1)
 		n := binary.PutUvarint(num, i)
 		if err := idx.AddKey(num[:n], offset); err != nil {

From f622c52520a6d92608a95c55d5e33f42959f0bd4 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 13:23:57 +0200
Subject: [PATCH 38/63] save

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 63b0be7f660..ae4ca87a5d5 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -475,7 +475,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 
 	br := &snapshot_format.MockBlockReader{}
 	snReader := freezeblocks.NewBeaconSnapshotReader(csn, br)
-	for i := uint64(1); i < to; i++ {
+	for i := uint64(0); i < to; i++ {
 		// Read the original canonical slot
 		data, err := beaconDB.GetBlock(ctx, tx, i)
 		if err != nil {

From a8446a43863c83ee7ca2f7bc4702cd15fe80a9f8 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 13:32:20 +0200
Subject: [PATCH 39/63] save

---
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index 9864f4fd5eb..c349fb15853 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -127,6 +127,7 @@ func BeaconBlocksIdx(ctx context.Context, sn snaptype.FileInfo, segmentFilePath
 		}
 		if i == 0 {
 			fmt.Println(offset, i)
+			fmt.Println(word[:140])
 		}
 		p.Processed.Add(1)
 		n := binary.PutUvarint(num, i)

From 06ac3260d663af1148b78f3b7d0272cb73cfecc3 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 14:06:35 +0200
Subject: [PATCH 40/63] save

---
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 1 -
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go    | 8 +++++++-
 2 files changed, 7 insertions(+), 2 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index 1ebae6ef710..74d6ed8f73e 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -61,7 +61,6 @@ func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
 	blockOffset := seg.idxSlot.OrdinalLookup(slot - seg.idxSlot.BaseDataID())
 
 	gg := seg.seg.MakeGetter()
-	fmt.Println(blockOffset, slot-seg.idxSlot.BaseDataID())
 	gg.Reset(blockOffset)
 	if !gg.HasNext() {
 		return nil, nil
diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index c349fb15853..b3d6e77c204 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -367,7 +367,13 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 		if err := snapshot_format.WriteBlockForSnapshot(obj.Data, &buf); err != nil {
 			return err
 		}
-		if err := sn.AddWord(buf.Bytes()); err != nil {
+		word := buf.Bytes()
+
+		if i == 0 {
+			fmt.Println(word[:140])
+		}
+
+		if err := sn.AddWord(word); err != nil {
 			return err
 		}
 		buf.Reset()

From 9b3c73c06ed019453a7eb23d32e79e96a354bc9e Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 14:08:00 +0200
Subject: [PATCH 41/63] save

---
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index b3d6e77c204..e1160294f61 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -370,6 +370,7 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 		word := buf.Bytes()
 
 		if i == 0 {
+			fmt.Println(obj.Data.Block.Slot, i)
 			fmt.Println(word[:140])
 		}
 

From a770ed5ae4707612ab7c16f371d9bd1d969bfd7d Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 14:12:27 +0200
Subject: [PATCH 42/63] save

---
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index e1160294f61..08c710f4dc1 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -369,7 +369,7 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 		}
 		word := buf.Bytes()
 
-		if i == 0 {
+		if i == 0 || i == 1 {
 			fmt.Println(obj.Data.Block.Slot, i)
 			fmt.Println(word[:140])
 		}

From c001617dfca21185b7e043fa306b8f6afc23b1ea Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 14:14:43 +0200
Subject: [PATCH 43/63] save

---
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index 08c710f4dc1..ce75c66e975 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -371,7 +371,7 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 
 		if i == 0 || i == 1 {
 			fmt.Println(obj.Data.Block.Slot, i)
-			fmt.Println(word[:140])
+			fmt.Println(word[:108])
 		}
 
 		if err := sn.AddWord(word); err != nil {

From 177a4a406679bd99ee3e9b5b3704ec8a50382c53 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 14:18:16 +0200
Subject: [PATCH 44/63] save

---
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index ce75c66e975..bf2b121204a 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -348,7 +348,6 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 	}
 	defer tx.Rollback()
 	// Generate .seg file, which is just the list of beacon blocks.
-	var buf bytes.Buffer
 	for i := fromSlot; i < toSlot; i++ {
 		obj, err := b.GetBlock(ctx, tx, i)
 		if err != nil {
@@ -364,6 +363,7 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 			}
 			continue
 		}
+		var buf bytes.Buffer
 		if err := snapshot_format.WriteBlockForSnapshot(obj.Data, &buf); err != nil {
 			return err
 		}
@@ -377,7 +377,7 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 		if err := sn.AddWord(word); err != nil {
 			return err
 		}
-		buf.Reset()
+
 	}
 	if err := sn.Compress(); err != nil {
 		return fmt.Errorf("compress: %w", err)

From fa288d6f968bad00873b3c7c6ac3f6f0554727cb Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 14:19:38 +0200
Subject: [PATCH 45/63] save

---
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index bf2b121204a..b49c7de6a8b 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -371,7 +371,7 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 
 		if i == 0 || i == 1 {
 			fmt.Println(obj.Data.Block.Slot, i)
-			fmt.Println(word[:108])
+			fmt.Println(word[:108+33])
 		}
 
 		if err := sn.AddWord(word); err != nil {

From d27dded986a2c8c9025231995993e28fbe7b579c Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 14:20:56 +0200
Subject: [PATCH 46/63] save

---
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index b49c7de6a8b..f0bc11129a1 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -371,7 +371,7 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 
 		if i == 0 || i == 1 {
 			fmt.Println(obj.Data.Block.Slot, i)
-			fmt.Println(word[:108+33])
+			fmt.Println(word[:200])
 		}
 
 		if err := sn.AddWord(word); err != nil {

From b039ec28b254584b0163c8dbb59b92bbc7513bf6 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 14:21:55 +0200
Subject: [PATCH 47/63] save

---
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index f0bc11129a1..d3863b29fcf 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -127,7 +127,7 @@ func BeaconBlocksIdx(ctx context.Context, sn snaptype.FileInfo, segmentFilePath
 		}
 		if i == 0 {
 			fmt.Println(offset, i)
-			fmt.Println(word[:140])
+			fmt.Println(word[:200])
 		}
 		p.Processed.Add(1)
 		n := binary.PutUvarint(num, i)

From d956d673fd6410f59fedf26e47cdff0212b5bec9 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 14:44:00 +0200
Subject: [PATCH 48/63] save

---
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 4 ++--
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go    | 2 +-
 2 files changed, 3 insertions(+), 3 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index 74d6ed8f73e..aecdfb53e1c 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -65,7 +65,7 @@ func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
 	if !gg.HasNext() {
 		return nil, nil
 	}
-	gg.Next(buf)
-	buf, _ = gg.Next(buf[:0])
+	buf, _ = gg.Next(buf)
+	fmt.Println("buf", buf[:200])
 	return buf, nil
 }
diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index d3863b29fcf..8980aa4745d 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -108,7 +108,6 @@ func BeaconBlocksIdx(ctx context.Context, sn snaptype.FileInfo, segmentFilePath
 			err = fmt.Errorf("BeaconBlocksIdx: at=%d-%d, %v, %s", blockFrom, blockTo, rec, dbg.Stack())
 		}
 	}()
-	num := make([]byte, 8)
 
 	// Calculate how many records there will be in the index
 	d, err := compress.NewDecompressor(path.Join(snapDir, segmentFilePath))
@@ -130,6 +129,7 @@ func BeaconBlocksIdx(ctx context.Context, sn snaptype.FileInfo, segmentFilePath
 			fmt.Println(word[:200])
 		}
 		p.Processed.Add(1)
+		num := make([]byte, 8)
 		n := binary.PutUvarint(num, i)
 		if err := idx.AddKey(num[:n], offset); err != nil {
 			return err

From dbe7a08dba1c1b1369e64fcd5ab39c3d7900fbe7 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 15:00:08 +0200
Subject: [PATCH 49/63] no prints

---
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 1 -
 turbo/snapshotsync/freezeblocks/caplin_snapshots.go    | 9 ---------
 2 files changed, 10 deletions(-)

diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index aecdfb53e1c..7e18488bfbd 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -66,6 +66,5 @@ func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
 		return nil, nil
 	}
 	buf, _ = gg.Next(buf)
-	fmt.Println("buf", buf[:200])
 	return buf, nil
 }
diff --git a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
index 8980aa4745d..221cbe79301 100644
--- a/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
+++ b/turbo/snapshotsync/freezeblocks/caplin_snapshots.go
@@ -124,10 +124,6 @@ func BeaconBlocksIdx(ctx context.Context, sn snaptype.FileInfo, segmentFilePath
 		if i%100_000 == 0 {
 			logger.Log(lvl, "Compressing beacon blocks", "progress", i)
 		}
-		if i == 0 {
-			fmt.Println(offset, i)
-			fmt.Println(word[:200])
-		}
 		p.Processed.Add(1)
 		num := make([]byte, 8)
 		n := binary.PutUvarint(num, i)
@@ -369,11 +365,6 @@ func dumpBeaconBlocksRange(ctx context.Context, db kv.RoDB, b persistence.BlockS
 		}
 		word := buf.Bytes()
 
-		if i == 0 || i == 1 {
-			fmt.Println(obj.Data.Block.Slot, i)
-			fmt.Println(word[:200])
-		}
-
 		if err := sn.AddWord(word); err != nil {
 			return err
 		}

From 68d2a767ffd51246f62156f53af1dbf102d3dac8 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 15:09:02 +0200
Subject: [PATCH 50/63] save

---
 cl/cltypes/beacon_block.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cl/cltypes/beacon_block.go b/cl/cltypes/beacon_block.go
index 18a7173e365..8125342241c 100644
--- a/cl/cltypes/beacon_block.go
+++ b/cl/cltypes/beacon_block.go
@@ -174,6 +174,8 @@ func (b *BeaconBody) DecodeSSZ(buf []byte, version int) error {
 		return fmt.Errorf("[BeaconBody] err: %s", ssz.ErrLowBufferSize)
 	}
 
+	b.ExecutionPayload = NewEth1Block(b.Version, b.beaconCfg)
+
 	err := ssz2.UnmarshalSSZ(buf, version, b.getSchema(false)...)
 	return err
 }

From 0f649e215eb789cfcd173131334b75268a6893c7 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 15:13:02 +0200
Subject: [PATCH 51/63] save

---
 cl/cltypes/eth1_block.go | 1 +
 1 file changed, 1 insertion(+)

diff --git a/cl/cltypes/eth1_block.go b/cl/cltypes/eth1_block.go
index ef5819c648c..e214b45f9a6 100644
--- a/cl/cltypes/eth1_block.go
+++ b/cl/cltypes/eth1_block.go
@@ -161,6 +161,7 @@ func (b *Eth1Block) EncodingSizeSSZ() (size int) {
 func (b *Eth1Block) DecodeSSZ(buf []byte, version int) error {
 	b.Extra = solid.NewExtraData()
 	b.Transactions = &solid.TransactionsSSZ{}
+	fmt.Println(b.beaconCfg)
 	b.Withdrawals = solid.NewStaticListSSZ[*Withdrawal](int(b.beaconCfg.MaxWithdrawalsPerPayload), 44)
 	b.version = clparams.StateVersion(version)
 	return ssz2.UnmarshalSSZ(buf, version, b.getSchema()...)

From 522a38264b053000bff954684ef86c76bff78a4c Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 15:25:06 +0200
Subject: [PATCH 52/63] save

---
 cmd/capcli/cli.go                                      | 2 +-
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 6 ++----
 2 files changed, 3 insertions(+), 5 deletions(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index ae4ca87a5d5..69d240f10b0 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -474,7 +474,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 	}
 
 	br := &snapshot_format.MockBlockReader{}
-	snReader := freezeblocks.NewBeaconSnapshotReader(csn, br)
+	snReader := freezeblocks.NewBeaconSnapshotReader(csn, br, beaconConfig)
 	for i := uint64(0); i < to; i++ {
 		// Read the original canonical slot
 		data, err := beaconDB.GetBlock(ctx, tx, i)
diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index 7e18488bfbd..fdced61a050 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -2,7 +2,6 @@ package freezeblocks
 
 import (
 	"bytes"
-	"fmt"
 
 	"github.com/ledgerwatch/erigon/cl/clparams"
 	"github.com/ledgerwatch/erigon/cl/cltypes"
@@ -25,8 +24,8 @@ type beaconSnapshotReader struct {
 	cfg        *clparams.BeaconChainConfig
 }
 
-func NewBeaconSnapshotReader(snapshots *CaplinSnapshots, eth1Getter snapshot_format.ExecutionBlockReaderByNumber) BeaconSnapshotReader {
-	return &beaconSnapshotReader{sn: snapshots, eth1Getter: eth1Getter}
+func NewBeaconSnapshotReader(snapshots *CaplinSnapshots, eth1Getter snapshot_format.ExecutionBlockReaderByNumber, cfg *clparams.BeaconChainConfig) BeaconSnapshotReader {
+	return &beaconSnapshotReader{sn: snapshots, eth1Getter: eth1Getter, cfg: cfg}
 }
 
 func (r *beaconSnapshotReader) FrozenSlots() uint64 {
@@ -57,7 +56,6 @@ func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
 	if seg.idxSlot == nil {
 		return nil, nil
 	}
-	fmt.Println(slot, seg.idxSlot.BaseDataID(), seg.seg.Count(), seg.idxSlot.KeyCount())
 	blockOffset := seg.idxSlot.OrdinalLookup(slot - seg.idxSlot.BaseDataID())
 
 	gg := seg.seg.MakeGetter()

From 92779a85e144add9a4e42536e565cb3ecf2c42d4 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 15:25:52 +0200
Subject: [PATCH 53/63] save

---
 cl/cltypes/eth1_block.go | 1 -
 1 file changed, 1 deletion(-)

diff --git a/cl/cltypes/eth1_block.go b/cl/cltypes/eth1_block.go
index e214b45f9a6..ef5819c648c 100644
--- a/cl/cltypes/eth1_block.go
+++ b/cl/cltypes/eth1_block.go
@@ -161,7 +161,6 @@ func (b *Eth1Block) EncodingSizeSSZ() (size int) {
 func (b *Eth1Block) DecodeSSZ(buf []byte, version int) error {
 	b.Extra = solid.NewExtraData()
 	b.Transactions = &solid.TransactionsSSZ{}
-	fmt.Println(b.beaconCfg)
 	b.Withdrawals = solid.NewStaticListSSZ[*Withdrawal](int(b.beaconCfg.MaxWithdrawalsPerPayload), 44)
 	b.version = clparams.StateVersion(version)
 	return ssz2.UnmarshalSSZ(buf, version, b.getSchema()...)

From ced23d777cf15d7c6022fe1acd7b7c42e96c7429 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 15:26:04 +0200
Subject: [PATCH 54/63] save

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 69d240f10b0..d5931ece112 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -508,7 +508,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 		hash2, _ := blk2.Block.HashSSZ()
 		if hash1 != hash2 {
 			log.Error("Mismatching blocks", "slot", i, "gotSlot", blk2.Block.Slot, "datadir", libcommon.Hash(hash1), "snapshot", libcommon.Hash(hash2))
-			//return nil
+			return nil
 		}
 		log.Info("Successfully checked", "slot", i)
 	}

From 575cef3e240c26cdbcdb987bf1e140bd3425abc7 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 15:27:00 +0200
Subject: [PATCH 55/63] save

---
 cmd/capcli/cli.go | 2 --
 1 file changed, 2 deletions(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index d5931ece112..a02c5740801 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -482,12 +482,10 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 			return err
 		}
 		if data == nil {
-			log.Info("Block not found", i)
 			continue
 		}
 		blk := data.Data
 		if blk == nil {
-			log.Info("Block not found", i)
 			continue
 		}
 		// first thing if the block is bellatrix update the mock block reader

From 09a939521d53506505d30fcfd66cdcd4347e8d43 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 15:47:26 +0200
Subject: [PATCH 56/63] save

---
 cmd/capcli/cli.go | 4 +++-
 1 file changed, 3 insertions(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index a02c5740801..516005bce13 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -436,6 +436,8 @@ func (c *DumpSnapshots) Run(ctx *Context) error {
 type CheckSnapshots struct {
 	chainCfg
 	outputFolder
+
+	Slot uint64 `arg:"" name:"slot" help:"slot to check"`
 }
 
 func (c *CheckSnapshots) Run(ctx *Context) error {
@@ -475,7 +477,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 
 	br := &snapshot_format.MockBlockReader{}
 	snReader := freezeblocks.NewBeaconSnapshotReader(csn, br, beaconConfig)
-	for i := uint64(0); i < to; i++ {
+	for i := uint64(c.Slot); i < to; i++ {
 		// Read the original canonical slot
 		data, err := beaconDB.GetBlock(ctx, tx, i)
 		if err != nil {

From 49a03ecfbb208e868bde481f5c24ad903b081c81 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 15:50:50 +0200
Subject: [PATCH 57/63] svae

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index 516005bce13..efdf4a45a33 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -437,7 +437,7 @@ type CheckSnapshots struct {
 	chainCfg
 	outputFolder
 
-	Slot uint64 `arg:"" name:"slot" help:"slot to check"`
+	Slot uint64 `name:"slot" help:"slot to check"`
 }
 
 func (c *CheckSnapshots) Run(ctx *Context) error {

From 5b829113abe782290a3124f069ae855e07a93144 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 16:13:40 +0200
Subject: [PATCH 58/63] save

---
 .../format/snapshot_format/blocks.go          |  6 ++++-
 cl/spectest/consensus_tests/ssz_static.go     | 25 ++++++++++++++++++-
 2 files changed, 29 insertions(+), 2 deletions(-)

diff --git a/cl/persistence/format/snapshot_format/blocks.go b/cl/persistence/format/snapshot_format/blocks.go
index f62443cbb94..cac0aa0c21e 100644
--- a/cl/persistence/format/snapshot_format/blocks.go
+++ b/cl/persistence/format/snapshot_format/blocks.go
@@ -119,6 +119,7 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	plainSSZ := []byte{}
 
 	block := cltypes.NewSignedBeaconBlock(cfg)
+	fmt.Println("BX")
 	// Metadata section is just the current hardfork of the block. TODO(give it a useful purpose)
 	v, err := readMetadataForBlock(r)
 	if err != nil {
@@ -130,7 +131,7 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	if err != nil {
 		return nil, err
 	}
-
+	fmt.Println("AX")
 	if dT1 != chunk_encoding.ChunkDataType {
 		return nil, fmt.Errorf("malformed beacon block, invalid chunk 1 type %d, expected: %d", dT1, chunk_encoding.ChunkDataType)
 	}
@@ -154,9 +155,11 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 		return nil, err
 	}
 	plainSSZ = append(plainSSZ, chunk2...)
+	fmt.Println("RX")
 	if v <= clparams.BellatrixVersion {
 		return block, block.DecodeSSZ(plainSSZ, int(v))
 	}
+	fmt.Println("FX")
 
 	// Read the 5h chunk
 	chunk3, dT5, err := chunk_encoding.ReadChunk(r)
@@ -166,6 +169,7 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	if dT5 != chunk_encoding.ChunkDataType {
 		return nil, fmt.Errorf("malformed beacon block, invalid chunk 5 type %d, expected: %d", dT5, chunk_encoding.ChunkDataType)
 	}
+	fmt.Println("PX")
 	plainSSZ = append(plainSSZ, chunk3...)
 
 	return block, block.DecodeSSZ(plainSSZ, int(v))
diff --git a/cl/spectest/consensus_tests/ssz_static.go b/cl/spectest/consensus_tests/ssz_static.go
index f5621181344..b38e71df336 100644
--- a/cl/spectest/consensus_tests/ssz_static.go
+++ b/cl/spectest/consensus_tests/ssz_static.go
@@ -1,10 +1,15 @@
 package consensus_tests
 
 import (
-	"github.com/ledgerwatch/erigon/spectest"
+	"bytes"
 	"io/fs"
 	"testing"
 
+	"github.com/ledgerwatch/erigon/spectest"
+
+	"github.com/ledgerwatch/erigon/cl/clparams"
+	"github.com/ledgerwatch/erigon/cl/cltypes"
+	"github.com/ledgerwatch/erigon/cl/persistence/format/snapshot_format"
 	"github.com/ledgerwatch/erigon/cl/phase1/core/state"
 
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
@@ -59,6 +64,24 @@ func getSSZStaticConsensusTest[T unmarshalerMarshalerHashable](ref T) spectest.H
 		haveEncoded, err := object.EncodeSSZ(nil)
 		require.NoError(t, err)
 		require.EqualValues(t, haveEncoded, encoded)
+		// Now let it do the encoding in snapshot format
+		if blk, ok := object.(*cltypes.SignedBeaconBlock); ok {
+			var b bytes.Buffer
+			require.NoError(t, snapshot_format.WriteBlockForSnapshot(blk, &b))
+			var br snapshot_format.MockBlockReader
+			if blk.Version() >= clparams.BellatrixVersion {
+				br = snapshot_format.MockBlockReader{Block: blk.Block.Body.ExecutionPayload}
+
+			}
+
+			blk2, err := snapshot_format.ReadBlockFromSnapshot(&b, &br, &clparams.MainnetBeaconConfig)
+			require.NoError(t, err)
+
+			haveRoot, err := blk2.HashSSZ()
+			require.NoError(t, err)
+			require.EqualValues(t, expectedRoot, haveRoot)
+		}
+
 		return nil
 	})
 }

From 7d3d7b1f0a2deddf0b975ecc6edc611df60d0f6f Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 16:15:33 +0200
Subject: [PATCH 59/63] save

---
 cl/persistence/format/chunk_encoding/chunks.go | 2 ++
 1 file changed, 2 insertions(+)

diff --git a/cl/persistence/format/chunk_encoding/chunks.go b/cl/persistence/format/chunk_encoding/chunks.go
index 05ba58e3c54..246fdd77c1d 100644
--- a/cl/persistence/format/chunk_encoding/chunks.go
+++ b/cl/persistence/format/chunk_encoding/chunks.go
@@ -2,6 +2,7 @@ package chunk_encoding
 
 import (
 	"encoding/binary"
+	"fmt"
 	"io"
 )
 
@@ -35,6 +36,7 @@ func ReadChunk(r io.Reader) (buf []byte, t DataType, err error) {
 	}
 	t = DataType(prefix[0])
 	prefix[0] = 0
+	fmt.Println("D")
 	buf = make([]byte, binary.BigEndian.Uint64(prefix))
 	if _, err = r.Read(buf); err != nil {
 		return

From f68c38b0d0dfb28b487f698c63f75d7637df1216 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 16:16:43 +0200
Subject: [PATCH 60/63] save

---
 cl/persistence/format/chunk_encoding/chunks.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cl/persistence/format/chunk_encoding/chunks.go b/cl/persistence/format/chunk_encoding/chunks.go
index 246fdd77c1d..005ae381ab3 100644
--- a/cl/persistence/format/chunk_encoding/chunks.go
+++ b/cl/persistence/format/chunk_encoding/chunks.go
@@ -36,7 +36,7 @@ func ReadChunk(r io.Reader) (buf []byte, t DataType, err error) {
 	}
 	t = DataType(prefix[0])
 	prefix[0] = 0
-	fmt.Println("D")
+	fmt.Println(binary.BigEndian.Uint64(prefix))
 	buf = make([]byte, binary.BigEndian.Uint64(prefix))
 	if _, err = r.Read(buf); err != nil {
 		return

From 33f6dd25ec438279ecfbc36b1781636b1677aed2 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 16:20:06 +0200
Subject: [PATCH 61/63] save

---
 cl/persistence/format/chunk_encoding/chunks.go         | 7 +++++--
 cl/persistence/format/snapshot_format/blocks_test.go   | 6 +++++-
 turbo/snapshotsync/freezeblocks/beacon_block_reader.go | 1 +
 3 files changed, 11 insertions(+), 3 deletions(-)

diff --git a/cl/persistence/format/chunk_encoding/chunks.go b/cl/persistence/format/chunk_encoding/chunks.go
index 005ae381ab3..eeefb3962ee 100644
--- a/cl/persistence/format/chunk_encoding/chunks.go
+++ b/cl/persistence/format/chunk_encoding/chunks.go
@@ -2,7 +2,6 @@ package chunk_encoding
 
 import (
 	"encoding/binary"
-	"fmt"
 	"io"
 )
 
@@ -36,7 +35,11 @@ func ReadChunk(r io.Reader) (buf []byte, t DataType, err error) {
 	}
 	t = DataType(prefix[0])
 	prefix[0] = 0
-	fmt.Println(binary.BigEndian.Uint64(prefix))
+
+	bufLen := binary.BigEndian.Uint64(prefix)
+	if bufLen == 0 {
+		return
+	}
 	buf = make([]byte, binary.BigEndian.Uint64(prefix))
 	if _, err = r.Read(buf); err != nil {
 		return
diff --git a/cl/persistence/format/snapshot_format/blocks_test.go b/cl/persistence/format/snapshot_format/blocks_test.go
index 571280b61d6..8021c3fcc38 100644
--- a/cl/persistence/format/snapshot_format/blocks_test.go
+++ b/cl/persistence/format/snapshot_format/blocks_test.go
@@ -31,7 +31,11 @@ var emptyBlock = cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
 
 // obtain the test blocks
 func getTestBlocks(t *testing.T) []*cltypes.SignedBeaconBlock {
+	var emptyBlockCapella = cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
+	emptyBlockCapella.Block.Slot = clparams.MainnetBeaconConfig.CapellaForkEpoch * 32
+
 	emptyBlock.EncodingSizeSSZ()
+	emptyBlockCapella.EncodingSizeSSZ()
 	denebBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
 	capellaBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
 	bellatrixBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
@@ -43,7 +47,7 @@ func getTestBlocks(t *testing.T) []*cltypes.SignedBeaconBlock {
 	require.NoError(t, utils.DecodeSSZSnappy(bellatrixBlock, bellatrixBlockSSZSnappy, int(clparams.BellatrixVersion)))
 	require.NoError(t, utils.DecodeSSZSnappy(altairBlock, altairBlockSSZSnappy, int(clparams.AltairVersion)))
 	require.NoError(t, utils.DecodeSSZSnappy(phase0Block, phase0BlockSSZSnappy, int(clparams.Phase0Version)))
-	return []*cltypes.SignedBeaconBlock{phase0Block, altairBlock, bellatrixBlock, capellaBlock, denebBlock, emptyBlock}
+	return []*cltypes.SignedBeaconBlock{phase0Block, altairBlock, bellatrixBlock, capellaBlock, denebBlock, emptyBlock, emptyBlockCapella}
 }
 
 func TestBlockSnapshotEncoding(t *testing.T) {
diff --git a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
index fdced61a050..78ef3fa9cc4 100644
--- a/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
+++ b/turbo/snapshotsync/freezeblocks/beacon_block_reader.go
@@ -48,6 +48,7 @@ func (r *beaconSnapshotReader) RawBlockSSZ(slot uint64) ([]byte, error) {
 	defer view.Close()
 
 	var buf []byte
+
 	seg, ok := view.BeaconBlocksSegment(slot)
 	if !ok {
 		return nil, nil

From 8f1ab65fef187697673daf9b31cce5594664a301 Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 16:21:08 +0200
Subject: [PATCH 62/63] save

---
 cl/persistence/format/snapshot_format/blocks.go | 5 -----
 1 file changed, 5 deletions(-)

diff --git a/cl/persistence/format/snapshot_format/blocks.go b/cl/persistence/format/snapshot_format/blocks.go
index cac0aa0c21e..8029ef78407 100644
--- a/cl/persistence/format/snapshot_format/blocks.go
+++ b/cl/persistence/format/snapshot_format/blocks.go
@@ -119,7 +119,6 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	plainSSZ := []byte{}
 
 	block := cltypes.NewSignedBeaconBlock(cfg)
-	fmt.Println("BX")
 	// Metadata section is just the current hardfork of the block. TODO(give it a useful purpose)
 	v, err := readMetadataForBlock(r)
 	if err != nil {
@@ -131,7 +130,6 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	if err != nil {
 		return nil, err
 	}
-	fmt.Println("AX")
 	if dT1 != chunk_encoding.ChunkDataType {
 		return nil, fmt.Errorf("malformed beacon block, invalid chunk 1 type %d, expected: %d", dT1, chunk_encoding.ChunkDataType)
 	}
@@ -155,11 +153,9 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 		return nil, err
 	}
 	plainSSZ = append(plainSSZ, chunk2...)
-	fmt.Println("RX")
 	if v <= clparams.BellatrixVersion {
 		return block, block.DecodeSSZ(plainSSZ, int(v))
 	}
-	fmt.Println("FX")
 
 	// Read the 5h chunk
 	chunk3, dT5, err := chunk_encoding.ReadChunk(r)
@@ -169,7 +165,6 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	if dT5 != chunk_encoding.ChunkDataType {
 		return nil, fmt.Errorf("malformed beacon block, invalid chunk 5 type %d, expected: %d", dT5, chunk_encoding.ChunkDataType)
 	}
-	fmt.Println("PX")
 	plainSSZ = append(plainSSZ, chunk3...)
 
 	return block, block.DecodeSSZ(plainSSZ, int(v))

From ba5ac0b6a4a11a4afe84cc5c02a27f9a3a982b5f Mon Sep 17 00:00:00 2001
From: Giulio <giulio.rebuffo@gmail.com>
Date: Tue, 24 Oct 2023 19:06:22 +0200
Subject: [PATCH 63/63] save

---
 cmd/capcli/cli.go | 2 +-
 1 file changed, 1 insertion(+), 1 deletion(-)

diff --git a/cmd/capcli/cli.go b/cmd/capcli/cli.go
index efdf4a45a33..e593e4f4de9 100644
--- a/cmd/capcli/cli.go
+++ b/cmd/capcli/cli.go
@@ -477,7 +477,7 @@ func (c *CheckSnapshots) Run(ctx *Context) error {
 
 	br := &snapshot_format.MockBlockReader{}
 	snReader := freezeblocks.NewBeaconSnapshotReader(csn, br, beaconConfig)
-	for i := uint64(c.Slot); i < to; i++ {
+	for i := c.Slot; i < to; i++ {
 		// Read the original canonical slot
 		data, err := beaconDB.GetBlock(ctx, tx, i)
 		if err != nil {