Added functional beacon snapshots reader and generator to Caplin

cl/cltypes/beacon_block.go

@@ -94,7 +94,9 @@ func NewBeaconBlock(beaconCfg *clparams.BeaconChainConfig) *BeaconBlock {
 }
 
 func NewBeaconBody(beaconCfg *clparams.BeaconChainConfig) *BeaconBody {
-	return &BeaconBody{beaconCfg: beaconCfg}
+	return &BeaconBody{
+		beaconCfg: beaconCfg,
+	}
 }
 
 // Version returns beacon block version.
@@ -172,6 +174,8 @@ func (b *BeaconBody) DecodeSSZ(buf []byte, version int) error {
 		return fmt.Errorf("[BeaconBody] err: %s", ssz.ErrLowBufferSize)
 	}
 
+	b.ExecutionPayload = NewEth1Block(b.Version, b.beaconCfg)
+
 	err := ssz2.UnmarshalSSZ(buf, version, b.getSchema(false)...)
 	return err
 }

cl/persistence/format/chunk_encoding/chunks.go

@@ -35,6 +35,11 @@ func ReadChunk(r io.Reader) (buf []byte, t DataType, err error) {
 	}
 	t = DataType(prefix[0])
 	prefix[0] = 0
+
+	bufLen := binary.BigEndian.Uint64(prefix)
+	if bufLen == 0 {
+		return
+	}
 	buf = make([]byte, binary.BigEndian.Uint64(prefix))
 	if _, err = r.Read(buf); err != nil {
 		return

cl/persistence/format/snapshot_format/blocks.go

@@ -86,25 +86,10 @@ func WriteBlockForSnapshot(block *cltypes.SignedBeaconBlock, w io.Writer) error
 	// count in body for phase0 fields
 	currentChunkLength += uint64(body.ProposerSlashings.EncodingSizeSSZ())
 	currentChunkLength += uint64(body.AttesterSlashings.EncodingSizeSSZ())
-
-	// Write the chunk and chunk attestations
-	if err := chunk_encoding.WriteChunk(w, encoded[:currentChunkLength], chunk_encoding.ChunkDataType); err != nil {
-		return err
-	}
-	encoded = encoded[currentChunkLength:]
-	snappyWriter := snappy.NewBufferedWriter(w)
-	if err := chunk_encoding.WriteChunk(snappyWriter, encoded[:uint64(body.Attestations.EncodingSizeSSZ())], chunk_encoding.ChunkDataType); err != nil {
-		return err
-	}
-	if err := snappyWriter.Close(); err != nil {
-		return err
-	}
-	encoded = encoded[body.Attestations.EncodingSizeSSZ():]
-	currentChunkLength = 0
-
+	currentChunkLength += uint64(body.Attestations.EncodingSizeSSZ())
 	currentChunkLength += uint64(body.Deposits.EncodingSizeSSZ())
 	currentChunkLength += uint64(body.VoluntaryExits.EncodingSizeSSZ())
-
+	// Write the chunk and chunk attestations
 	if err := chunk_encoding.WriteChunk(w, encoded[:currentChunkLength], chunk_encoding.ChunkDataType); err != nil {
 		return err
 	}
@@ -140,6 +125,60 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 		return nil, err
 	}
 
+	// Read the first chunk
+	chunk1, dT1, err := chunk_encoding.ReadChunk(r)
+	if err != nil {
+		return nil, err
+	}
+	if dT1 != chunk_encoding.ChunkDataType {
+		return nil, fmt.Errorf("malformed beacon block, invalid chunk 1 type %d, expected: %d", dT1, chunk_encoding.ChunkDataType)
+	}
+	plainSSZ = append(plainSSZ, chunk1...)
+
+	if v <= clparams.AltairVersion {
+		return block, block.DecodeSSZ(plainSSZ, int(v))
+	}
+	// Read the block pointer and retrieve chunk4 from the execution reader
+	blockPointer, err := readExecutionBlockPtr(r)
+	if err != nil {
+		return nil, err
+	}
+	executionBlock, err := executionReader.BlockByNumber(blockPointer)
+	if err != nil {
+		return nil, err
+	}
+	// Read the 4th chunk
+	chunk2, err := executionBlock.EncodeSSZ(nil)
+	if err != nil {
+		return nil, err
+	}
+	plainSSZ = append(plainSSZ, chunk2...)
+	if v <= clparams.BellatrixVersion {
+		return block, block.DecodeSSZ(plainSSZ, int(v))
+	}
+
+	// Read the 5h chunk
+	chunk3, dT5, err := chunk_encoding.ReadChunk(r)
+	if err != nil {
+		return nil, err
+	}
+	if dT5 != chunk_encoding.ChunkDataType {
+		return nil, fmt.Errorf("malformed beacon block, invalid chunk 5 type %d, expected: %d", dT5, chunk_encoding.ChunkDataType)
+	}
+	plainSSZ = append(plainSSZ, chunk3...)
+
+	return block, block.DecodeSSZ(plainSSZ, int(v))
+}
+
+func ReadRawBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNumber, cfg *clparams.BeaconChainConfig) ([]byte, error) {
+	plainSSZ := []byte{}
+
+	// Metadata section is just the current hardfork of the block. TODO(give it a useful purpose)
+	v, err := readMetadataForBlock(r)
+	if err != nil {
+		return nil, err
+	}
+
 	// Read the first chunk
 	chunk1, dT1, err := chunk_encoding.ReadChunk(r)
 	if err != nil {
@@ -168,7 +207,7 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	}
 	plainSSZ = append(plainSSZ, chunk3...)
 	if v <= clparams.AltairVersion {
-		return block, block.DecodeSSZ(plainSSZ, int(v))
+		return plainSSZ, nil
 	}
 	// Read the block pointer and retrieve chunk4 from the execution reader
 	blockPointer, err := readExecutionBlockPtr(r)
@@ -186,7 +225,7 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	}
 	plainSSZ = append(plainSSZ, chunk4...)
 	if v <= clparams.BellatrixVersion {
-		return block, block.DecodeSSZ(plainSSZ, int(v))
+		return plainSSZ, nil
 	}
 
 	// Read the 5h chunk
@@ -199,5 +238,5 @@ func ReadBlockFromSnapshot(r io.Reader, executionReader ExecutionBlockReaderByNu
 	}
 	plainSSZ = append(plainSSZ, chunk5...)
 
-	return block, block.DecodeSSZ(plainSSZ, int(v))
+	return plainSSZ, nil
 }

cl/persistence/format/snapshot_format/blocks_test.go

@@ -27,8 +27,15 @@ var capellaBlockSSZSnappy []byte
 //go:embed test_data/deneb.ssz_snappy
 var denebBlockSSZSnappy []byte
 
+var emptyBlock = cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
+
 // obtain the test blocks
 func getTestBlocks(t *testing.T) []*cltypes.SignedBeaconBlock {
+	var emptyBlockCapella = cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
+	emptyBlockCapella.Block.Slot = clparams.MainnetBeaconConfig.CapellaForkEpoch * 32
+
+	emptyBlock.EncodingSizeSSZ()
+	emptyBlockCapella.EncodingSizeSSZ()
 	denebBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
 	capellaBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
 	bellatrixBlock := cltypes.NewSignedBeaconBlock(&clparams.MainnetBeaconConfig)
@@ -40,22 +47,14 @@ func getTestBlocks(t *testing.T) []*cltypes.SignedBeaconBlock {
 	require.NoError(t, utils.DecodeSSZSnappy(bellatrixBlock, bellatrixBlockSSZSnappy, int(clparams.BellatrixVersion)))
 	require.NoError(t, utils.DecodeSSZSnappy(altairBlock, altairBlockSSZSnappy, int(clparams.AltairVersion)))
 	require.NoError(t, utils.DecodeSSZSnappy(phase0Block, phase0BlockSSZSnappy, int(clparams.Phase0Version)))
-	return []*cltypes.SignedBeaconBlock{phase0Block, altairBlock, bellatrixBlock, capellaBlock, denebBlock}
-}
-
-type TestBlockReader struct {
-	Block *cltypes.Eth1Block
-}
-
-func (t *TestBlockReader) BlockByNumber(number uint64) (*cltypes.Eth1Block, error) {
-	return t.Block, nil
+	return []*cltypes.SignedBeaconBlock{phase0Block, altairBlock, bellatrixBlock, capellaBlock, denebBlock, emptyBlock, emptyBlockCapella}
 }
 
 func TestBlockSnapshotEncoding(t *testing.T) {
 	for _, blk := range getTestBlocks(t) {
-		var br TestBlockReader
+		var br snapshot_format.MockBlockReader
 		if blk.Version() >= clparams.BellatrixVersion {
-			br = TestBlockReader{Block: blk.Block.Body.ExecutionPayload}
+			br = snapshot_format.MockBlockReader{Block: blk.Block.Body.ExecutionPayload}
 		}
 		var b bytes.Buffer
 		require.NoError(t, snapshot_format.WriteBlockForSnapshot(blk, &b))

cl/persistence/format/snapshot_format/test_util.go

@@ -0,0 +1,11 @@
+package snapshot_format
+
+import "github.com/ledgerwatch/erigon/cl/cltypes"
+
+type MockBlockReader struct {
+	Block *cltypes.Eth1Block
+}
+
+func (t *MockBlockReader) BlockByNumber(number uint64) (*cltypes.Eth1Block, error) {
+	return t.Block, nil
+}

cl/spectest/consensus_tests/ssz_static.go

@@ -1,10 +1,15 @@
 package consensus_tests
 
 import (
-	"github.com/ledgerwatch/erigon/spectest"
+	"bytes"
 	"io/fs"
 	"testing"
 
+	"github.com/ledgerwatch/erigon/spectest"
+
+	"github.com/ledgerwatch/erigon/cl/clparams"
+	"github.com/ledgerwatch/erigon/cl/cltypes"
+	"github.com/ledgerwatch/erigon/cl/persistence/format/snapshot_format"
 	"github.com/ledgerwatch/erigon/cl/phase1/core/state"
 
 	libcommon "github.com/ledgerwatch/erigon-lib/common"
@@ -59,6 +64,24 @@ func getSSZStaticConsensusTest[T unmarshalerMarshalerHashable](ref T) spectest.H
 		haveEncoded, err := object.EncodeSSZ(nil)
 		require.NoError(t, err)
 		require.EqualValues(t, haveEncoded, encoded)
+		// Now let it do the encoding in snapshot format
+		if blk, ok := object.(*cltypes.SignedBeaconBlock); ok {
+			var b bytes.Buffer
+			require.NoError(t, snapshot_format.WriteBlockForSnapshot(blk, &b))
+			var br snapshot_format.MockBlockReader
+			if blk.Version() >= clparams.BellatrixVersion {
+				br = snapshot_format.MockBlockReader{Block: blk.Block.Body.ExecutionPayload}
+
+			}
+
+			blk2, err := snapshot_format.ReadBlockFromSnapshot(&b, &br, &clparams.MainnetBeaconConfig)
+			require.NoError(t, err)
+
+			haveRoot, err := blk2.HashSSZ()
+			require.NoError(t, err)
+			require.EqualValues(t, expectedRoot, haveRoot)
+		}
+
 		return nil
 	})
 }