Skip to content

Commit

Permalink
rlpgen: step 1 (#11112)
Browse files Browse the repository at this point in the history
- no behavior changes
- move encbuffer and encreader to `encbuffer.go`
- copy-paste rlpgen package
  • Loading branch information
AskAlexSharov authored Jul 15, 2024
1 parent ad150b8 commit 459b8b5
Show file tree
Hide file tree
Showing 26 changed files with 2,421 additions and 248 deletions.
4 changes: 4 additions & 0 deletions eth/stagedsync/stage_bodies_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,7 @@ import (
"github.com/ledgerwatch/erigon-lib/config3"
"github.com/ledgerwatch/erigon-lib/kv"
"github.com/ledgerwatch/erigon-lib/kv/rawdbv3"
"github.com/ledgerwatch/erigon-lib/log/v3"
"github.com/stretchr/testify/require"

"github.com/ledgerwatch/erigon/core/rawdb"
Expand All @@ -50,6 +51,9 @@ func testingHeaderBody(t *testing.T) (h *types.Header, b *types.RawBody) {
}

func TestBodiesCanonical(t *testing.T) {
defer log.Root().SetHandler(log.Root().GetHandler())
log.Root().SetHandler(log.LvlFilterHandler(log.LvlError, log.StderrHandler))

m := mock.Mock(t)
tx, err := m.DB.BeginRw(m.Ctx)
require := require.New(t)
Expand Down
16 changes: 8 additions & 8 deletions eth/stagedsync/stage_bor_heimdall_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -47,7 +47,7 @@ func TestBorHeimdallForwardPersistsSpans(t *testing.T) {
testHarness := stagedsynctest.InitHarness(ctx, t, stagedsynctest.HarnessCfg{
ChainConfig: stagedsynctest.BorDevnetChainConfigWithNoBlockSealDelays(),
GenerateChainNumBlocks: numBlocks,
LogLvl: log.LvlInfo,
LogLvl: log.LvlError,
})
// pretend-update previous stage progress
testHarness.SaveStageProgress(ctx, t, stages.Headers, uint64(numBlocks))
Expand Down Expand Up @@ -77,7 +77,7 @@ func TestBorHeimdallForwardFetchesFirstSpanDuringSecondSprintStart(t *testing.T)
testHarness := stagedsynctest.InitHarness(ctx, t, stagedsynctest.HarnessCfg{
ChainConfig: stagedsynctest.BorDevnetChainConfigWithNoBlockSealDelays(),
GenerateChainNumBlocks: numBlocks,
LogLvl: log.LvlInfo,
LogLvl: log.LvlError,
})
// pretend-update previous stage progress
testHarness.SaveStageProgress(ctx, t, stages.Headers, uint64(numBlocks))
Expand Down Expand Up @@ -112,7 +112,7 @@ func TestBorHeimdallForwardFetchesFirstSpanAfterSecondSprintStart(t *testing.T)
testHarness := stagedsynctest.InitHarness(ctx, t, stagedsynctest.HarnessCfg{
ChainConfig: stagedsynctest.BorDevnetChainConfigWithNoBlockSealDelays(),
GenerateChainNumBlocks: numBlocks,
LogLvl: log.LvlInfo,
LogLvl: log.LvlError,
})
// pretend-update previous stage progress
testHarness.SaveStageProgress(ctx, t, stages.Headers, uint64(numBlocks))
Expand Down Expand Up @@ -143,7 +143,7 @@ func TestBorHeimdallForwardFetchesNextSpanDuringLastSprintOfCurrentSpan(t *testi
testHarness := stagedsynctest.InitHarness(ctx, t, stagedsynctest.HarnessCfg{
ChainConfig: stagedsynctest.BorDevnetChainConfigWithNoBlockSealDelays(),
GenerateChainNumBlocks: numBlocks,
LogLvl: log.LvlInfo,
LogLvl: log.LvlError,
})
// pretend-update previous stage progress
testHarness.SaveStageProgress(ctx, t, stages.Headers, uint64(numBlocks))
Expand Down Expand Up @@ -174,7 +174,7 @@ func TestBorHeimdallForwardPersistsStateSyncEvents(t *testing.T) {
testHarness := stagedsynctest.InitHarness(ctx, t, stagedsynctest.HarnessCfg{
ChainConfig: stagedsynctest.BorDevnetChainConfigWithNoBlockSealDelays(),
GenerateChainNumBlocks: numBlocks,
LogLvl: log.LvlInfo,
LogLvl: log.LvlError,
})
// pretend-update previous stage progress
testHarness.SaveStageProgress(ctx, t, stages.Headers, uint64(numBlocks))
Expand Down Expand Up @@ -211,7 +211,7 @@ func TestBorHeimdallForwardErrHeaderValidatorsLengthMismatch(t *testing.T) {
testHarness := stagedsynctest.InitHarness(ctx, t, stagedsynctest.HarnessCfg{
ChainConfig: stagedsynctest.BorDevnetChainConfigWithNoBlockSealDelays(),
GenerateChainNumBlocks: numBlocks,
LogLvl: log.LvlInfo,
LogLvl: log.LvlError,
HeimdallProducersOverride: map[uint64][]valset.Validator{
1: {
*valset.NewValidator(crypto.PubkeyToAddress(validatorKey1.PublicKey), 1),
Expand All @@ -236,7 +236,7 @@ func TestBorHeimdallForwardErrHeaderValidatorsBytesMismatch(t *testing.T) {
testHarness := stagedsynctest.InitHarness(ctx, t, stagedsynctest.HarnessCfg{
ChainConfig: stagedsynctest.BorDevnetChainConfigWithNoBlockSealDelays(),
GenerateChainNumBlocks: numBlocks,
LogLvl: log.LvlInfo,
LogLvl: log.LvlError,
HeimdallProducersOverride: map[uint64][]valset.Validator{
1: {
*valset.NewValidator(crypto.PubkeyToAddress(validatorKey1.PublicKey), 1),
Expand All @@ -260,7 +260,7 @@ func TestBorHeimdallForwardDetectsUnauthorizedSignerError(t *testing.T) {
testHarness := stagedsynctest.InitHarness(ctx, t, stagedsynctest.HarnessCfg{
ChainConfig: chainConfig,
GenerateChainNumBlocks: numBlocks,
LogLvl: log.LvlInfo,
LogLvl: log.LvlError,
})

// prepare invalid header and insert it in the db
Expand Down
4 changes: 2 additions & 2 deletions eth/stagedsync/stage_mining_bor_heimdall_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -37,7 +37,7 @@ func TestMiningBorHeimdallForwardPersistsSpans(t *testing.T) {
testHarness := stagedsynctest.InitHarness(ctx, t, stagedsynctest.HarnessCfg{
ChainConfig: stagedsynctest.BorDevnetChainConfigWithNoBlockSealDelays(),
GenerateChainNumBlocks: numBlocks,
LogLvl: log.LvlInfo,
LogLvl: log.LvlError,
})
// pretend-update previous stage progress
testHarness.SetMiningBlockEmptyHeader(ctx, t, uint64(numBlocks))
Expand Down Expand Up @@ -69,7 +69,7 @@ func TestMiningBorHeimdallForwardPersistsStateSyncEvents(t *testing.T) {
testHarness := stagedsynctest.InitHarness(ctx, t, stagedsynctest.HarnessCfg{
ChainConfig: stagedsynctest.BorDevnetChainConfigWithNoBlockSealDelays(),
GenerateChainNumBlocks: numBlocks,
LogLvl: log.LvlInfo,
LogLvl: log.LvlError,
})
// pretend-update previous stage progress
testHarness.SetMiningBlockEmptyHeader(ctx, t, uint64(numBlocks))
Expand Down
2 changes: 1 addition & 1 deletion go.mod
Original file line number Diff line number Diff line change
Expand Up @@ -277,7 +277,7 @@ require (
go.uber.org/multierr v1.11.0 // indirect
golang.org/x/mod v0.19.0 // indirect
golang.org/x/text v0.16.0 // indirect
golang.org/x/tools v0.23.0 // indirect
golang.org/x/tools v0.23.0
gopkg.in/cenkalti/backoff.v1 v1.1.0 // indirect
gopkg.in/tomb.v1 v1.0.0-20141024135613-dd632973f1e7 // indirect
gotest.tools/v3 v3.5.1 // indirect
Expand Down
227 changes: 227 additions & 0 deletions rlp/encbuffer.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,227 @@
// Copyright 2014 The go-ethereum Authors
// (original work)
// Copyright 2024 The Erigon Authors
// (modifications)
// This file is part of Erigon.
//
// Erigon is free software: you can redistribute it and/or modify
// it under the terms of the GNU Lesser General Public License as published by
// the Free Software Foundation, either version 3 of the License, or
// (at your option) any later version.
//
// Erigon is distributed in the hope that it will be useful,
// but WITHOUT ANY WARRANTY; without even the implied warranty of
// MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the
// GNU Lesser General Public License for more details.
//
// You should have received a copy of the GNU Lesser General Public License
// along with Erigon. If not, see <http://www.gnu.org/licenses/>.

package rlp

import (
"io"
"reflect"
"sync"
)

type encBuffer struct {
str []byte // string data, contains everything except list headers
lheads []listhead // all list headers
lhsize int // sum of sizes of all encoded list headers
sizebuf [9]byte // auxiliary buffer for uint encoding
bufvalue reflect.Value // used in writeByteArrayCopy
}

// encbufs are pooled.
var encBufferPool = sync.Pool{
New: func() interface{} {
var bytes []byte
return &encBuffer{bufvalue: reflect.ValueOf(&bytes).Elem()}
},
}

func (w *encBuffer) reset() {
w.lhsize = 0
w.str = w.str[:0]
w.lheads = w.lheads[:0]
}

// encBuffer implements io.Writer so it can be passed it into EncodeRLP.
func (w *encBuffer) Write(b []byte) (int, error) {
w.str = append(w.str, b...)
return len(b), nil
}

func (w *encBuffer) encode(val interface{}) error {
rval := reflect.ValueOf(val)
writer, err := cachedWriter(rval.Type())
if err != nil {
return err
}
return writer(rval, w)
}

func (w *encBuffer) encodeStringHeader(size int) {
if size < 56 {
w.str = append(w.str, EmptyStringCode+byte(size))
} else {
sizesize := putint(w.sizebuf[1:], uint64(size))
w.sizebuf[0] = 0xB7 + byte(sizesize)
w.str = append(w.str, w.sizebuf[:sizesize+1]...)
}
}

func (w *encBuffer) encodeString(b []byte) {
if len(b) == 1 && b[0] <= 0x7F {
// fits single byte, no string header
w.str = append(w.str, b[0])
} else {
w.encodeStringHeader(len(b))
w.str = append(w.str, b...)
}
}

func (w *encBuffer) encodeUint(i uint64) {
if i == 0 {
w.str = append(w.str, 0x80)
} else if i < 128 {
// fits single byte
w.str = append(w.str, byte(i))
} else {
s := putint(w.sizebuf[1:], i)
w.sizebuf[0] = 0x80 + byte(s)
w.str = append(w.str, w.sizebuf[:s+1]...)
}
}

// list adds a new list header to the header stack. It returns the index
// of the header. The caller must call listEnd with this index after encoding
// the content of the list.
func (w *encBuffer) list() int {
w.lheads = append(w.lheads, listhead{offset: len(w.str), size: w.lhsize})
return len(w.lheads) - 1
}

func (w *encBuffer) listEnd(index int) {
lh := &w.lheads[index]
lh.size = w.size() - lh.offset - lh.size
if lh.size < 56 {
w.lhsize++ // length encoded into kind tag
} else {
w.lhsize += 1 + intsize(uint64(lh.size))
}
}

func (w *encBuffer) size() int {
return len(w.str) + w.lhsize
}

func (w *encBuffer) toBytes() []byte {
out := make([]byte, w.size())
strpos := 0
pos := 0
for _, head := range w.lheads {
// write string data before header
n := copy(out[pos:], w.str[strpos:head.offset])
pos += n
strpos += n
// write the header
enc := head.encode(out[pos:])
pos += len(enc)
}
// copy string data after the last list header
copy(out[pos:], w.str[strpos:])
return out
}

func (w *encBuffer) toWriter(out io.Writer) (err error) {
strpos := 0
for _, head := range w.lheads {
// write string data before header
if head.offset-strpos > 0 {
n, nErr := out.Write(w.str[strpos:head.offset])
strpos += n
if nErr != nil {
return nErr
}
}
// write the header
enc := head.encode(w.sizebuf[:])
if _, wErr := out.Write(enc); wErr != nil {
return wErr
}
}
if strpos < len(w.str) {
// write string data after the last list header
_, err = out.Write(w.str[strpos:])
}
return err
}

// encReader is the io.Reader returned by EncodeToReader.
// It releases its encbuf at EOF.
type encReader struct {
buf *encBuffer // the buffer we're reading from. this is nil when we're at EOF.
lhpos int // index of list header that we're reading
strpos int // current position in string buffer
piece []byte // next piece to be read
}

func (r *encReader) Read(b []byte) (n int, err error) {
for {
if r.piece = r.next(); r.piece == nil {
// Put the encode buffer back into the pool at EOF when it
// is first encountered. Subsequent calls still return EOF
// as the error but the buffer is no longer valid.
if r.buf != nil {
encBufferPool.Put(r.buf)
r.buf = nil
}
return n, io.EOF
}
nn := copy(b[n:], r.piece)
n += nn
if nn < len(r.piece) {
// piece didn't fit, see you next time.
r.piece = r.piece[nn:]
return n, nil
}
r.piece = nil
}
}

// next returns the next piece of data to be read.
// it returns nil at EOF.
func (r *encReader) next() []byte {
switch {
case r.buf == nil:
return nil

case r.piece != nil:
// There is still data available for reading.
return r.piece

case r.lhpos < len(r.buf.lheads):
// We're before the last list header.
head := r.buf.lheads[r.lhpos]
sizebefore := head.offset - r.strpos
if sizebefore > 0 {
// String data before header.
p := r.buf.str[r.strpos:head.offset]
r.strpos += sizebefore
return p
}
r.lhpos++
return head.encode(r.buf.sizebuf[:])

case r.strpos < len(r.buf.str):
// String data at the end, after all list headers.
p := r.buf.str[r.strpos:]
r.strpos = len(r.buf.str)
return p

default:
return nil
}
}
Loading

0 comments on commit 459b8b5

Please sign in to comment.