Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Use uint256.Int rather than common.Hash for storage values to reduce memory allocation in opSload & opSstore #575

Merged
merged 11 commits into from
May 25, 2020
Merged

Use uint256.Int rather than common.Hash for storage values to reduce memory allocation in opSload & opSstore #575

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
11 commits
Select commit Hold shift + click to select a range
bb35756
Produce less garbage in GetState
yperbasis May 24, 2020
717ed62
Still playing with mem allocation in GetCommittedState
yperbasis May 24, 2020
843b908
Pass key by pointer in GetState as well
yperbasis May 24, 2020
0f157eb
linter
yperbasis May 24, 2020
32cc8de
Avoid a memory allocation in opSload
yperbasis May 24, 2020
5daf25e
Merge branch 'master' into gc-5
yperbasis May 24, 2020
8e8c0e8
Use uint256.Int rather than common.Hash for storage values to reduce …
yperbasis May 25, 2020
0d32832
Merge branch 'master' into gc-6
yperbasis May 25, 2020
ea9ab1e
linter
yperbasis May 25, 2020
2361eda
linters
yperbasis May 25, 2020
2862148
small clean up
yperbasis May 25, 2020
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 4 additions & 2 deletions accounts/abi/bind/backends/simulated.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,8 @@ import (
"sync"
"time"

"github.com/holiman/uint256"

ethereum "github.com/ledgerwatch/turbo-geth"
"github.com/ledgerwatch/turbo-geth/accounts/abi"
"github.com/ledgerwatch/turbo-geth/accounts/abi/bind"
Expand Down Expand Up @@ -235,9 +237,9 @@ func (b *SimulatedBackend) StorageAt(ctx context.Context, contract common.Addres
if err != nil {
return nil, err
}
var val common.Hash
var val uint256.Int
statedb.GetState(contract, &key, &val)
return val[:], nil
return val.Bytes(), nil
}

// TransactionReceipt returns the receipt of a transaction.
Expand Down
45 changes: 23 additions & 22 deletions core/blockchain_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -27,6 +27,7 @@ import (
"testing"
"time"

"github.com/holiman/uint256"
"github.com/stretchr/testify/assert"

"github.com/ledgerwatch/turbo-geth/common"
Expand Down Expand Up @@ -2763,26 +2764,26 @@ func TestDeleteRecreateSlots(t *testing.T) {

// If all is correct, then slot 1 and 2 are zero
key1 := common.HexToHash("01")
var got common.Hash
var got uint256.Int
statedb.GetState(aa, &key1, &got)
if exp := (common.Hash{}); got != exp {
t.Errorf("got %x exp %x", got, exp)
if !got.IsZero() {
t.Errorf("got %x exp %x", got, 0)
}
key2 := common.HexToHash("02")
statedb.GetState(aa, &key2, &got)
if exp := (common.Hash{}); got != exp {
t.Errorf("got %x exp %x", got, exp)
if !got.IsZero() {
t.Errorf("got %x exp %x", got, 0)
}
// Also, 3 and 4 should be set
key3 := common.HexToHash("03")
statedb.GetState(aa, &key3, &got)
if exp := common.HexToHash("03"); got != exp {
t.Fatalf("got %x exp %x", got, exp)
if got.Uint64() != 3 {
t.Fatalf("got %x exp %x", got, 3)
}
key4 := common.HexToHash("04")
statedb.GetState(aa, &key4, &got)
if exp := common.HexToHash("04"); got != exp {
t.Fatalf("got %x exp %x", got, exp)
if got.Uint64() != 4 {
t.Fatalf("got %x exp %x", got, 4)
}
}

Expand Down Expand Up @@ -2860,15 +2861,15 @@ func TestDeleteRecreateAccount(t *testing.T) {

// If all is correct, then both slots are zero
key1 := common.HexToHash("01")
var got common.Hash
var got uint256.Int
statedb.GetState(aa, &key1, &got)
if exp := (common.Hash{}); got != exp {
t.Errorf("got %x exp %x", got, exp)
if !got.IsZero() {
t.Errorf("got %x exp %x", got, 0)
}
key2 := common.HexToHash("02")
statedb.GetState(aa, &key2, &got)
if exp := (common.Hash{}); got != exp {
t.Errorf("got %x exp %x", got, exp)
if !got.IsZero() {
t.Errorf("got %x exp %x", got, 0)
}
}

Expand Down Expand Up @@ -3053,15 +3054,15 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) {
statedb, _, _ := chain.State()
// If all is correct, then slot 1 and 2 are zero
key1 := common.HexToHash("01")
var got common.Hash
var got uint256.Int
statedb.GetState(aa, &key1, &got)
if exp := (common.Hash{}); got != exp {
t.Errorf("block %d, got %x exp %x", blockNum, got, exp)
if !got.IsZero() {
t.Errorf("block %d, got %x exp %x", blockNum, got, 0)
}
key2 := common.HexToHash("02")
statedb.GetState(aa, &key2, &got)
if exp := (common.Hash{}); got != exp {
t.Errorf("block %d, got %x exp %x", blockNum, got, exp)
if !got.IsZero() {
t.Errorf("block %d, got %x exp %x", blockNum, got, 0)
}
exp := expectations[i]
if exp.exist {
Expand All @@ -3070,10 +3071,10 @@ func TestDeleteRecreateSlotsAcrossManyBlocks(t *testing.T) {
}
for slot, val := range exp.values {
key := asHash(slot)
var gotValue common.Hash
var gotValue uint256.Int
statedb.GetState(aa, &key, &gotValue)
if expValue := asHash(val); gotValue != expValue {
t.Fatalf("block %d, slot %d, got %x exp %x", blockNum, slot, gotValue, expValue)
if gotValue.Uint64() != uint64(val) {
t.Fatalf("block %d, slot %d, got %x exp %x", blockNum, slot, gotValue, val)
}
}
} else {
Expand Down
9 changes: 6 additions & 3 deletions core/genesis.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,14 +18,15 @@ package core

import (
"bytes"
"context"
"encoding/hex"
"encoding/json"
"errors"
"fmt"
"math/big"
"strings"

"context"
"github.com/holiman/uint256"

"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/common/hexutil"
Expand All @@ -41,7 +42,7 @@ import (
)

var UsePlainStateExecution = false // FIXME: when we can move the hashed state forward.
// ^--- will be overriden e when parsing flags anyway
// ^--- will be overridden when parsing flags anyway

//go:generate gencodec -type Genesis -field-override genesisSpecMarshaling -out gen_genesis.go
//go:generate gencodec -type GenesisAccount -field-override genesisAccountMarshaling -out gen_genesis_account.go
Expand Down Expand Up @@ -253,7 +254,9 @@ func (g *Genesis) ToBlock(db ethdb.Database, history bool) (*types.Block, *state
statedb.SetCode(addr, account.Code)
statedb.SetNonce(addr, account.Nonce)
for key, value := range account.Storage {
statedb.SetState(addr, key, value)
key := key
val := uint256.NewInt().SetBytes(value.Bytes())
statedb.SetState(addr, &key, *val)
}

if len(account.Code) > 0 || len(account.Storage) > 0 {
Expand Down
10 changes: 4 additions & 6 deletions core/state/change_set_writer.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -4,6 +4,8 @@ import (
"context"
"fmt"

"github.com/holiman/uint256"

"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/common/changeset"
"github.com/ledgerwatch/turbo-geth/common/dbutils"
Expand Down Expand Up @@ -139,7 +141,7 @@ func (w *ChangeSetWriter) DeleteAccount(ctx context.Context, address common.Addr
return nil
}

func (w *ChangeSetWriter) WriteAccountStorage(ctx context.Context, address common.Address, incarnation uint64, key, original, value *common.Hash) error {
func (w *ChangeSetWriter) WriteAccountStorage(ctx context.Context, address common.Address, incarnation uint64, key *common.Hash, original, value *uint256.Int) error {
if *original == *value {
return nil
}
Expand All @@ -149,11 +151,7 @@ func (w *ChangeSetWriter) WriteAccountStorage(ctx context.Context, address commo
return err
}

o := cleanUpTrailingZeroes(original[:])
originalValue := make([]byte, len(o))
copy(originalValue, o)

w.storageChanges[string(compositeKey)] = originalValue
w.storageChanges[string(compositeKey)] = original.Bytes()
w.storageChanged[address] = true

return nil
Expand Down
14 changes: 8 additions & 6 deletions core/state/database.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -28,6 +28,8 @@ import (
"sync"
"sync/atomic"

"github.com/holiman/uint256"

"github.com/ledgerwatch/turbo-geth/common"
"github.com/ledgerwatch/turbo-geth/common/dbutils"
"github.com/ledgerwatch/turbo-geth/core/rawdb"
Expand Down Expand Up @@ -61,7 +63,7 @@ type StateWriter interface {
UpdateAccountData(ctx context.Context, address common.Address, original, account *accounts.Account) error
UpdateAccountCode(address common.Address, incarnation uint64, codeHash common.Hash, code []byte) error
DeleteAccount(ctx context.Context, address common.Address, original *accounts.Account) error
WriteAccountStorage(ctx context.Context, address common.Address, incarnation uint64, key, original, value *common.Hash) error
WriteAccountStorage(ctx context.Context, address common.Address, incarnation uint64, key *common.Hash, original, value *uint256.Int) error
CreateContract(address common.Address) error
}

Expand Down Expand Up @@ -89,15 +91,15 @@ func (nw *NoopWriter) UpdateAccountCode(address common.Address, incarnation uint
return nil
}

func (nw *NoopWriter) WriteAccountStorage(_ context.Context, address common.Address, incarnation uint64, key, original, value *common.Hash) error {
func (nw *NoopWriter) WriteAccountStorage(_ context.Context, address common.Address, incarnation uint64, key *common.Hash, original, value *uint256.Int) error {
return nil
}

func (nw *NoopWriter) CreateContract(address common.Address) error {
return nil
}

// Structure holding updates, deletes, and reads registered within one change period
// Buffer is a structure holding updates, deletes, and reads registered within one change period
// A change period can be transaction within a block, or a block within group of blocks
type Buffer struct {
codeReads map[common.Hash]common.Hash
Expand Down Expand Up @@ -1350,13 +1352,13 @@ func (tsw *TrieStateWriter) UpdateAccountCode(address common.Address, incarnatio
return nil
}

func (tsw *TrieStateWriter) WriteAccountStorage(_ context.Context, address common.Address, incarnation uint64, key, original, value *common.Hash) error {
func (tsw *TrieStateWriter) WriteAccountStorage(_ context.Context, address common.Address, incarnation uint64, key *common.Hash, original, value *uint256.Int) error {
addrHash, err := tsw.tds.pw.HashAddress(address, false /*save*/)
if err != nil {
return err
}

v := bytes.TrimLeft(value[:], "\x00")
v := value.Bytes()
m, ok := tsw.tds.currentBuffer.storageUpdates[addrHash]
if !ok {
m = make(map[common.Hash][]byte)
Expand All @@ -1373,7 +1375,7 @@ func (tsw *TrieStateWriter) WriteAccountStorage(_ context.Context, address commo
}
m1[seckey] = struct{}{}
if len(v) > 0 {
m[seckey] = common.CopyBytes(v)
m[seckey] = v
} else {
m[seckey] = nil
}
Expand Down
40 changes: 22 additions & 18 deletions core/state/database_test.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -24,6 +24,7 @@ import (
"testing"

"github.com/davecgh/go-spew/spew"
"github.com/holiman/uint256"
"github.com/stretchr/testify/assert"

"github.com/ledgerwatch/turbo-geth/accounts/abi/bind"
Expand Down Expand Up @@ -170,9 +171,9 @@ func TestCreate2Revive(t *testing.T) {
}
// We expect number 0x42 in the position [2], because it is the block number 2
key2 := common.BigToHash(big.NewInt(2))
var check2 common.Hash
var check2 uint256.Int
st.GetState(create2address, &key2, &check2)
if check2 != common.HexToHash("0x42") {
if check2.Uint64() != 0x42 {
t.Errorf("expected 0x42 in position 2, got: %x", check2)
}

Expand Down Expand Up @@ -205,14 +206,14 @@ func TestCreate2Revive(t *testing.T) {
}
// We expect number 0x42 in the position [4], because it is the block number 4
key4 := common.BigToHash(big.NewInt(4))
var check4 common.Hash
var check4 uint256.Int
st.GetState(create2address, &key4, &check4)
if check4 != common.HexToHash("0x42") {
if check4.Uint64() != 0x42 {
t.Errorf("expected 0x42 in position 4, got: %x", check4)
}
// We expect number 0x0 in the position [2], because it is the block number 4
st.GetState(create2address, &key2, &check2)
if check2 != common.HexToHash("0x0") {
if !check2.IsZero() {
t.Errorf("expected 0x0 in position 2, got: %x", check2)
}
}
Expand Down Expand Up @@ -322,7 +323,8 @@ func TestReorgOverSelfDestruct(t *testing.T) {
}

// Remember value of field "x" (storage item 0) after the first block, to check after rewinding
var key0, correctValueX common.Hash
var key0 common.Hash
var correctValueX uint256.Int
st.GetState(contractAddress, &key0, &correctValueX)

// BLOCKS 2 + 3
Expand Down Expand Up @@ -351,7 +353,7 @@ func TestReorgOverSelfDestruct(t *testing.T) {
t.Fatal(err)
}
st, _, _ = blockchain.State()
var valueX common.Hash
var valueX uint256.Int
st.GetState(contractAddress, &key0, &valueX)
if valueX != correctValueX {
t.Fatalf("storage value has changed after reorg: %x, expected %x", valueX, correctValueX)
Expand Down Expand Up @@ -457,7 +459,8 @@ func TestReorgOverStateChange(t *testing.T) {
}

// Remember value of field "x" (storage item 0) after the first block, to check after rewinding
var key0, correctValueX common.Hash
var key0 common.Hash
var correctValueX uint256.Int
st.GetState(contractAddress, &key0, &correctValueX)

fmt.Println("Insert block 2")
Expand All @@ -482,7 +485,7 @@ func TestReorgOverStateChange(t *testing.T) {
t.Fatal(err)
}
st, _, _ = blockchain.State()
var valueX common.Hash
var valueX uint256.Int
st.GetState(contractAddress, &key0, &valueX)
if valueX != correctValueX {
t.Fatalf("storage value has changed after reorg: %x, expected %x", valueX, correctValueX)
Expand Down Expand Up @@ -746,9 +749,10 @@ func TestCreateOnExistingStorage(t *testing.T) {
t.Error("expected contractAddress to exist at the block 1", contractAddress.String())
}

var key0, check0 common.Hash
var key0 common.Hash
var check0 uint256.Int
st.GetState(contractAddress, &key0, &check0)
if check0 != common.HexToHash("0x0") {
if !check0.IsZero() {
t.Errorf("expected 0x00 in position 0, got: %x", check0)
}
}
Expand All @@ -760,12 +764,12 @@ func TestReproduceCrash(t *testing.T) {
// 1. Setting storageKey 1 to a non-zero value
// 2. Setting storageKey 2 to a non-zero value
// 3. Setting both storageKey1 and storageKey2 to zero values
value0 := common.Hash{}
value0 := uint256.NewInt()
contract := common.HexToAddress("0x71dd1027069078091B3ca48093B00E4735B20624")
storageKey1 := common.HexToHash("0x0e4c0e7175f9d22279a4f63ff74f7fa28b7a954a6454debaa62ce43dd9132541")
value1 := common.HexToHash("0x016345785d8a0000")
value1 := uint256.NewInt().SetUint64(0x016345785d8a0000)
storageKey2 := common.HexToHash("0x0e4c0e7175f9d22279a4f63ff74f7fa28b7a954a6454debaa62ce43dd9132542")
value2 := common.HexToHash("0x58c00a51")
value2 := uint256.NewInt().SetUint64(0x58c00a51)
db := ethdb.NewMemDatabase()
tds := state.NewTrieDbState(common.Hash{}, db, 0)

Expand All @@ -780,22 +784,22 @@ func TestReproduceCrash(t *testing.T) {
}
// Start the 2nd transaction
tds.StartNewBuffer()
intraBlockState.SetState(contract, storageKey1, value1)
intraBlockState.SetState(contract, &storageKey1, *value1)
if err := intraBlockState.FinalizeTx(ctx, tsw); err != nil {
t.Errorf("error finalising 1st tx: %v", err)
}
// Start the 3rd transaction
tds.StartNewBuffer()
intraBlockState.AddBalance(contract, big.NewInt(1000000000))
intraBlockState.SetState(contract, storageKey2, value2)
intraBlockState.SetState(contract, &storageKey2, *value2)
if err := intraBlockState.FinalizeTx(ctx, tsw); err != nil {
t.Errorf("error finalising 1st tx: %v", err)
}
// Start the 4th transaction - clearing both storage cells
tds.StartNewBuffer()
intraBlockState.SubBalance(contract, big.NewInt(1000000000))
intraBlockState.SetState(contract, storageKey1, value0)
intraBlockState.SetState(contract, storageKey2, value0)
intraBlockState.SetState(contract, &storageKey1, *value0)
intraBlockState.SetState(contract, &storageKey2, *value0)
if err := intraBlockState.FinalizeTx(ctx, tsw); err != nil {
t.Errorf("error finalising 1st tx: %v", err)
}
Expand Down
Loading