Skip to content

Commit

Permalink
Made in memory mutation compatible with all buckets (erigontech#4454)
Browse files Browse the repository at this point in the history 
* progress #1

* progress #2

* proper file naming

* more mature memory mutation
  • Loading branch information
@Giulio2002
Giulio2002 authored Jun 14, 2022
1 parent cad32ee commit ff5cbcb
Show file tree
Hide file tree
Showing 3 changed files with 39 additions and 30 deletions.
20 changes: 11 additions & 9 deletions ethdb/olddb/memorymutation.go → ethdb/olddb/memory_mutation.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,7 +30,6 @@ type memorymutation struct {
memDb kv.RwDB
deletedEntries map[string]map[string]struct{}
clearedTables map[string]struct{}
dupsortTables map[string]struct{}
db kv.Tx
}

Expand All @@ -54,11 +53,6 @@ func NewMemoryBatch(tx kv.Tx) *memorymutation {
memTx: memTx,
deletedEntries: make(map[string]map[string]struct{}),
clearedTables: make(map[string]struct{}),
dupsortTables: map[string]struct{}{
kv.AccountChangeSet: {},
kv.StorageChangeSet: {},
kv.HashedStorage: {},
},
}
}

Expand Down Expand Up @@ -305,7 +299,7 @@ func (m *memorymutation) Flush(tx kv.RwTx) error {
}
// Iterate over each bucket and apply changes accordingly.
for _, bucket := range buckets {
if _, ok := m.dupsortTables[bucket]; ok && bucket != kv.HashedStorage {
if isTablePurelyDupsort(bucket) {
cbucket, err := m.memTx.CursorDupSort(bucket)
if err != nil {
return err
Expand Down Expand Up @@ -343,6 +337,16 @@ func (m *memorymutation) Flush(tx kv.RwTx) error {
return nil
}

// Check if a bucket is dupsorted and has dupsort conversion off
func isTablePurelyDupsort(bucket string) bool {
config, ok := kv.ChaindataTablesCfg[bucket]
// If we do not have the configuration we assume it is not dupsorted
if !ok {
return false
}
return !config.AutoDupSortKeysConversion && config.Flags == kv.DupSort
}

// Cursor creates a new cursor (the real fun begins here)
func (m *memorymutation) makeCursor(bucket string) (kv.RwCursorDupSort, error) {
c := &memorymutationcursor{}
Expand All @@ -361,8 +365,6 @@ func (m *memorymutation) makeCursor(bucket string) (kv.RwCursorDupSort, error) {
if err != nil {
return nil, err
}
_, isDupsort := m.dupsortTables[bucket]
c.isDupsort = isDupsort
c.memCursor = c.memDupCursor
c.mutation = m
return c, err
Expand Down
49 changes: 28 additions & 21 deletions ethdb/olddb/memorymutationcursor.go → ethdb/olddb/memory_mutation_cursor.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -36,8 +36,6 @@ type memorymutationcursor struct {
memDupCursor kv.RwCursorDupSort
// we keep the index in the slice of pairs we are at.
isPrevFromDb bool
// Flag for dupsort mode
isDupsort bool
// entry history
currentPair cursorentry
currentDbEntry cursorentry
Expand Down Expand Up @@ -65,7 +63,7 @@ func (m *memorymutationcursor) First() ([]byte, []byte, error) {
}
}

return m.goForward(memKey, memValue, dbKey, dbValue)
return m.goForward(memKey, memValue, dbKey, dbValue, false)
}

func (m *memorymutationcursor) getNextOnDb(dup bool) (key []byte, value []byte, err error) {
Expand Down Expand Up @@ -98,47 +96,56 @@ func (m *memorymutationcursor) getNextOnDb(dup bool) (key []byte, value []byte,
}

func (m *memorymutationcursor) convertAutoDupsort(key []byte, value []byte) []byte {
// The only dupsorted table we are interested is HashedStorage
if m.table != kv.HashedStorage {
config, ok := kv.ChaindataTablesCfg[m.table]
// If we do not have the configuration we assume it is not dupsorted
if !ok || !config.AutoDupSortKeysConversion {
return key
}
return append(key, value[:32]...)
if len(key) != config.DupToLen {
return key
}
return append(key, value[:config.DupFromLen-config.DupToLen]...)
}

// Current return the current key and values the cursor is on.
func (m *memorymutationcursor) Current() ([]byte, []byte, error) {
return common.CopyBytes(m.currentPair.key), common.CopyBytes(m.currentPair.value), nil
}

func (m *memorymutationcursor) skipIntersection(memKey, memValue, dbKey, dbValue []byte) (newDbKey []byte, newDbValue []byte, err error) {
func (m *memorymutationcursor) skipIntersection(memKey, memValue, dbKey, dbValue []byte, dup bool) (newDbKey []byte, newDbValue []byte, err error) {
newDbKey = dbKey
newDbValue = dbValue
config, ok := kv.ChaindataTablesCfg[m.table]
dupsortOffset := 0
if ok && config.AutoDupSortKeysConversion {
dupsortOffset = config.DupFromLen - config.DupToLen
}
// Check for duplicates
if bytes.Compare(memKey, dbKey) == 0 {
if !m.isDupsort {
if newDbKey, newDbValue, err = m.getNextOnDb(false); err != nil {
if !dup {
if newDbKey, newDbValue, err = m.getNextOnDb(dup); err != nil {
return
}
} else if bytes.Compare(memValue, dbValue) == 0 {
if newDbKey, newDbValue, err = m.getNextOnDb(true); err != nil {
if newDbKey, newDbValue, err = m.getNextOnDb(dup); err != nil {
return
}
} else if len(memValue) >= 32 && len(dbValue) >= 32 && m.table == kv.HashedStorage && bytes.Compare(memValue[:32], dbValue[:32]) == 0 {
if newDbKey, newDbValue, err = m.getNextOnDb(true); err != nil {
} else if dupsortOffset != 0 && len(memValue) >= dupsortOffset && len(dbValue) >= dupsortOffset && bytes.Compare(memValue[:dupsortOffset], dbValue[:dupsortOffset]) == 0 {
if newDbKey, newDbValue, err = m.getNextOnDb(dup); err != nil {
return
}
}
}
return
}

func (m *memorymutationcursor) goForward(memKey, memValue, dbKey, dbValue []byte) ([]byte, []byte, error) {
func (m *memorymutationcursor) goForward(memKey, memValue, dbKey, dbValue []byte, dup bool) ([]byte, []byte, error) {
var err error
if memValue == nil && dbValue == nil {
return nil, nil, nil
}

dbKey, dbValue, err = m.skipIntersection(memKey, memValue, dbKey, dbValue)
dbKey, dbValue, err = m.skipIntersection(memKey, memValue, dbKey, dbValue, dup)
if err != nil {
return nil, nil, err
}
Expand Down Expand Up @@ -173,15 +180,15 @@ func (m *memorymutationcursor) Next() ([]byte, []byte, error) {
if err != nil {
return nil, nil, err
}
return m.goForward(m.currentMemEntry.key, m.currentMemEntry.value, k, v)
return m.goForward(m.currentMemEntry.key, m.currentMemEntry.value, k, v, false)
}

memK, memV, err := m.memCursor.Next()
if err != nil {
return nil, nil, err
}

return m.goForward(memK, memV, m.currentDbEntry.key, m.currentDbEntry.value)
return m.goForward(memK, memV, m.currentDbEntry.key, m.currentDbEntry.value, false)
}

// NextDup returns the next element of the mutation.
Expand All @@ -192,15 +199,15 @@ func (m *memorymutationcursor) NextDup() ([]byte, []byte, error) {
if err != nil {
return nil, nil, err
}
return m.goForward(m.currentMemEntry.key, m.currentMemEntry.value, k, v)
return m.goForward(m.currentMemEntry.key, m.currentMemEntry.value, k, v, true)
}

memK, memV, err := m.memDupCursor.NextDup()
if err != nil {
return nil, nil, err
}

return m.goForward(memK, memV, m.currentDbEntry.key, m.currentDbEntry.value)
return m.goForward(memK, memV, m.currentDbEntry.key, m.currentDbEntry.value, true)
}

// Seek move pointer to a key at a certain position.
Expand All @@ -222,7 +229,7 @@ func (m *memorymutationcursor) Seek(seek []byte) ([]byte, []byte, error) {
if err != nil {
return nil, nil, err
}
return m.goForward(memKey, memValue, dbKey, dbValue)
return m.goForward(memKey, memValue, dbKey, dbValue, false)
}

// Seek move pointer to a key at a certain position.
Expand Down Expand Up @@ -309,7 +316,7 @@ func (m *memorymutationcursor) SeekBothRange(key, value []byte) ([]byte, error)
if err != nil {
return nil, err
}
_, retValue, err := m.goForward(key, memValue, key, dbValue)
_, retValue, err := m.goForward(key, memValue, key, dbValue, true)
return retValue, err
}

Expand All @@ -325,7 +332,7 @@ func (m *memorymutationcursor) Last() ([]byte, []byte, error) {
return nil, nil, err
}

dbKey, dbValue, err = m.skipIntersection(memKey, memValue, dbKey, dbValue)
dbKey, dbValue, err = m.skipIntersection(memKey, memValue, dbKey, dbValue, false)
if err != nil {
return nil, nil, err
}
Expand Down
0 ethdb/olddb/memorymutation_test.go → ethdb/olddb/memory_mutation_test.go
View file
File renamed without changes.

0 comments on commit ff5cbcb

Please sign in to comment.