From 4e04454a5ce9aa943a8f9851cbc18c801042a771 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sat, 9 Dec 2023 15:14:59 +0330 Subject: [PATCH 01/30] feat: implement lru cache for account in store --- store/account.go | 36 ++++++++++++++++++++++-------------- store/account_test.go | 4 +++- 2 files changed, 25 insertions(+), 15 deletions(-) diff --git a/store/account.go b/store/account.go index e58f38554..a491d6ec3 100644 --- a/store/account.go +++ b/store/account.go @@ -1,6 +1,7 @@ package store import ( + lru "github.com/hashicorp/golang-lru/v2" "github.com/pactus-project/pactus/crypto" "github.com/pactus-project/pactus/types/account" "github.com/pactus-project/pactus/util/logger" @@ -8,18 +9,25 @@ import ( "github.com/syndtr/goleveldb/leveldb/util" ) +const ( + lruCacheSize = 1024 +) + type accountStore struct { - db *leveldb.DB - addressMap map[crypto.Address]*account.Account - total int32 + db *leveldb.DB + addrLruCache *lru.Cache[crypto.Address, *account.Account] + total int32 } func accountKey(addr crypto.Address) []byte { return append(accountPrefix, addr.Bytes()...) } func newAccountStore(db *leveldb.DB) *accountStore { total := int32(0) - numberMap := make(map[int32]*account.Account) - addressMap := make(map[crypto.Address]*account.Account) + addrLruCache, err := lru.New[crypto.Address, *account.Account](lruCacheSize) + if err != nil { + logger.Panic("unable to create new instance of lru cache", "error", err) + } + r := util.BytesPrefix(accountPrefix) iter := db.NewIterator(r, nil) for iter.Next() { @@ -34,26 +42,25 @@ func newAccountStore(db *leveldb.DB) *accountStore { var addr crypto.Address copy(addr[:], key[1:]) - numberMap[acc.Number()] = acc - addressMap[addr] = acc + addrLruCache.Add(addr, acc) total++ } iter.Release() return &accountStore{ - db: db, - total: total, - addressMap: addressMap, + db: db, + total: total, + addrLruCache: addrLruCache, } } func (as *accountStore) hasAccount(addr crypto.Address) bool { - _, ok := as.addressMap[addr] + _, ok := as.addrLruCache.Get(addr) return ok } func (as *accountStore) account(addr crypto.Address) (*account.Account, error) { - acc, ok := as.addressMap[addr] + acc, ok := as.addrLruCache.Get(addr) if ok { return acc.Clone(), nil } @@ -62,7 +69,8 @@ func (as *accountStore) account(addr crypto.Address) (*account.Account, error) { } func (as *accountStore) iterateAccounts(consumer func(crypto.Address, *account.Account) (stop bool)) { - for addr, acc := range as.addressMap { + for _, addr := range as.addrLruCache.Keys() { + acc, _ := as.addrLruCache.Get(addr) stopped := consumer(addr, acc.Clone()) if stopped { return @@ -81,7 +89,7 @@ func (as *accountStore) updateAccount(batch *leveldb.Batch, addr crypto.Address, if !as.hasAccount(addr) { as.total++ } - as.addressMap[addr] = acc + as.addrLruCache.Add(addr, acc) batch.Put(accountKey(addr), data) } diff --git a/store/account_test.go b/store/account_test.go index 6470e0026..81f7d4bf5 100644 --- a/store/account_test.go +++ b/store/account_test.go @@ -140,5 +140,7 @@ func TestAccountDeepCopy(t *testing.T) { acc2, _ := td.store.Account(addr) acc2.AddToBalance(1) - assert.NotEqual(t, td.store.accountStore.addressMap[addr].Hash(), acc2.Hash()) + + expectedAcc, _ := td.store.accountStore.addrLruCache.Get(addr) + assert.NotEqual(t, expectedAcc.Hash(), acc2.Hash()) } From 13902f93c960cfe1c19e432cee9394fa3845b4fb Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sat, 9 Dec 2023 15:46:35 +0330 Subject: [PATCH 02/30] feat: implement lru cache for public key in store --- store/account.go | 4 ++-- store/store.go | 21 ++++++++++++++++++--- store/store_test.go | 10 +++++++++- 3 files changed, 29 insertions(+), 6 deletions(-) diff --git a/store/account.go b/store/account.go index a491d6ec3..a49ae9d26 100644 --- a/store/account.go +++ b/store/account.go @@ -10,7 +10,7 @@ import ( ) const ( - lruCacheSize = 1024 + accLruCacheSize = 1024 ) type accountStore struct { @@ -23,7 +23,7 @@ func accountKey(addr crypto.Address) []byte { return append(accountPrefix, addr. func newAccountStore(db *leveldb.DB) *accountStore { total := int32(0) - addrLruCache, err := lru.New[crypto.Address, *account.Account](lruCacheSize) + addrLruCache, err := lru.New[crypto.Address, *account.Account](accLruCacheSize) if err != nil { logger.Panic("unable to create new instance of lru cache", "error", err) } diff --git a/store/store.go b/store/store.go index c0847bdef..1a8faaa48 100644 --- a/store/store.go +++ b/store/store.go @@ -5,6 +5,7 @@ import ( "errors" "sync" + lru "github.com/hashicorp/golang-lru/v2" "github.com/pactus-project/pactus/crypto" "github.com/pactus-project/pactus/crypto/bls" "github.com/pactus-project/pactus/crypto/hash" @@ -25,9 +26,10 @@ var ( ErrBadOffset = errors.New("offset is out of range") ) -const lastStoreVersion = int32(1) - -// TODO: add cache for me +const ( + lastStoreVersion = int32(1) + pubKeyLruCacheSize = 1024 +) var ( lastInfoKey = []byte{0x00} @@ -53,6 +55,7 @@ type store struct { lk sync.RWMutex config *Config + pubKeyLruCache *lru.Cache[crypto.Address, *bls.PublicKey] db *leveldb.DB batch *leveldb.Batch blockStore *blockStore @@ -66,6 +69,12 @@ func NewStore(conf *Config) (Store, error) { Strict: opt.DefaultStrict, Compression: opt.NoCompression, } + + pubKeyLruCache, err := lru.New[crypto.Address, *bls.PublicKey](pubKeyLruCacheSize) + if err != nil { + return nil, err + } + db, err := leveldb.OpenFile(conf.StorePath(), options) if err != nil { return nil, err @@ -73,6 +82,7 @@ func NewStore(conf *Config) (Store, error) { s := &store{ config: conf, db: db, + pubKeyLruCache: pubKeyLruCache, batch: new(leveldb.Batch), blockStore: newBlockStore(db), txStore: newTxStore(db), @@ -156,6 +166,10 @@ func (s *store) BlockHash(height uint32) hash.Hash { } func (s *store) PublicKey(addr crypto.Address) (*bls.PublicKey, error) { + if pubKey, ok := s.pubKeyLruCache.Get(addr); ok { + return pubKey, nil + } + bs, err := tryGet(s.db, publicKeyKey(addr)) if err != nil { return nil, err @@ -165,6 +179,7 @@ func (s *store) PublicKey(addr crypto.Address) (*bls.PublicKey, error) { return nil, err } + s.pubKeyLruCache.Add(addr, pubKey) return pubKey, err } diff --git a/store/store_test.go b/store/store_test.go index 91b9d5a2d..02ba18c84 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -111,8 +111,11 @@ func TestIndexingPublicKeys(t *testing.T) { for _, trx := range blk.Transactions() { addr := trx.Payload().Signer() pub, found := td.store.PublicKey(addr) + pubKeyLruCache, ok := td.store.pubKeyLruCache.Get(addr) assert.NoError(t, found) + assert.True(t, ok) + assert.Equal(t, pub, pubKeyLruCache) if addr.IsAccountAddress() { assert.Equal(t, pub.AccountAddress(), addr) @@ -121,9 +124,14 @@ func TestIndexingPublicKeys(t *testing.T) { } } - pub, found := td.store.PublicKey(td.RandValAddress()) + randValAddress := td.RandValAddress() + pub, found := td.store.PublicKey(randValAddress) + pubKeyLruCache, ok := td.store.pubKeyLruCache.Get(randValAddress) + assert.Error(t, found) assert.Nil(t, pub) + assert.False(t, ok) + assert.Nil(t, pubKeyLruCache) } func TestStrippedPublicKey(t *testing.T) { From 6630b6f9acbb6aba7bb62db7171522b903688432 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sun, 10 Dec 2023 00:39:14 +0330 Subject: [PATCH 03/30] feat: implement lru cache for transactions --- go.mod | 1 + go.sum | 2 ++ sandbox/sandbox.go | 3 +++ store/account.go | 40 ++++++++++++++++++++++++---------------- store/block.go | 12 ++---------- store/store.go | 19 ++++++++++++++----- store/tx.go | 4 ++++ 7 files changed, 50 insertions(+), 31 deletions(-) diff --git a/go.mod b/go.mod index 2797d502f..7394ef766 100644 --- a/go.mod +++ b/go.mod @@ -3,6 +3,7 @@ module github.com/pactus-project/pactus go 1.21 require ( + github.com/edwingeng/deque/v2 v2.1.1 github.com/fxamacker/cbor/v2 v2.5.0 github.com/google/uuid v1.4.0 github.com/gorilla/handlers v1.5.1 diff --git a/go.sum b/go.sum index a678756bd..6a6540eee 100644 --- a/go.sum +++ b/go.sum @@ -59,6 +59,8 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= +github.com/edwingeng/deque/v2 v2.1.1 h1:+xjC3TnaeMPLZMi7QQf9jN2K00MZmTwruApqplbL9IY= +github.com/edwingeng/deque/v2 v2.1.1/go.mod h1:HukI8CQe9KDmZCcURPZRYVYjH79Zy2tIjTF9sN3Bgb0= github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= diff --git a/sandbox/sandbox.go b/sandbox/sandbox.go index 3965f5ac9..6cf30d6c6 100644 --- a/sandbox/sandbox.go +++ b/sandbox/sandbox.go @@ -287,6 +287,9 @@ func (sb *sandbox) VerifyProof(blockHeight uint32, proof sortition.Proof, val *v if err != nil { return false } + // block height reach to block + // parse block + // find sortition seed := blk.Header().SortitionSeed() return sortition.VerifyProof(seed, proof, val.PublicKey(), sb.totalPower, val.Power()) } diff --git a/store/account.go b/store/account.go index a49ae9d26..a51499b2a 100644 --- a/store/account.go +++ b/store/account.go @@ -31,18 +31,6 @@ func newAccountStore(db *leveldb.DB) *accountStore { r := util.BytesPrefix(accountPrefix) iter := db.NewIterator(r, nil) for iter.Next() { - key := iter.Key() - value := iter.Value() - - acc, err := account.FromBytes(value) - if err != nil { - logger.Panic("unable to decode account", "error", err) - } - - var addr crypto.Address - copy(addr[:], key[1:]) - - addrLruCache.Add(addr, acc) total++ } iter.Release() @@ -55,7 +43,10 @@ func newAccountStore(db *leveldb.DB) *accountStore { } func (as *accountStore) hasAccount(addr crypto.Address) bool { - _, ok := as.addrLruCache.Get(addr) + ok := as.addrLruCache.Contains(addr) + if !ok { + ok = tryHas(as.db, accountKey(addr)) + } return ok } @@ -65,17 +56,34 @@ func (as *accountStore) account(addr crypto.Address) (*account.Account, error) { return acc.Clone(), nil } - return nil, ErrNotFound + rawData, err := tryGet(as.db, accountKey(addr)) + if err != nil { + return nil, err + } + return account.FromBytes(rawData) } func (as *accountStore) iterateAccounts(consumer func(crypto.Address, *account.Account) (stop bool)) { - for _, addr := range as.addrLruCache.Keys() { - acc, _ := as.addrLruCache.Get(addr) + r := util.BytesPrefix(accountPrefix) + iter := as.db.NewIterator(r, nil) + for iter.Next() { + key := iter.Key() + value := iter.Value() + + acc, err := account.FromBytes(value) + if err != nil { + logger.Panic("unable to decode account", "error", err) + } + + var addr crypto.Address + copy(addr[:], key[1:]) + stopped := consumer(addr, acc.Clone()) if stopped { return } } + iter.Release() } // This function takes ownership of the account pointer. diff --git a/store/block.go b/store/block.go index c134cbe95..b42d0e10e 100644 --- a/store/block.go +++ b/store/block.go @@ -114,17 +114,9 @@ func (bs *blockStore) blockHeight(h hash.Hash) uint32 { } func (bs *blockStore) hasBlock(height uint32) bool { - has, err := bs.db.Has(blockKey(height), nil) - if err != nil { - return false - } - return has + return tryHas(bs.db, blockKey(height)) } func (bs *blockStore) hasPublicKey(addr crypto.Address) bool { - has, err := bs.db.Has(publicKeyKey(addr), nil) - if err != nil { - return false - } - return has + return tryHas(bs.db, publicKeyKey(addr)) } diff --git a/store/store.go b/store/store.go index 1a8faaa48..81f573bf2 100644 --- a/store/store.go +++ b/store/store.go @@ -5,6 +5,7 @@ import ( "errors" "sync" + "github.com/edwingeng/deque/v2" lru "github.com/hashicorp/golang-lru/v2" "github.com/pactus-project/pactus/crypto" "github.com/pactus-project/pactus/crypto/bls" @@ -45,16 +46,26 @@ func tryGet(db *leveldb.DB, key []byte) ([]byte, error) { data, err := db.Get(key, nil) if err != nil { // Probably key doesn't exist in database - logger.Trace("database error", "error", err, "key", key) + logger.Trace("database get error", "error", err, "key", key) return nil, err } return data, nil } +func tryHas(db *leveldb.DB, key []byte) bool { + ok, err := db.Has(key, nil) + if err != nil { + logger.Error("database has error", "error", err, "key", key) + return false + } + return ok +} + type store struct { lk sync.RWMutex config *Config + txQueue *deque.Deque[tx.Tx] pubKeyLruCache *lru.Cache[crypto.Address, *bls.PublicKey] db *leveldb.DB batch *leveldb.Batch @@ -82,6 +93,7 @@ func NewStore(conf *Config) (Store, error) { s := &store{ config: conf, db: db, + txQueue: deque.NewDeque[tx.Tx](), pubKeyLruCache: pubKeyLruCache, batch: new(leveldb.Batch), blockStore: newBlockStore(db), @@ -211,14 +223,11 @@ func (s *store) Transaction(id tx.ID) (*CommittedTx, error) { }, nil } -// TODO implement Dequeue for this function, for the better performance. func (s *store) AnyRecentTransaction(id tx.ID) bool { s.lk.Lock() defer s.lk.Unlock() - pos, _ := s.txStore.tx(id) - - return pos != nil + return s.txStore.hasTX(id) } func (s *store) HasAccount(addr crypto.Address) bool { diff --git a/store/tx.go b/store/tx.go index 89b61ee0a..89a0b9c3a 100644 --- a/store/tx.go +++ b/store/tx.go @@ -37,6 +37,10 @@ func (ts *txStore) saveTx(batch *leveldb.Batch, id tx.ID, reg *blockRegion) { batch.Put(txKey, w.Bytes()) } +func (ts *txStore) hasTX(id tx.ID) bool { + return tryHas(ts.db, txKey(id)) +} + func (ts *txStore) tx(id tx.ID) (*blockRegion, error) { data, err := tryGet(ts.db, txKey(id)) if err != nil { From ec8a222ef67de4470ed43bc3982e3bac1a46391e Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sun, 10 Dec 2023 18:02:56 +0330 Subject: [PATCH 04/30] fix: rename print errors and addr cache name --- store/account.go | 18 +++++++++--------- store/account_test.go | 2 +- store/store.go | 4 ++-- 3 files changed, 12 insertions(+), 12 deletions(-) diff --git a/store/account.go b/store/account.go index a51499b2a..431adf67f 100644 --- a/store/account.go +++ b/store/account.go @@ -14,9 +14,9 @@ const ( ) type accountStore struct { - db *leveldb.DB - addrLruCache *lru.Cache[crypto.Address, *account.Account] - total int32 + db *leveldb.DB + addrCache *lru.Cache[crypto.Address, *account.Account] + total int32 } func accountKey(addr crypto.Address) []byte { return append(accountPrefix, addr.Bytes()...) } @@ -36,14 +36,14 @@ func newAccountStore(db *leveldb.DB) *accountStore { iter.Release() return &accountStore{ - db: db, - total: total, - addrLruCache: addrLruCache, + db: db, + total: total, + addrCache: addrLruCache, } } func (as *accountStore) hasAccount(addr crypto.Address) bool { - ok := as.addrLruCache.Contains(addr) + ok := as.addrCache.Contains(addr) if !ok { ok = tryHas(as.db, accountKey(addr)) } @@ -51,7 +51,7 @@ func (as *accountStore) hasAccount(addr crypto.Address) bool { } func (as *accountStore) account(addr crypto.Address) (*account.Account, error) { - acc, ok := as.addrLruCache.Get(addr) + acc, ok := as.addrCache.Get(addr) if ok { return acc.Clone(), nil } @@ -97,7 +97,7 @@ func (as *accountStore) updateAccount(batch *leveldb.Batch, addr crypto.Address, if !as.hasAccount(addr) { as.total++ } - as.addrLruCache.Add(addr, acc) + as.addrCache.Add(addr, acc) batch.Put(accountKey(addr), data) } diff --git a/store/account_test.go b/store/account_test.go index 81f7d4bf5..aa54b72f0 100644 --- a/store/account_test.go +++ b/store/account_test.go @@ -141,6 +141,6 @@ func TestAccountDeepCopy(t *testing.T) { acc2, _ := td.store.Account(addr) acc2.AddToBalance(1) - expectedAcc, _ := td.store.accountStore.addrLruCache.Get(addr) + expectedAcc, _ := td.store.accountStore.addrCache.Get(addr) assert.NotEqual(t, expectedAcc.Hash(), acc2.Hash()) } diff --git a/store/store.go b/store/store.go index 81f573bf2..54407cdd4 100644 --- a/store/store.go +++ b/store/store.go @@ -46,7 +46,7 @@ func tryGet(db *leveldb.DB, key []byte) ([]byte, error) { data, err := db.Get(key, nil) if err != nil { // Probably key doesn't exist in database - logger.Trace("database get error", "error", err, "key", key) + logger.Trace("database `get` error", "error", err, "key", key) return nil, err } return data, nil @@ -55,7 +55,7 @@ func tryGet(db *leveldb.DB, key []byte) ([]byte, error) { func tryHas(db *leveldb.DB, key []byte) bool { ok, err := db.Has(key, nil) if err != nil { - logger.Error("database has error", "error", err, "key", key) + logger.Error("database `has` error", "error", err, "key", key) return false } return ok From d9c4cdd1db0c18bea958dd2ae9d7d0cc90f091d7 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Mon, 11 Dec 2023 13:37:54 +0330 Subject: [PATCH 05/30] feat: implement recent transactions and sortition seeds cache --- go.mod | 2 +- go.sum | 4 ++-- node/node.go | 3 ++- sandbox/sandbox.go | 20 +++++------------ store/account.go | 9 +++++++- store/account_test.go | 4 ++-- store/block.go | 28 ++++++++++++++++++++--- store/interface.go | 2 ++ store/mock.go | 6 +++++ store/store.go | 34 +++++++++++++++------------- store/store_test.go | 6 ++--- store/tx.go | 44 ++++++++++++++++++++++++++++--------- store/validator_test.go | 6 ++--- tests/main_test.go | 2 +- util/linkedmap/linkedmap.go | 24 +++++++++++++++++--- 15 files changed, 134 insertions(+), 60 deletions(-) diff --git a/go.mod b/go.mod index 7394ef766..d9f410578 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,7 @@ module github.com/pactus-project/pactus go 1.21 require ( - github.com/edwingeng/deque/v2 v2.1.1 + github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 github.com/fxamacker/cbor/v2 v2.5.0 github.com/google/uuid v1.4.0 github.com/gorilla/handlers v1.5.1 diff --git a/go.sum b/go.sum index 6a6540eee..b00ad96f0 100644 --- a/go.sum +++ b/go.sum @@ -59,8 +59,8 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/edwingeng/deque/v2 v2.1.1 h1:+xjC3TnaeMPLZMi7QQf9jN2K00MZmTwruApqplbL9IY= -github.com/edwingeng/deque/v2 v2.1.1/go.mod h1:HukI8CQe9KDmZCcURPZRYVYjH79Zy2tIjTF9sN3Bgb0= +github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 h1:8EXxF+tCLqaVk8AOC29zl2mnhQjwyLxxOTuhUazWRsg= +github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4/go.mod h1:I5sHm0Y0T1u5YjlyqC5GVArM7aNZRUYtTjmJ8mPJFds= github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= diff --git a/node/node.go b/node/node.go index 120160f70..17e19f378 100644 --- a/node/node.go +++ b/node/node.go @@ -61,7 +61,8 @@ func NewNode(genDoc *genesis.Genesis, conf *config.Config, txPool := txpool.NewTxPool(conf.TxPool, messageCh) // TODO implement dequeue for recent transaction - str, err := store.NewStore(conf.Store) + str, err := store.NewStore(conf.Store, + genDoc.Params().TransactionToLiveInterval, genDoc.Params().SortitionInterval) if err != nil { return nil, err } diff --git a/sandbox/sandbox.go b/sandbox/sandbox.go index 6cf30d6c6..96268d20c 100644 --- a/sandbox/sandbox.go +++ b/sandbox/sandbox.go @@ -277,21 +277,11 @@ func (sb *sandbox) PowerDelta() int64 { // VerifyProof verifies proof of a sortition transaction. func (sb *sandbox) VerifyProof(blockHeight uint32, proof sortition.Proof, val *validator.Validator) bool { - committedBlock, err := sb.store.Block(blockHeight) - if err != nil { - return false - } - // TODO: improvement: - // We can get the sortition seed without parsing the block - blk, err := committedBlock.ToBlock() - if err != nil { - return false - } - // block height reach to block - // parse block - // find sortition - seed := blk.Header().SortitionSeed() - return sortition.VerifyProof(seed, proof, val.PublicKey(), sb.totalPower, val.Power()) + sb.lk.RLock() + defer sb.lk.RUnlock() + + seed := sb.store.SortitionSeed(blockHeight, sb.height+1) + return sortition.VerifyProof(*seed, proof, val.PublicKey(), sb.totalPower, val.Power()) } func (sb *sandbox) CommitTransaction(trx *tx.Tx) { diff --git a/store/account.go b/store/account.go index 431adf67f..33eedc7f0 100644 --- a/store/account.go +++ b/store/account.go @@ -60,7 +60,14 @@ func (as *accountStore) account(addr crypto.Address) (*account.Account, error) { if err != nil { return nil, err } - return account.FromBytes(rawData) + + acc, err = account.FromBytes(rawData) + if err != nil { + return nil, err + } + + as.addrCache.Add(addr, acc) + return acc, nil } func (as *accountStore) iterateAccounts(consumer func(crypto.Address, *account.Account) (stop bool)) { diff --git a/store/account_test.go b/store/account_test.go index aa54b72f0..f775690c7 100644 --- a/store/account_test.go +++ b/store/account_test.go @@ -57,7 +57,7 @@ func TestAccountBatchSaving(t *testing.T) { t.Run("Close and load db", func(t *testing.T) { td.store.Close() - store, _ := NewStore(td.store.config) + store, _ := NewStore(td.store.config, 8640, 17) assert.Equal(t, store.TotalAccounts(), total) }) } @@ -93,7 +93,7 @@ func TestAccountByAddress(t *testing.T) { t.Run("Reopen the store", func(t *testing.T) { td.store.Close() - store, _ := NewStore(td.store.config) + store, _ := NewStore(td.store.config, 8640, 17) acc, err := store.Account(lastAddr) assert.NoError(t, err) diff --git a/store/block.go b/store/block.go index b42d0e10e..c452ef38c 100644 --- a/store/block.go +++ b/store/block.go @@ -3,8 +3,10 @@ package store import ( "bytes" + "github.com/eapache/queue/v2" "github.com/pactus-project/pactus/crypto" "github.com/pactus-project/pactus/crypto/hash" + "github.com/pactus-project/pactus/sortition" "github.com/pactus-project/pactus/types/block" "github.com/pactus-project/pactus/util" "github.com/pactus-project/pactus/util/encoding" @@ -22,12 +24,16 @@ func blockHashKey(h hash.Hash) []byte { } type blockStore struct { - db *leveldb.DB + db *leveldb.DB + sortitionSeedQueue *queue.Queue[*sortition.VerifiableSeed] + sortitionInterval uint32 } -func newBlockStore(db *leveldb.DB) *blockStore { +func newBlockStore(db *leveldb.DB, sortitionInterval uint32) *blockStore { return &blockStore{ - db: db, + db: db, + sortitionSeedQueue: queue.New[*sortition.VerifiableSeed](), + sortitionInterval: sortitionInterval, } } @@ -93,6 +99,13 @@ func (bs *blockStore) saveBlock(batch *leveldb.Batch, height uint32, blk *block. batch.Put(blockKey, w.Bytes()) batch.Put(blockHashKey, util.Uint32ToSlice(height)) + sortitionSeed := blk.Header().SortitionSeed() + + bs.sortitionSeedQueue.Add(&sortitionSeed) + if bs.sortitionSeedQueue.Length() > int(bs.sortitionInterval) { + bs.sortitionSeedQueue.Remove() + } + return regs } @@ -113,6 +126,15 @@ func (bs *blockStore) blockHeight(h hash.Hash) uint32 { return util.SliceToUint32(data) } +func (bs *blockStore) sortitionSeed(currentHeight, height uint32) *sortition.VerifiableSeed { + index := currentHeight - height + if index > bs.sortitionInterval { + return nil + } + + return bs.sortitionSeedQueue.Get(int(index)) +} + func (bs *blockStore) hasBlock(height uint32) bool { return tryHas(bs.db, blockKey(height)) } diff --git a/store/interface.go b/store/interface.go index 830a364a4..38d9f2b0a 100644 --- a/store/interface.go +++ b/store/interface.go @@ -4,6 +4,7 @@ import ( "github.com/pactus-project/pactus/crypto" "github.com/pactus-project/pactus/crypto/bls" "github.com/pactus-project/pactus/crypto/hash" + "github.com/pactus-project/pactus/sortition" "github.com/pactus-project/pactus/types/account" "github.com/pactus-project/pactus/types/block" "github.com/pactus-project/pactus/types/certificate" @@ -81,6 +82,7 @@ type Reader interface { Block(height uint32) (*CommittedBlock, error) BlockHeight(h hash.Hash) uint32 BlockHash(height uint32) hash.Hash + SortitionSeed(currentHeight, height uint32) *sortition.VerifiableSeed Transaction(id tx.ID) (*CommittedTx, error) AnyRecentTransaction(id tx.ID) bool PublicKey(addr crypto.Address) (*bls.PublicKey, error) diff --git a/store/mock.go b/store/mock.go index 3114060be..99cd6dc56 100644 --- a/store/mock.go +++ b/store/mock.go @@ -6,6 +6,7 @@ import ( "github.com/pactus-project/pactus/crypto" "github.com/pactus-project/pactus/crypto/bls" "github.com/pactus-project/pactus/crypto/hash" + "github.com/pactus-project/pactus/sortition" "github.com/pactus-project/pactus/types/account" "github.com/pactus-project/pactus/types/block" "github.com/pactus-project/pactus/types/certificate" @@ -65,6 +66,11 @@ func (m *MockStore) BlockHeight(h hash.Hash) uint32 { return 0 } +func (m *MockStore) SortitionSeed(_, _ uint32) *sortition.VerifiableSeed { + // TODO implement me + panic("implement me") +} + func (m *MockStore) PublicKey(addr crypto.Address) (*bls.PublicKey, error) { for _, block := range m.Blocks { for _, trx := range block.Transactions() { diff --git a/store/store.go b/store/store.go index 54407cdd4..e1b133a41 100644 --- a/store/store.go +++ b/store/store.go @@ -5,11 +5,11 @@ import ( "errors" "sync" - "github.com/edwingeng/deque/v2" lru "github.com/hashicorp/golang-lru/v2" "github.com/pactus-project/pactus/crypto" "github.com/pactus-project/pactus/crypto/bls" "github.com/pactus-project/pactus/crypto/hash" + "github.com/pactus-project/pactus/sortition" "github.com/pactus-project/pactus/types/account" "github.com/pactus-project/pactus/types/block" "github.com/pactus-project/pactus/types/certificate" @@ -65,8 +65,7 @@ type store struct { lk sync.RWMutex config *Config - txQueue *deque.Deque[tx.Tx] - pubKeyLruCache *lru.Cache[crypto.Address, *bls.PublicKey] + pubKeyCache *lru.Cache[crypto.Address, *bls.PublicKey] db *leveldb.DB batch *leveldb.Batch blockStore *blockStore @@ -75,13 +74,13 @@ type store struct { validatorStore *validatorStore } -func NewStore(conf *Config) (Store, error) { +func NewStore(conf *Config, transactionToLiveInterval, sortitionInterval uint32) (Store, error) { options := &opt.Options{ Strict: opt.DefaultStrict, Compression: opt.NoCompression, } - pubKeyLruCache, err := lru.New[crypto.Address, *bls.PublicKey](pubKeyLruCacheSize) + pubKeyCache, err := lru.New[crypto.Address, *bls.PublicKey](pubKeyLruCacheSize) if err != nil { return nil, err } @@ -93,11 +92,10 @@ func NewStore(conf *Config) (Store, error) { s := &store{ config: conf, db: db, - txQueue: deque.NewDeque[tx.Tx](), - pubKeyLruCache: pubKeyLruCache, + pubKeyCache: pubKeyCache, batch: new(leveldb.Batch), - blockStore: newBlockStore(db), - txStore: newTxStore(db), + blockStore: newBlockStore(db, sortitionInterval), + txStore: newTxStore(db, transactionToLiveInterval), accountStore: newAccountStore(db), validatorStore: newValidatorStore(db), } @@ -116,10 +114,9 @@ func (s *store) SaveBlock(blk *block.Block, cert *certificate.Certificate) { defer s.lk.Unlock() height := cert.Height() - reg := s.blockStore.saveBlock(s.batch, height, blk) - for i, trx := range blk.Transactions() { - s.txStore.saveTx(s.batch, trx.ID(), ®[i]) - } + regs := s.blockStore.saveBlock(s.batch, height, blk) + s.txStore.saveTxs(s.batch, blk.Transactions(), regs) + s.txStore.pruneCache(height) // Save last certificate: [version: 4 bytes]+[certificate: variant] w := bytes.NewBuffer(make([]byte, 0, 4+cert.SerializeSize())) @@ -177,8 +174,15 @@ func (s *store) BlockHash(height uint32) hash.Hash { return hash.UndefHash } +func (s *store) SortitionSeed(currentHeight, height uint32) *sortition.VerifiableSeed { + s.lk.RLock() + defer s.lk.RUnlock() + + return s.blockStore.sortitionSeed(currentHeight, height) +} + func (s *store) PublicKey(addr crypto.Address) (*bls.PublicKey, error) { - if pubKey, ok := s.pubKeyLruCache.Get(addr); ok { + if pubKey, ok := s.pubKeyCache.Get(addr); ok { return pubKey, nil } @@ -191,7 +195,7 @@ func (s *store) PublicKey(addr crypto.Address) (*bls.PublicKey, error) { return nil, err } - s.pubKeyLruCache.Add(addr, pubKey) + s.pubKeyCache.Add(addr, pubKey) return pubKey, err } diff --git a/store/store_test.go b/store/store_test.go index 02ba18c84..c39c62fc2 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -27,7 +27,7 @@ func setup(t *testing.T) *testData { conf := &Config{ Path: util.TempDirPath(), } - s, err := NewStore(conf) + s, err := NewStore(conf, 8640, 17) require.NoError(t, err) td := &testData{ @@ -111,7 +111,7 @@ func TestIndexingPublicKeys(t *testing.T) { for _, trx := range blk.Transactions() { addr := trx.Payload().Signer() pub, found := td.store.PublicKey(addr) - pubKeyLruCache, ok := td.store.pubKeyLruCache.Get(addr) + pubKeyLruCache, ok := td.store.pubKeyCache.Get(addr) assert.NoError(t, found) assert.True(t, ok) @@ -126,7 +126,7 @@ func TestIndexingPublicKeys(t *testing.T) { randValAddress := td.RandValAddress() pub, found := td.store.PublicKey(randValAddress) - pubKeyLruCache, ok := td.store.pubKeyLruCache.Get(randValAddress) + pubKeyLruCache, ok := td.store.pubKeyCache.Get(randValAddress) assert.Error(t, found) assert.Nil(t, pub) diff --git a/store/tx.go b/store/tx.go index 89a0b9c3a..d08ce410a 100644 --- a/store/tx.go +++ b/store/tx.go @@ -3,8 +3,10 @@ package store import ( "bytes" + "github.com/pactus-project/pactus/types/block" "github.com/pactus-project/pactus/types/tx" "github.com/pactus-project/pactus/util/encoding" + "github.com/pactus-project/pactus/util/linkedmap" "github.com/syndtr/goleveldb/leveldb" ) @@ -17,28 +19,50 @@ type blockRegion struct { func txKey(id tx.ID) []byte { return append(txPrefix, id.Bytes()...) } type txStore struct { - db *leveldb.DB + db *leveldb.DB + txCache *linkedmap.LinkedMap[tx.ID, uint32] + ttl uint32 } -func newTxStore(db *leveldb.DB) *txStore { +func newTxStore(db *leveldb.DB, ttl uint32) *txStore { return &txStore{ - db: db, + db: db, + txCache: linkedmap.NewLinkedMap[tx.ID, uint32](0), + ttl: ttl, } } -func (ts *txStore) saveTx(batch *leveldb.Batch, id tx.ID, reg *blockRegion) { +func (ts *txStore) saveTxs(batch *leveldb.Batch, txs block.Txs, regs []blockRegion) { w := bytes.NewBuffer(make([]byte, 0, 32+4)) - err := encoding.WriteElements(w, ®.height, ®.offset, ®.length) - if err != nil { - panic(err) + + for i, trx := range txs { + reg := regs[i] + err := encoding.WriteElements(w, ®.height, ®.offset, ®.length) + if err != nil { + panic(err) + } + + id := trx.ID() + txKey := txKey(id) + batch.Put(txKey, w.Bytes()) + ts.txCache.PushBack(id, reg.height) } +} + +func (ts *txStore) pruneCache(currentHeight uint32) { + for { + head := ts.txCache.HeadNode() + txHeight := head.Data.Value - txKey := txKey(id) - batch.Put(txKey, w.Bytes()) + if currentHeight-txHeight <= ts.ttl { + break + } + ts.txCache.RemoveHead() + } } func (ts *txStore) hasTX(id tx.ID) bool { - return tryHas(ts.db, txKey(id)) + return ts.txCache.Has(id) } func (ts *txStore) tx(id tx.ID) (*blockRegion, error) { diff --git a/store/validator_test.go b/store/validator_test.go index aab47d47c..f8bc5d1ce 100644 --- a/store/validator_test.go +++ b/store/validator_test.go @@ -60,7 +60,7 @@ func TestValidatorBatchSaving(t *testing.T) { t.Run("Close and load db", func(t *testing.T) { td.store.Close() - store, _ := NewStore(td.store.config) + store, _ := NewStore(td.store.config, 8640, 17) assert.Equal(t, store.TotalValidators(), total) }) } @@ -116,7 +116,7 @@ func TestValidatorByNumber(t *testing.T) { t.Run("Reopen the store", func(t *testing.T) { td.store.Close() - store, _ := NewStore(td.store.config) + store, _ := NewStore(td.store.config, 8640, 17) num := td.RandInt32(total) val, err := store.ValidatorByNumber(num) @@ -160,7 +160,7 @@ func TestValidatorByAddress(t *testing.T) { t.Run("Reopen the store", func(t *testing.T) { td.store.Close() - store, _ := NewStore(td.store.config) + store, _ := NewStore(td.store.config, 8640, 17) num := td.RandInt32(total) val0, _ := store.ValidatorByNumber(num) diff --git a/tests/main_test.go b/tests/main_test.go index 6f48ba35d..1dfdfd95b 100644 --- a/tests/main_test.go +++ b/tests/main_test.go @@ -180,7 +180,7 @@ func TestMain(m *testing.M) { tNodes[i].Stop() } - s, _ := store.NewStore(tConfigs[tNodeIdx1].Store) + s, _ := store.NewStore(tConfigs[tNodeIdx1].Store, 8640, 17) total := int64(0) s.IterateAccounts(func(addr crypto.Address, acc *account.Account) bool { total += acc.Balance() diff --git a/util/linkedmap/linkedmap.go b/util/linkedmap/linkedmap.go index fcfa09e70..29827e0db 100644 --- a/util/linkedmap/linkedmap.go +++ b/util/linkedmap/linkedmap.go @@ -87,6 +87,13 @@ func (lm *LinkedMap[K, V]) TailNode() *ll.Element[Pair[K, V]] { return ln } +func (lm *LinkedMap[K, V]) RemoveTail() { + tail := lm.list.Tail + key := tail.Data.Key + lm.list.Delete(tail) + delete(lm.hashmap, key) +} + // HeadNode returns the LinkNode at the beginning (head) of the LinkedMap. func (lm *LinkedMap[K, V]) HeadNode() *ll.Element[Pair[K, V]] { ln := lm.list.Head @@ -96,6 +103,13 @@ func (lm *LinkedMap[K, V]) HeadNode() *ll.Element[Pair[K, V]] { return ln } +func (lm *LinkedMap[K, V]) RemoveHead() { + head := lm.list.Head + key := head.Data.Key + lm.list.Delete(head) + delete(lm.hashmap, key) +} + // Remove removes the key-value pair with the specified key from the LinkedMap. // It returns true if the key was found and removed, otherwise false. func (lm *LinkedMap[K, V]) Remove(key K) bool { @@ -135,10 +149,14 @@ func (lm *LinkedMap[K, V]) Clear() { // prune removes excess elements from the LinkedMap if its size exceeds the capacity. func (lm *LinkedMap[K, V]) prune() { + if lm.capacity == 0 { + return + } + for lm.list.Length() > lm.capacity { - front := lm.list.Head - key := front.Data.Key - lm.list.Delete(front) + head := lm.list.Head + key := head.Data.Key + lm.list.Delete(head) delete(lm.hashmap, key) } } From c76d80230b7f79b7a9029820289fff02fe2d049f Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Wed, 13 Dec 2023 20:11:20 +0330 Subject: [PATCH 06/30] feat: implement load sortition seed and transactions and save them in cache --- store/block.go | 13 ++++++++----- store/store.go | 31 +++++++++++++++++++++++++++++++ store/tx.go | 6 +++++- 3 files changed, 44 insertions(+), 6 deletions(-) diff --git a/store/block.go b/store/block.go index c452ef38c..8b1af8193 100644 --- a/store/block.go +++ b/store/block.go @@ -100,11 +100,7 @@ func (bs *blockStore) saveBlock(batch *leveldb.Batch, height uint32, blk *block. batch.Put(blockHashKey, util.Uint32ToSlice(height)) sortitionSeed := blk.Header().SortitionSeed() - - bs.sortitionSeedQueue.Add(&sortitionSeed) - if bs.sortitionSeedQueue.Length() > int(bs.sortitionInterval) { - bs.sortitionSeedQueue.Remove() - } + bs.saveToCache(sortitionSeed) return regs } @@ -142,3 +138,10 @@ func (bs *blockStore) hasBlock(height uint32) bool { func (bs *blockStore) hasPublicKey(addr crypto.Address) bool { return tryHas(bs.db, publicKeyKey(addr)) } + +func (bs *blockStore) saveToCache(sortitionSeed sortition.VerifiableSeed) { + bs.sortitionSeedQueue.Add(&sortitionSeed) + if bs.sortitionSeedQueue.Length() > int(bs.sortitionInterval) { + bs.sortitionSeedQueue.Remove() + } +} diff --git a/store/store.go b/store/store.go index e1b133a41..3431fdec1 100644 --- a/store/store.go +++ b/store/store.go @@ -99,6 +99,37 @@ func NewStore(conf *Config, transactionToLiveInterval, sortitionInterval uint32) accountStore: newAccountStore(db), validatorStore: newValidatorStore(db), } + + lc := s.LastCertificate() + if lc == nil { + return s, nil + } + + currentHeight := lc.Height() + startHeight := uint32(1) + if currentHeight > transactionToLiveInterval { + startHeight = currentHeight - transactionToLiveInterval + } + + for i := startHeight; i < currentHeight; i++ { + committedBlock, err := s.Block(i) + if err != nil { + return nil, err + } + blk, err := committedBlock.ToBlock() + if err != nil { + return nil, err + } + + txs := blk.Transactions() + for _, transaction := range txs { + s.txStore.saveToCache(transaction.ID(), i) + } + + sortitionSeed := blk.Header().SortitionSeed() + s.blockStore.saveToCache(sortitionSeed) + } + return s, nil } diff --git a/store/tx.go b/store/tx.go index d08ce410a..016130eba 100644 --- a/store/tx.go +++ b/store/tx.go @@ -45,7 +45,7 @@ func (ts *txStore) saveTxs(batch *leveldb.Batch, txs block.Txs, regs []blockRegi id := trx.ID() txKey := txKey(id) batch.Put(txKey, w.Bytes()) - ts.txCache.PushBack(id, reg.height) + ts.saveToCache(id, reg.height) } } @@ -78,3 +78,7 @@ func (ts *txStore) tx(id tx.ID) (*blockRegion, error) { } return reg, nil } + +func (ts *txStore) saveToCache(id tx.ID, height uint32) { + ts.txCache.PushBack(id, height) +} From b323db904ef3a458215f49d4fd7ff4e4792f9e51 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Wed, 13 Dec 2023 21:27:19 +0330 Subject: [PATCH 07/30] fix: change rlock to lock for sortition seed --- sandbox/sandbox.go | 3 +++ store/store.go | 4 ++-- util/linkedlist/linkedlist.go | 5 +++++ 3 files changed, 10 insertions(+), 2 deletions(-) diff --git a/sandbox/sandbox.go b/sandbox/sandbox.go index 96268d20c..ce1d56d81 100644 --- a/sandbox/sandbox.go +++ b/sandbox/sandbox.go @@ -281,6 +281,9 @@ func (sb *sandbox) VerifyProof(blockHeight uint32, proof sortition.Proof, val *v defer sb.lk.RUnlock() seed := sb.store.SortitionSeed(blockHeight, sb.height+1) + if seed == nil { + return false + } return sortition.VerifyProof(*seed, proof, val.PublicKey(), sb.totalPower, val.Power()) } diff --git a/store/store.go b/store/store.go index 3431fdec1..9a76b3e53 100644 --- a/store/store.go +++ b/store/store.go @@ -206,8 +206,8 @@ func (s *store) BlockHash(height uint32) hash.Hash { } func (s *store) SortitionSeed(currentHeight, height uint32) *sortition.VerifiableSeed { - s.lk.RLock() - defer s.lk.RUnlock() + s.lk.Lock() + defer s.lk.Unlock() return s.blockStore.sortitionSeed(currentHeight, height) } diff --git a/util/linkedlist/linkedlist.go b/util/linkedlist/linkedlist.go index cd9fed740..4ebf89b57 100644 --- a/util/linkedlist/linkedlist.go +++ b/util/linkedlist/linkedlist.go @@ -29,6 +29,11 @@ func New[T any]() *LinkedList[T] { } } +func (l *LinkedList[T]) Get(index int) T { + values := l.Values() + return values[index] +} + // InsertAtHead inserts a new node at the head of the list. func (l *LinkedList[T]) InsertAtHead(data T) *Element[T] { newNode := NewElement(data) From b45cb1bbe1b4ee99698532aaa1960238dbc4129a Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Thu, 14 Dec 2023 14:26:14 +0330 Subject: [PATCH 08/30] fix: load sorition seed from cache for invalid proof --- store/block.go | 19 ++++++++++--------- store/store.go | 6 +++--- 2 files changed, 13 insertions(+), 12 deletions(-) diff --git a/store/block.go b/store/block.go index 8b1af8193..30b541d00 100644 --- a/store/block.go +++ b/store/block.go @@ -3,13 +3,13 @@ package store import ( "bytes" - "github.com/eapache/queue/v2" "github.com/pactus-project/pactus/crypto" "github.com/pactus-project/pactus/crypto/hash" "github.com/pactus-project/pactus/sortition" "github.com/pactus-project/pactus/types/block" "github.com/pactus-project/pactus/util" "github.com/pactus-project/pactus/util/encoding" + "github.com/pactus-project/pactus/util/linkedlist" "github.com/pactus-project/pactus/util/logger" "github.com/syndtr/goleveldb/leveldb" ) @@ -25,14 +25,14 @@ func blockHashKey(h hash.Hash) []byte { type blockStore struct { db *leveldb.DB - sortitionSeedQueue *queue.Queue[*sortition.VerifiableSeed] + sortitionSeedCache *linkedlist.LinkedList[*sortition.VerifiableSeed] sortitionInterval uint32 } func newBlockStore(db *leveldb.DB, sortitionInterval uint32) *blockStore { return &blockStore{ db: db, - sortitionSeedQueue: queue.New[*sortition.VerifiableSeed](), + sortitionSeedCache: linkedlist.New[*sortition.VerifiableSeed](), sortitionInterval: sortitionInterval, } } @@ -122,13 +122,14 @@ func (bs *blockStore) blockHeight(h hash.Hash) uint32 { return util.SliceToUint32(data) } -func (bs *blockStore) sortitionSeed(currentHeight, height uint32) *sortition.VerifiableSeed { - index := currentHeight - height +func (bs *blockStore) sortitionSeed(blockHeight, currentHeight uint32) *sortition.VerifiableSeed { + index := currentHeight - blockHeight if index > bs.sortitionInterval { return nil } - return bs.sortitionSeedQueue.Get(int(index)) + index = uint32(bs.sortitionSeedCache.Length()) - index + return bs.sortitionSeedCache.Get(int(index)) } func (bs *blockStore) hasBlock(height uint32) bool { @@ -140,8 +141,8 @@ func (bs *blockStore) hasPublicKey(addr crypto.Address) bool { } func (bs *blockStore) saveToCache(sortitionSeed sortition.VerifiableSeed) { - bs.sortitionSeedQueue.Add(&sortitionSeed) - if bs.sortitionSeedQueue.Length() > int(bs.sortitionInterval) { - bs.sortitionSeedQueue.Remove() + bs.sortitionSeedCache.InsertAtTail(&sortitionSeed) + if bs.sortitionSeedCache.Length() > int(bs.sortitionInterval) { + bs.sortitionSeedCache.DeleteAtHead() } } diff --git a/store/store.go b/store/store.go index 9a76b3e53..dde92d8af 100644 --- a/store/store.go +++ b/store/store.go @@ -111,7 +111,7 @@ func NewStore(conf *Config, transactionToLiveInterval, sortitionInterval uint32) startHeight = currentHeight - transactionToLiveInterval } - for i := startHeight; i < currentHeight; i++ { + for i := startHeight; i < currentHeight+1; i++ { committedBlock, err := s.Block(i) if err != nil { return nil, err @@ -205,11 +205,11 @@ func (s *store) BlockHash(height uint32) hash.Hash { return hash.UndefHash } -func (s *store) SortitionSeed(currentHeight, height uint32) *sortition.VerifiableSeed { +func (s *store) SortitionSeed(blockHeight, currentHeight uint32) *sortition.VerifiableSeed { s.lk.Lock() defer s.lk.Unlock() - return s.blockStore.sortitionSeed(currentHeight, height) + return s.blockStore.sortitionSeed(blockHeight, currentHeight) } func (s *store) PublicKey(addr crypto.Address) (*bls.PublicKey, error) { From c17a629895ab3ed529f9db0d13fef3b31e943a29 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Tue, 19 Dec 2023 22:32:50 +0330 Subject: [PATCH 09/30] chore: remove unused packages --- go.mod | 1 - go.sum | 2 -- node/node.go | 1 + 3 files changed, 1 insertion(+), 3 deletions(-) diff --git a/go.mod b/go.mod index d9f410578..2797d502f 100644 --- a/go.mod +++ b/go.mod @@ -3,7 +3,6 @@ module github.com/pactus-project/pactus go 1.21 require ( - github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 github.com/fxamacker/cbor/v2 v2.5.0 github.com/google/uuid v1.4.0 github.com/gorilla/handlers v1.5.1 diff --git a/go.sum b/go.sum index b00ad96f0..a678756bd 100644 --- a/go.sum +++ b/go.sum @@ -59,8 +59,6 @@ github.com/docker/go-units v0.4.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDD github.com/docker/go-units v0.5.0 h1:69rxXcBk27SvSaaxTtLh/8llcHD8vYHT7WSdRZ/jvr4= github.com/docker/go-units v0.5.0/go.mod h1:fgPhTUdO+D/Jk86RDLlptpiXQzgHJF7gydDDbaIK4Dk= github.com/dustin/go-humanize v1.0.0/go.mod h1:HtrtbFcZ19U5GC7JDqmcUSB87Iq5E25KnS6fMYU6eOk= -github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4 h1:8EXxF+tCLqaVk8AOC29zl2mnhQjwyLxxOTuhUazWRsg= -github.com/eapache/queue/v2 v2.0.0-20230407133247-75960ed334e4/go.mod h1:I5sHm0Y0T1u5YjlyqC5GVArM7aNZRUYtTjmJ8mPJFds= github.com/elastic/gosigar v0.12.0/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= github.com/elastic/gosigar v0.14.2 h1:Dg80n8cr90OZ7x+bAax/QjoW/XqTI11RmA79ZwIm9/4= github.com/elastic/gosigar v0.14.2/go.mod h1:iXRIGg2tLnu7LBdpqzyQfGDEidKCfWcCMS0WKyPWoMs= diff --git a/node/node.go b/node/node.go index 17e19f378..992415c95 100644 --- a/node/node.go +++ b/node/node.go @@ -155,6 +155,7 @@ func (n *Node) Stop() { } // these methods are using by GUI. + func (n *Node) ConsManager() consensus.ManagerReader { return n.consMgr } From 10a98e1688296330327dca6cf73c71bf2e2de0da Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Thu, 21 Dec 2023 18:49:58 +0330 Subject: [PATCH 10/30] feat: implement private configs for store --- cmd/cmd.go | 16 ++++++++-------- config/config.go | 26 +++++++++++++++++++++++--- node/node.go | 3 +-- store/account.go | 8 ++------ store/config.go | 12 +++++++++++- store/store.go | 17 ++++++++--------- 6 files changed, 53 insertions(+), 29 deletions(-) diff --git a/cmd/cmd.go b/cmd/cmd.go index 36626715d..0a711452b 100644 --- a/cmd/cmd.go +++ b/cmd/cmd.go @@ -293,7 +293,7 @@ func CreateNode(numValidators int, chain genesis.ChainType, workingDir string, if err := genDoc.SaveToFile(genPath); err != nil { return nil, nil, err } - conf := config.DefaultConfigTestnet() + conf := config.DefaultConfigTestnet(genDoc.Params()) if err := conf.Save(confPath); err != nil { return nil, nil, err } @@ -304,7 +304,7 @@ func CreateNode(numValidators int, chain genesis.ChainType, workingDir string, return nil, nil, err } - conf := config.DefaultConfigLocalnet() + conf := config.DefaultConfigLocalnet(genDoc.Params()) if err := conf.Save(confPath); err != nil { return nil, nil, err } @@ -338,7 +338,7 @@ func StartNode(workingDir string, passwordFetcher func(*wallet.Wallet) (string, } confPath := PactusConfigPath(workingDir) - conf, err := tryLoadConfig(gen.ChainType(), confPath) + conf, err := tryLoadConfig(gen, confPath) if err != nil { return nil, nil, err } @@ -461,15 +461,15 @@ func makeLocalGenesis(w wallet.Wallet) *genesis.Genesis { return gen } -func tryLoadConfig(chainType genesis.ChainType, confPath string) (*config.Config, error) { +func tryLoadConfig(genDoc *genesis.Genesis, confPath string) (*config.Config, error) { var defConf *config.Config - switch chainType { + switch genDoc.ChainType() { case genesis.Mainnet: panic("not yet implemented!") case genesis.Testnet: - defConf = config.DefaultConfigTestnet() + defConf = config.DefaultConfigTestnet(genDoc.Params()) case genesis.Localnet: - defConf = config.DefaultConfigLocalnet() + defConf = config.DefaultConfigLocalnet(genDoc.Params()) } conf, err := config.LoadFromFile(confPath, true, defConf) @@ -496,7 +496,7 @@ func tryLoadConfig(chainType genesis.ChainType, confPath string) (*config.Config } PrintSuccessMsgf("Config updated.") } else { - switch chainType { + switch genDoc.ChainType() { case genesis.Mainnet: err = config.SaveMainnetConfig(confPath) if err != nil { diff --git a/config/config.go b/config/config.go index 20a2a1548..f7edc86ea 100644 --- a/config/config.go +++ b/config/config.go @@ -3,6 +3,7 @@ package config import ( "bytes" _ "embed" + "github.com/pactus-project/pactus/types/param" "os" "github.com/pactus-project/pactus/consensus" @@ -77,14 +78,21 @@ func defaultConfig() *Config { return conf } -func DefaultConfigMainnet() *Config { +func DefaultConfigMainnet(genParams *param.Params) *Config { conf := defaultConfig() // TO BE DEFINED + + // Store private configs + conf.Store.TransactionToLiveInterval = genParams.TransactionToLiveInterval + conf.Store.SortitionInterval = genParams.SortitionInterval + conf.Store.AccountCacheSize = 1024 + conf.Store.PublicKeyCacheSize = 1024 + return conf } //nolint:lll // long multi-address -func DefaultConfigTestnet() *Config { +func DefaultConfigTestnet(genParams *param.Params) *Config { conf := defaultConfig() conf.Network.ListenAddrStrings = []string{ "/ip4/0.0.0.0/tcp/21777", "/ip4/0.0.0.0/udp/21777/quic-v1", @@ -122,10 +130,16 @@ func DefaultConfigTestnet() *Config { conf.Nanomsg.Enable = false conf.Nanomsg.Listen = "tcp://127.0.0.1:40799" + // Store private configs + conf.Store.TransactionToLiveInterval = genParams.TransactionToLiveInterval + conf.Store.SortitionInterval = genParams.SortitionInterval + conf.Store.AccountCacheSize = 1024 + conf.Store.PublicKeyCacheSize = 1024 + return conf } -func DefaultConfigLocalnet() *Config { +func DefaultConfigLocalnet(genParams *param.Params) *Config { conf := defaultConfig() conf.Network.ListenAddrStrings = []string{} conf.Network.EnableRelay = false @@ -144,6 +158,12 @@ func DefaultConfigLocalnet() *Config { conf.Nanomsg.Enable = true conf.Nanomsg.Listen = "tcp://127.0.0.1:0" + // Store private configs + conf.Store.TransactionToLiveInterval = genParams.TransactionToLiveInterval + conf.Store.SortitionInterval = genParams.SortitionInterval + conf.Store.AccountCacheSize = 1024 + conf.Store.PublicKeyCacheSize = 1024 + return conf } diff --git a/node/node.go b/node/node.go index 992415c95..2cf7ec5cb 100644 --- a/node/node.go +++ b/node/node.go @@ -61,8 +61,7 @@ func NewNode(genDoc *genesis.Genesis, conf *config.Config, txPool := txpool.NewTxPool(conf.TxPool, messageCh) // TODO implement dequeue for recent transaction - str, err := store.NewStore(conf.Store, - genDoc.Params().TransactionToLiveInterval, genDoc.Params().SortitionInterval) + str, err := store.NewStore(conf.Store) if err != nil { return nil, err } diff --git a/store/account.go b/store/account.go index 33eedc7f0..cd8d48ba9 100644 --- a/store/account.go +++ b/store/account.go @@ -9,10 +9,6 @@ import ( "github.com/syndtr/goleveldb/leveldb/util" ) -const ( - accLruCacheSize = 1024 -) - type accountStore struct { db *leveldb.DB addrCache *lru.Cache[crypto.Address, *account.Account] @@ -21,9 +17,9 @@ type accountStore struct { func accountKey(addr crypto.Address) []byte { return append(accountPrefix, addr.Bytes()...) } -func newAccountStore(db *leveldb.DB) *accountStore { +func newAccountStore(db *leveldb.DB, cacheSize int) *accountStore { total := int32(0) - addrLruCache, err := lru.New[crypto.Address, *account.Account](accLruCacheSize) + addrLruCache, err := lru.New[crypto.Address, *account.Account](cacheSize) if err != nil { logger.Panic("unable to create new instance of lru cache", "error", err) } diff --git a/store/config.go b/store/config.go index 20ec421ed..01d7945ed 100644 --- a/store/config.go +++ b/store/config.go @@ -10,11 +10,21 @@ import ( type Config struct { Path string `toml:"path"` + + // Private configs + TransactionToLiveInterval uint32 `toml:"-"` + SortitionInterval uint32 `toml:"-"` + AccountCacheSize int `toml:"-"` + PublicKeyCacheSize int `toml:"-"` } func DefaultConfig() *Config { return &Config{ - Path: "data", + Path: "data", + TransactionToLiveInterval: 8640, + SortitionInterval: 17, + AccountCacheSize: 1024, + PublicKeyCacheSize: 1024, } } diff --git a/store/store.go b/store/store.go index dde92d8af..a815d62e1 100644 --- a/store/store.go +++ b/store/store.go @@ -28,8 +28,7 @@ var ( ) const ( - lastStoreVersion = int32(1) - pubKeyLruCacheSize = 1024 + lastStoreVersion = int32(1) ) var ( @@ -74,13 +73,13 @@ type store struct { validatorStore *validatorStore } -func NewStore(conf *Config, transactionToLiveInterval, sortitionInterval uint32) (Store, error) { +func NewStore(conf *Config) (Store, error) { options := &opt.Options{ Strict: opt.DefaultStrict, Compression: opt.NoCompression, } - pubKeyCache, err := lru.New[crypto.Address, *bls.PublicKey](pubKeyLruCacheSize) + pubKeyCache, err := lru.New[crypto.Address, *bls.PublicKey](conf.PublicKeyCacheSize) if err != nil { return nil, err } @@ -94,9 +93,9 @@ func NewStore(conf *Config, transactionToLiveInterval, sortitionInterval uint32) db: db, pubKeyCache: pubKeyCache, batch: new(leveldb.Batch), - blockStore: newBlockStore(db, sortitionInterval), - txStore: newTxStore(db, transactionToLiveInterval), - accountStore: newAccountStore(db), + blockStore: newBlockStore(db, conf.SortitionInterval), + txStore: newTxStore(db, conf.TransactionToLiveInterval), + accountStore: newAccountStore(db, conf.AccountCacheSize), validatorStore: newValidatorStore(db), } @@ -107,8 +106,8 @@ func NewStore(conf *Config, transactionToLiveInterval, sortitionInterval uint32) currentHeight := lc.Height() startHeight := uint32(1) - if currentHeight > transactionToLiveInterval { - startHeight = currentHeight - transactionToLiveInterval + if currentHeight > conf.TransactionToLiveInterval { + startHeight = currentHeight - conf.TransactionToLiveInterval } for i := startHeight; i < currentHeight+1; i++ { From ac96375a27f7d3db7e8acfe6871bffec4d9ac910 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sat, 23 Dec 2023 23:55:44 +0330 Subject: [PATCH 11/30] test: change tests based in new store with cache --- config/config.go | 2 +- config/config_test.go | 11 ++++++----- node/node_test.go | 2 +- store/account_test.go | 4 ++-- store/config.go | 7 +++++++ store/store_test.go | 34 ++++++++++++++++++++++++++-------- store/tx.go | 7 +++---- store/validator_test.go | 6 +++--- tests/main_test.go | 4 ++-- util/encoding/encoding.go | 10 +++++----- 10 files changed, 56 insertions(+), 31 deletions(-) diff --git a/config/config.go b/config/config.go index f7edc86ea..bf48860fe 100644 --- a/config/config.go +++ b/config/config.go @@ -3,7 +3,6 @@ package config import ( "bytes" _ "embed" - "github.com/pactus-project/pactus/types/param" "os" "github.com/pactus-project/pactus/consensus" @@ -12,6 +11,7 @@ import ( "github.com/pactus-project/pactus/store" "github.com/pactus-project/pactus/sync" "github.com/pactus-project/pactus/txpool" + "github.com/pactus-project/pactus/types/param" "github.com/pactus-project/pactus/util" "github.com/pactus-project/pactus/util/errors" "github.com/pactus-project/pactus/util/logger" diff --git a/config/config_test.go b/config/config_test.go index 3eef8482b..79c9653aa 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -4,6 +4,7 @@ import ( "strings" "testing" + "github.com/pactus-project/pactus/types/param" "github.com/pactus-project/pactus/util" "github.com/pactus-project/pactus/util/testsuite" "github.com/stretchr/testify/assert" @@ -13,7 +14,7 @@ func TestSaveMainnetConfig(t *testing.T) { path := util.TempFilePath() assert.NoError(t, SaveMainnetConfig(path)) - defConf := DefaultConfigMainnet() + defConf := DefaultConfigMainnet(param.DefaultParams()) conf, err := LoadFromFile(path, true, defConf) assert.NoError(t, err) @@ -25,7 +26,7 @@ func TestSaveConfig(t *testing.T) { conf := defaultConfig() assert.NoError(t, conf.Save(path)) - defConf := DefaultConfigTestnet() + defConf := DefaultConfigTestnet(param.DefaultParams()) conf, err := LoadFromFile(path, true, defConf) assert.NoError(t, err) @@ -35,7 +36,7 @@ func TestSaveConfig(t *testing.T) { } func TestLocalnetConfig(t *testing.T) { - conf := DefaultConfigLocalnet() + conf := DefaultConfigLocalnet(param.DefaultParams()) assert.NoError(t, conf.BasicCheck()) assert.Empty(t, conf.Network.ListenAddrStrings) @@ -46,7 +47,7 @@ func TestLocalnetConfig(t *testing.T) { func TestLoadFromFile(t *testing.T) { path := util.TempFilePath() - defConf := DefaultConfigTestnet() + defConf := DefaultConfigTestnet(param.DefaultParams()) _, err := LoadFromFile(path, true, defConf) assert.Error(t, err, "not exists") @@ -73,7 +74,7 @@ func TestExampleConfig(t *testing.T) { } } - defaultConf := DefaultConfigMainnet() + defaultConf := DefaultConfigMainnet(param.DefaultParams()) defaultToml := string(defaultConf.toTOML()) exampleToml = strings.ReplaceAll(exampleToml, "##", "") diff --git a/node/node_test.go b/node/node_test.go index dbfccb14c..03d088076 100644 --- a/node/node_test.go +++ b/node/node_test.go @@ -30,7 +30,7 @@ func TestRunningNode(t *testing.T) { gen := genesis.MakeGenesis(util.Now(), map[crypto.Address]*account.Account{crypto.TreasuryAddress: acc}, []*validator.Validator{val}, param.DefaultParams()) - conf := config.DefaultConfigMainnet() + conf := config.DefaultConfigMainnet(param.DefaultParams()) conf.GRPC.Enable = false conf.HTTP.Enable = false conf.Store.Path = util.TempDirPath() diff --git a/store/account_test.go b/store/account_test.go index f775690c7..aa54b72f0 100644 --- a/store/account_test.go +++ b/store/account_test.go @@ -57,7 +57,7 @@ func TestAccountBatchSaving(t *testing.T) { t.Run("Close and load db", func(t *testing.T) { td.store.Close() - store, _ := NewStore(td.store.config, 8640, 17) + store, _ := NewStore(td.store.config) assert.Equal(t, store.TotalAccounts(), total) }) } @@ -93,7 +93,7 @@ func TestAccountByAddress(t *testing.T) { t.Run("Reopen the store", func(t *testing.T) { td.store.Close() - store, _ := NewStore(td.store.config, 8640, 17) + store, _ := NewStore(td.store.config) acc, err := store.Account(lastAddr) assert.NoError(t, err) diff --git a/store/config.go b/store/config.go index 01d7945ed..bf92bf873 100644 --- a/store/config.go +++ b/store/config.go @@ -41,5 +41,12 @@ func (conf *Config) BasicCheck() error { if !util.IsValidDirPath(conf.Path) { return errors.Errorf(errors.ErrInvalidConfig, "path is not valid") } + + if conf.AccountCacheSize == 0 || + conf.PublicKeyCacheSize == 0 || + conf.SortitionInterval == 0 || + conf.TransactionToLiveInterval == 0 { + return errors.Errorf(errors.ErrInvalidConfig, "private configs is not valid") + } return nil } diff --git a/store/store_test.go b/store/store_test.go index c39c62fc2..213f1f619 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -1,11 +1,13 @@ package store import ( + "log" "testing" "github.com/pactus-project/pactus/crypto/bls" "github.com/pactus-project/pactus/crypto/hash" "github.com/pactus-project/pactus/types/block" + "github.com/pactus-project/pactus/types/param" "github.com/pactus-project/pactus/types/tx" "github.com/pactus-project/pactus/util" "github.com/pactus-project/pactus/util/testsuite" @@ -23,11 +25,16 @@ func setup(t *testing.T) *testData { t.Helper() ts := testsuite.NewTestSuite(t) + params := param.DefaultParams() conf := &Config{ - Path: util.TempDirPath(), + Path: util.TempDirPath(), + TransactionToLiveInterval: params.TransactionToLiveInterval, + SortitionInterval: params.SortitionInterval, + AccountCacheSize: 1024, + PublicKeyCacheSize: 1024, } - s, err := NewStore(conf, 8640, 17) + s, err := NewStore(conf) require.NoError(t, err) td := &testData{ @@ -39,6 +46,12 @@ func setup(t *testing.T) *testData { for height := uint32(0); height < 10; height++ { blk, cert := td.GenerateTestBlock(height + 1) + if height == 9 { + for _, trx := range blk.Transactions() { + log.Println(trx.ID()) + } + } + td.store.SaveBlock(blk, cert) assert.NoError(t, td.store.WriteBatch()) } @@ -92,14 +105,19 @@ func TestRetrieveBlockAndTransactions(t *testing.T) { blk, _ := committedBlock.ToBlock() assert.Equal(t, blk.PrevCertificate().Height(), lastHeight-1) - for _, trx := range blk.Transactions() { + for i, trx := range blk.Transactions() { + if i == 0 { + continue + } + log.Printf("test number: %d", i) committedTx, err := td.store.Transaction(trx.ID()) - assert.NoError(t, err) - assert.Equal(t, blk.Header().UnixTime(), committedTx.BlockTime) - assert.Equal(t, trx.ID(), committedTx.TxID) - assert.Equal(t, lastHeight, committedTx.Height) + require.NoError(t, err) + require.Equal(t, blk.Header().UnixTime(), committedTx.BlockTime) + require.Equal(t, trx.ID(), committedTx.TxID) + require.Equal(t, lastHeight, committedTx.Height) trx2, _ := committedTx.ToTx() - assert.Equal(t, trx2.ID(), trx.ID()) + log.Printf("got: %v", trx2.ID()) + require.Equal(t, trx.ID(), trx2.ID()) } } diff --git a/store/tx.go b/store/tx.go index 016130eba..c83adec18 100644 --- a/store/tx.go +++ b/store/tx.go @@ -43,8 +43,8 @@ func (ts *txStore) saveTxs(batch *leveldb.Batch, txs block.Txs, regs []blockRegi } id := trx.ID() - txKey := txKey(id) - batch.Put(txKey, w.Bytes()) + key := txKey(id) + batch.Put(key, w.Bytes()) ts.saveToCache(id, reg.height) } } @@ -72,8 +72,7 @@ func (ts *txStore) tx(id tx.ID) (*blockRegion, error) { } r := bytes.NewReader(data) reg := new(blockRegion) - err = encoding.ReadElements(r, ®.height, ®.offset, ®.length) - if err != nil { + if err := encoding.ReadElements(r, ®.height, ®.offset, ®.length); err != nil { return nil, err } return reg, nil diff --git a/store/validator_test.go b/store/validator_test.go index f8bc5d1ce..aab47d47c 100644 --- a/store/validator_test.go +++ b/store/validator_test.go @@ -60,7 +60,7 @@ func TestValidatorBatchSaving(t *testing.T) { t.Run("Close and load db", func(t *testing.T) { td.store.Close() - store, _ := NewStore(td.store.config, 8640, 17) + store, _ := NewStore(td.store.config) assert.Equal(t, store.TotalValidators(), total) }) } @@ -116,7 +116,7 @@ func TestValidatorByNumber(t *testing.T) { t.Run("Reopen the store", func(t *testing.T) { td.store.Close() - store, _ := NewStore(td.store.config, 8640, 17) + store, _ := NewStore(td.store.config) num := td.RandInt32(total) val, err := store.ValidatorByNumber(num) @@ -160,7 +160,7 @@ func TestValidatorByAddress(t *testing.T) { t.Run("Reopen the store", func(t *testing.T) { td.store.Close() - store, _ := NewStore(td.store.config, 8640, 17) + store, _ := NewStore(td.store.config) num := td.RandInt32(total) val0, _ := store.ValidatorByNumber(num) diff --git a/tests/main_test.go b/tests/main_test.go index 1af8f5f9a..38fcf6b95 100644 --- a/tests/main_test.go +++ b/tests/main_test.go @@ -67,7 +67,7 @@ func TestMain(m *testing.M) { tValKeys[i][0] = bls.NewValidatorKey(key0) tValKeys[i][1] = bls.NewValidatorKey(key1) tValKeys[i][2] = bls.NewValidatorKey(key2) - tConfigs[i] = config.DefaultConfigMainnet() + tConfigs[i] = config.DefaultConfigMainnet(param.DefaultParams()) tConfigs[i].Store.Path = util.TempDirPath() tConfigs[i].Consensus.ChangeProposerTimeout = 4 * time.Second @@ -179,7 +179,7 @@ func TestMain(m *testing.M) { tNodes[i].Stop() } - s, _ := store.NewStore(tConfigs[tNodeIdx1].Store, 8640, 17) + s, _ := store.NewStore(tConfigs[tNodeIdx1].Store) total := int64(0) s.IterateAccounts(func(addr crypto.Address, acc *account.Account) bool { total += acc.Balance() diff --git a/util/encoding/encoding.go b/util/encoding/encoding.go index 8399435d6..07e2f461f 100644 --- a/util/encoding/encoding.go +++ b/util/encoding/encoding.go @@ -12,7 +12,7 @@ import ( const ( // MaxPayloadSize is the maximum bytes a message can be regardless of other // individual limits imposed by messages themselves. - MaxPayloadSize = (1024 * 1024 * 32) // 32MB + MaxPayloadSize = 1024 * 1024 * 32 // 32MB // binaryFreeListMaxItems is the number of buffers to keep in the free // list to use for binary serialization and deserialization. binaryFreeListMaxItems = 1024 @@ -157,7 +157,7 @@ func (l binaryFreeList) PutUint64(w io.Writer, val uint64) error { // deserializing primitive integer values to and from io.Readers and io.Writers. var binarySerializer binaryFreeList = make(chan []byte, binaryFreeListMaxItems) -// readElement reads the next sequence of bytes from r using little endian +// ReadElement reads the next sequence of bytes from r using little endian // depending on the concrete type of element pointed to. func ReadElement(r io.Reader, element interface{}) error { // Attempt to read the element based on the concrete type via fast @@ -207,7 +207,7 @@ func ReadElement(r io.Reader, element interface{}) error { return err } -// readElements reads multiple items from r. It is equivalent to multiple +// ReadElements reads multiple items from r. It is equivalent to multiple // calls to readElement. func ReadElements(r io.Reader, elements ...interface{}) error { for _, element := range elements { @@ -219,7 +219,7 @@ func ReadElements(r io.Reader, elements ...interface{}) error { return nil } -// writeElement writes the little endian representation of element to w. +// WriteElement writes the little endian representation of element to w. func WriteElement(w io.Writer, element interface{}) error { // Attempt to write the element based on the concrete type via fast // type assertions first. @@ -258,7 +258,7 @@ func WriteElement(w io.Writer, element interface{}) error { return err } -// writeElements writes multiple items to w. It is equivalent to multiple +// WriteElements writes multiple items to w. It is equivalent to multiple // calls to writeElement. func WriteElements(w io.Writer, elements ...interface{}) error { for _, element := range elements { From fd5b778b4d30ccc7843519a116c31a305357589d Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sun, 24 Dec 2023 00:42:32 +0330 Subject: [PATCH 12/30] chore: remove unused variables etc. --- store/store_test.go | 25 ++++++------------------- 1 file changed, 6 insertions(+), 19 deletions(-) diff --git a/store/store_test.go b/store/store_test.go index 213f1f619..814cf13be 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -1,7 +1,6 @@ package store import ( - "log" "testing" "github.com/pactus-project/pactus/crypto/bls" @@ -45,13 +44,6 @@ func setup(t *testing.T) *testData { // Save 10 blocks for height := uint32(0); height < 10; height++ { blk, cert := td.GenerateTestBlock(height + 1) - - if height == 9 { - for _, trx := range blk.Transactions() { - log.Println(trx.ID()) - } - } - td.store.SaveBlock(blk, cert) assert.NoError(t, td.store.WriteBatch()) } @@ -105,19 +97,14 @@ func TestRetrieveBlockAndTransactions(t *testing.T) { blk, _ := committedBlock.ToBlock() assert.Equal(t, blk.PrevCertificate().Height(), lastHeight-1) - for i, trx := range blk.Transactions() { - if i == 0 { - continue - } - log.Printf("test number: %d", i) + for _, trx := range blk.Transactions() { committedTx, err := td.store.Transaction(trx.ID()) - require.NoError(t, err) - require.Equal(t, blk.Header().UnixTime(), committedTx.BlockTime) - require.Equal(t, trx.ID(), committedTx.TxID) - require.Equal(t, lastHeight, committedTx.Height) + assert.NoError(t, err) + assert.Equal(t, blk.Header().UnixTime(), committedTx.BlockTime) + assert.Equal(t, trx.ID(), committedTx.TxID) + assert.Equal(t, lastHeight, committedTx.Height) trx2, _ := committedTx.ToTx() - log.Printf("got: %v", trx2.ID()) - require.Equal(t, trx.ID(), trx2.ID()) + assert.Equal(t, trx.ID(), trx2.ID()) } } From 0260d886a0ba6905f4813d73c1e45ea01779dc01 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sun, 24 Dec 2023 01:01:31 +0330 Subject: [PATCH 13/30] test: fix save txs write buffer --- store/tx.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/store/tx.go b/store/tx.go index c83adec18..cc15a597c 100644 --- a/store/tx.go +++ b/store/tx.go @@ -33,9 +33,9 @@ func newTxStore(db *leveldb.DB, ttl uint32) *txStore { } func (ts *txStore) saveTxs(batch *leveldb.Batch, txs block.Txs, regs []blockRegion) { - w := bytes.NewBuffer(make([]byte, 0, 32+4)) - for i, trx := range txs { + w := bytes.NewBuffer(make([]byte, 0, 32+4)) + reg := regs[i] err := encoding.WriteElements(w, ®.height, ®.offset, ®.length) if err != nil { From 7d0ea222dd4e65183be803f236c5c90766fef96c Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sun, 24 Dec 2023 23:36:38 +0330 Subject: [PATCH 14/30] fix: race condition, dead lock and out of range index in sortiotion seed --- store/block.go | 7 ++++++- store/mock.go | 9 ++++++--- tests/main_test.go | 2 +- 3 files changed, 13 insertions(+), 5 deletions(-) diff --git a/store/block.go b/store/block.go index 30b541d00..e93e611c5 100644 --- a/store/block.go +++ b/store/block.go @@ -128,7 +128,12 @@ func (bs *blockStore) sortitionSeed(blockHeight, currentHeight uint32) *sortitio return nil } - index = uint32(bs.sortitionSeedCache.Length()) - index + if index != 0 { + index = uint32(bs.sortitionSeedCache.Length()) - index + } else { + index = uint32(bs.sortitionSeedCache.Length()) - 1 + } + return bs.sortitionSeedCache.Get(int(index)) } diff --git a/store/mock.go b/store/mock.go index 99cd6dc56..e47f5135f 100644 --- a/store/mock.go +++ b/store/mock.go @@ -66,9 +66,12 @@ func (m *MockStore) BlockHeight(h hash.Hash) uint32 { return 0 } -func (m *MockStore) SortitionSeed(_, _ uint32) *sortition.VerifiableSeed { - // TODO implement me - panic("implement me") +func (m *MockStore) SortitionSeed(blockHeight, _ uint32) *sortition.VerifiableSeed { + if blk, ok := m.Blocks[blockHeight]; ok { + sortitionSeed := blk.Header().SortitionSeed() + return &sortitionSeed + } + return nil } func (m *MockStore) PublicKey(addr crypto.Address) (*bls.PublicKey, error) { diff --git a/tests/main_test.go b/tests/main_test.go index 38fcf6b95..71d5596c3 100644 --- a/tests/main_test.go +++ b/tests/main_test.go @@ -173,7 +173,7 @@ func TestMain(m *testing.M) { panic("Sortition didn't work") } - // Let's shutdown the nodes + // Lets shutdown the nodes tCtx.Done() for i := 0; i < tTotalNodes; i++ { tNodes[i].Stop() From aaaeeb6637924eb875037dfbec08517f1c5a2c64 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Wed, 27 Dec 2023 01:47:29 +0330 Subject: [PATCH 15/30] perf: change linkedlist to slice for better performance --- store/block.go | 17 ++++++++--------- store/tx.go | 16 ++++++++-------- util/linkedlist/linkedlist.go | 5 ----- 3 files changed, 16 insertions(+), 22 deletions(-) diff --git a/store/block.go b/store/block.go index e93e611c5..1b11e1eff 100644 --- a/store/block.go +++ b/store/block.go @@ -9,7 +9,6 @@ import ( "github.com/pactus-project/pactus/types/block" "github.com/pactus-project/pactus/util" "github.com/pactus-project/pactus/util/encoding" - "github.com/pactus-project/pactus/util/linkedlist" "github.com/pactus-project/pactus/util/logger" "github.com/syndtr/goleveldb/leveldb" ) @@ -25,14 +24,14 @@ func blockHashKey(h hash.Hash) []byte { type blockStore struct { db *leveldb.DB - sortitionSeedCache *linkedlist.LinkedList[*sortition.VerifiableSeed] + sortitionSeedCache []*sortition.VerifiableSeed sortitionInterval uint32 } func newBlockStore(db *leveldb.DB, sortitionInterval uint32) *blockStore { return &blockStore{ db: db, - sortitionSeedCache: linkedlist.New[*sortition.VerifiableSeed](), + sortitionSeedCache: make([]*sortition.VerifiableSeed, 0, sortitionInterval), sortitionInterval: sortitionInterval, } } @@ -129,12 +128,12 @@ func (bs *blockStore) sortitionSeed(blockHeight, currentHeight uint32) *sortitio } if index != 0 { - index = uint32(bs.sortitionSeedCache.Length()) - index + index = uint32(len(bs.sortitionSeedCache)) - index } else { - index = uint32(bs.sortitionSeedCache.Length()) - 1 + index = uint32(len(bs.sortitionSeedCache)) - 1 } - return bs.sortitionSeedCache.Get(int(index)) + return bs.sortitionSeedCache[index] } func (bs *blockStore) hasBlock(height uint32) bool { @@ -146,8 +145,8 @@ func (bs *blockStore) hasPublicKey(addr crypto.Address) bool { } func (bs *blockStore) saveToCache(sortitionSeed sortition.VerifiableSeed) { - bs.sortitionSeedCache.InsertAtTail(&sortitionSeed) - if bs.sortitionSeedCache.Length() > int(bs.sortitionInterval) { - bs.sortitionSeedCache.DeleteAtHead() + bs.sortitionSeedCache = append(bs.sortitionSeedCache, &sortitionSeed) + if len(bs.sortitionSeedCache) > int(bs.sortitionInterval) { + bs.sortitionSeedCache = bs.sortitionSeedCache[1:] } } diff --git a/store/tx.go b/store/tx.go index cc15a597c..1c7c7a8de 100644 --- a/store/tx.go +++ b/store/tx.go @@ -19,16 +19,16 @@ type blockRegion struct { func txKey(id tx.ID) []byte { return append(txPrefix, id.Bytes()...) } type txStore struct { - db *leveldb.DB - txCache *linkedmap.LinkedMap[tx.ID, uint32] - ttl uint32 + db *leveldb.DB + txCache *linkedmap.LinkedMap[tx.ID, uint32] + txCacheSize uint32 } -func newTxStore(db *leveldb.DB, ttl uint32) *txStore { +func newTxStore(db *leveldb.DB, txCacheSize uint32) *txStore { return &txStore{ - db: db, - txCache: linkedmap.NewLinkedMap[tx.ID, uint32](0), - ttl: ttl, + db: db, + txCache: linkedmap.NewLinkedMap[tx.ID, uint32](0), + txCacheSize: txCacheSize, } } @@ -54,7 +54,7 @@ func (ts *txStore) pruneCache(currentHeight uint32) { head := ts.txCache.HeadNode() txHeight := head.Data.Value - if currentHeight-txHeight <= ts.ttl { + if currentHeight-txHeight <= ts.txCacheSize { break } ts.txCache.RemoveHead() diff --git a/util/linkedlist/linkedlist.go b/util/linkedlist/linkedlist.go index 4ebf89b57..cd9fed740 100644 --- a/util/linkedlist/linkedlist.go +++ b/util/linkedlist/linkedlist.go @@ -29,11 +29,6 @@ func New[T any]() *LinkedList[T] { } } -func (l *LinkedList[T]) Get(index int) T { - values := l.Values() - return values[index] -} - // InsertAtHead inserts a new node at the head of the list. func (l *LinkedList[T]) InsertAtHead(data T) *Element[T] { newNode := NewElement(data) From 887eeca9a88582badb0161df620bf9cdcd4bfb9e Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Wed, 27 Dec 2023 01:47:59 +0330 Subject: [PATCH 16/30] test: testing sortition seed cache --- store/block_test.go | 48 ++++++++++++++++++++++++++++++++++++++------- 1 file changed, 41 insertions(+), 7 deletions(-) diff --git a/store/block_test.go b/store/block_test.go index f8469d77d..33c61c4d4 100644 --- a/store/block_test.go +++ b/store/block_test.go @@ -4,6 +4,7 @@ import ( "bytes" "testing" + "github.com/pactus-project/pactus/sortition" "github.com/stretchr/testify/assert" ) @@ -12,8 +13,8 @@ func TestBlockStore(t *testing.T) { lastCert := td.store.LastCertificate() lastHeight := lastCert.Height() - nextBlk, nextCrert := td.GenerateTestBlock(lastHeight + 1) - nextNextBlk, nextNextCrert := td.GenerateTestBlock(lastHeight + 2) + nextBlk, nextCert := td.GenerateTestBlock(lastHeight + 1) + nextNextBlk, nextNextCert := td.GenerateTestBlock(lastHeight + 2) t.Run("Missed block, Should panic ", func(t *testing.T) { defer func() { @@ -21,18 +22,18 @@ func TestBlockStore(t *testing.T) { t.Errorf("The code did not panic") } }() - td.store.SaveBlock(nextNextBlk, nextNextCrert) + td.store.SaveBlock(nextNextBlk, nextNextCert) }) t.Run("Add block, don't batch write", func(t *testing.T) { - td.store.SaveBlock(nextBlk, nextCrert) + td.store.SaveBlock(nextBlk, nextCert) b2, err := td.store.Block(lastHeight + 1) assert.Error(t, err) assert.Nil(t, b2) }) t.Run("Add block, batch write", func(t *testing.T) { - td.store.SaveBlock(nextBlk, nextCrert) + td.store.SaveBlock(nextBlk, nextCert) assert.NoError(t, td.store.WriteBatch()) committedBlock, err := td.store.Block(lastHeight + 1) @@ -44,7 +45,7 @@ func TestBlockStore(t *testing.T) { cert := td.store.LastCertificate() assert.NoError(t, err) - assert.Equal(t, cert.Hash(), nextCrert.Hash()) + assert.Equal(t, cert.Hash(), nextCert.Hash()) }) t.Run("Duplicated block, Should panic ", func(t *testing.T) { @@ -53,6 +54,39 @@ func TestBlockStore(t *testing.T) { t.Errorf("The code did not panic") } }() - td.store.SaveBlock(nextBlk, nextCrert) + td.store.SaveBlock(nextBlk, nextCert) }) + + t.Run("Should not be old sortition seed in cache plus check last sortition seed be in cache", + func(t *testing.T) { + var oldSortitionSeed sortition.VerifiableSeed + var lastSortitionSeed sortition.VerifiableSeed + lastHeight = td.store.LastCertificate().Height() + generateBlkCount := 100 + + for i := 0; i <= generateBlkCount; i++ { + lastHeight++ + nNextBlk, nNextCert := td.GenerateTestBlock(lastHeight) + + td.store.SaveBlock(nNextBlk, nNextCert) + assert.NoError(t, td.store.WriteBatch()) + + if i == 0 { + oldSortitionSeed = nNextBlk.Header().SortitionSeed() + } + + if i == generateBlkCount { + lastSortitionSeed = nNextBlk.Header().SortitionSeed() + } + } + + // check old sortition seed doesn't exist in cache + for _, seed := range td.store.blockStore.sortitionSeedCache { + assert.NotEqual(t, &oldSortitionSeed, seed) + } + + // last sortition seed should exist at last index of cache + assert.Equal(t, &lastSortitionSeed, td.store.blockStore. + sortitionSeedCache[len(td.store.blockStore.sortitionSeedCache)-1]) + }) } From e86ac4648f43c21980305f63af15be8b82488e83 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Thu, 28 Dec 2023 15:03:04 +0330 Subject: [PATCH 17/30] test: get more coverage for linkedmap --- go.mod | 2 +- util/linkedmap/linkedmap.go | 16 +++++++------- util/linkedmap/linkedmap_test.go | 37 +++++++++++++++++++++++++++++--- 3 files changed, 43 insertions(+), 12 deletions(-) diff --git a/go.mod b/go.mod index a49775a5a..1004628b7 100644 --- a/go.mod +++ b/go.mod @@ -4,6 +4,7 @@ go 1.21 require ( github.com/fxamacker/cbor/v2 v2.5.0 + github.com/gofrs/flock v0.8.1 github.com/google/uuid v1.4.0 github.com/gorilla/handlers v1.5.1 github.com/gorilla/mux v1.8.0 @@ -53,7 +54,6 @@ require ( github.com/go-logr/stdr v1.2.2 // indirect github.com/go-task/slim-sprig v0.0.0-20230315185526-52ccab3ef572 // indirect github.com/godbus/dbus/v5 v5.1.0 // indirect - github.com/gofrs/flock v0.8.1 // indirect github.com/gogo/protobuf v1.3.2 // indirect github.com/golang/protobuf v1.5.3 // indirect github.com/golang/snappy v0.0.4 // indirect diff --git a/util/linkedmap/linkedmap.go b/util/linkedmap/linkedmap.go index 29827e0db..3257e22d8 100644 --- a/util/linkedmap/linkedmap.go +++ b/util/linkedmap/linkedmap.go @@ -88,10 +88,7 @@ func (lm *LinkedMap[K, V]) TailNode() *ll.Element[Pair[K, V]] { } func (lm *LinkedMap[K, V]) RemoveTail() { - tail := lm.list.Tail - key := tail.Data.Key - lm.list.Delete(tail) - delete(lm.hashmap, key) + lm.remove(lm.list.Tail, lm.list.Tail.Data.Key) } // HeadNode returns the LinkNode at the beginning (head) of the LinkedMap. @@ -104,10 +101,7 @@ func (lm *LinkedMap[K, V]) HeadNode() *ll.Element[Pair[K, V]] { } func (lm *LinkedMap[K, V]) RemoveHead() { - head := lm.list.Head - key := head.Data.Key - lm.list.Delete(head) - delete(lm.hashmap, key) + lm.remove(lm.list.Head, lm.list.Head.Data.Key) } // Remove removes the key-value pair with the specified key from the LinkedMap. @@ -121,6 +115,12 @@ func (lm *LinkedMap[K, V]) Remove(key K) bool { return found } +// remove removes the key-value pair with the specified key from the LinkedMap and linkedlist.LinkedList. +func (lm *LinkedMap[K, V]) remove(element *ll.Element[Pair[K, V]], key K) { + lm.list.Delete(element) + delete(lm.hashmap, key) +} + // Empty checks if the LinkedMap is empty (contains no key-value pairs). func (lm *LinkedMap[K, V]) Empty() bool { return lm.Size() == 0 diff --git a/util/linkedmap/linkedmap_test.go b/util/linkedmap/linkedmap_test.go index 48f68956b..17910536b 100644 --- a/util/linkedmap/linkedmap_test.go +++ b/util/linkedmap/linkedmap_test.go @@ -55,6 +55,28 @@ func TestLinkedMap(t *testing.T) { assert.False(t, lm.Remove(2)) }) + t.Run("Test RemoveTail", func(t *testing.T) { + lm := NewLinkedMap[int, string](4) + lm.PushBack(0, "-") + lm.PushBack(1, "a") + lm.PushBack(2, "b") + + lm.RemoveTail() + assert.Equal(t, lm.TailNode().Data.Value, "a") + assert.NotEqual(t, lm.TailNode().Data.Value, "b") + }) + + t.Run("Test RemoveHead", func(t *testing.T) { + lm := NewLinkedMap[int, string](4) + lm.PushBack(0, "-") + lm.PushBack(1, "a") + lm.PushBack(2, "b") + + lm.RemoveHead() + assert.Equal(t, lm.HeadNode().Data.Value, "a") + assert.NotEqual(t, lm.HeadNode().Data.Value, "-") + }) + t.Run("Should updates v", func(t *testing.T) { lm := NewLinkedMap[int, string](4) lm.PushBack(1, "a") @@ -180,7 +202,16 @@ func TestLinkedMap(t *testing.T) { } func TestCapacity(t *testing.T) { - capacity := 100 - lm := NewLinkedMap[int, string](capacity) - assert.Equal(t, lm.Capacity(), capacity) + t.Run("Check Capacity", func(t *testing.T) { + capacity := 100 + lm := NewLinkedMap[int, string](capacity) + assert.Equal(t, lm.Capacity(), capacity) + }) + + t.Run("No Capacity", func(t *testing.T) { + capacity := 0 + lm := NewLinkedMap[int, string](capacity) + lm.prune() + assert.Equal(t, lm.Capacity(), capacity) + }) } From cffe24f2a5040496e642e1507193d995be45da17 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Thu, 28 Dec 2023 15:20:10 +0330 Subject: [PATCH 18/30] fix: zero capacity for test linkedmap --- util/linkedmap/linkedmap_test.go | 196 ++++++++++++++++++++++++++++++- 1 file changed, 191 insertions(+), 5 deletions(-) diff --git a/util/linkedmap/linkedmap_test.go b/util/linkedmap/linkedmap_test.go index 17910536b..3dd8e453e 100644 --- a/util/linkedmap/linkedmap_test.go +++ b/util/linkedmap/linkedmap_test.go @@ -208,10 +208,196 @@ func TestCapacity(t *testing.T) { assert.Equal(t, lm.Capacity(), capacity) }) - t.Run("No Capacity", func(t *testing.T) { - capacity := 0 - lm := NewLinkedMap[int, string](capacity) - lm.prune() - assert.Equal(t, lm.Capacity(), capacity) + t.Run("Test FirstNode with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + assert.Nil(t, lm.HeadNode()) + + lm.PushFront(3, "c") + lm.PushFront(2, "b") + lm.PushFront(1, "a") + + assert.Equal(t, lm.HeadNode().Data.Key, 1) + assert.Equal(t, lm.HeadNode().Data.Value, "a") + }) + + t.Run("Test LastNode with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + assert.Nil(t, lm.TailNode()) + + lm.PushBack(1, "a") + lm.PushBack(2, "b") + lm.PushBack(3, "c") + + assert.Equal(t, lm.TailNode().Data.Key, 3) + assert.Equal(t, lm.TailNode().Data.Value, "c") + }) + + t.Run("Test Get with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + + lm.PushBack(2, "b") + lm.PushBack(1, "a") + + n := lm.GetNode(2) + assert.Equal(t, n.Data.Key, 2) + assert.Equal(t, n.Data.Value, "b") + + n = lm.GetNode(5) + assert.Nil(t, n) + }) + + t.Run("Test Remove with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + + lm.PushBack(0, "-") + lm.PushBack(2, "b") + lm.PushBack(1, "a") + assert.True(t, lm.Remove(2)) + assert.False(t, lm.Remove(2)) + }) + + t.Run("Test RemoveTail with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + lm.PushBack(0, "-") + lm.PushBack(1, "a") + lm.PushBack(2, "b") + + lm.RemoveTail() + assert.Equal(t, lm.TailNode().Data.Value, "a") + assert.NotEqual(t, lm.TailNode().Data.Value, "b") + }) + + t.Run("Test RemoveHead with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + lm.PushBack(0, "-") + lm.PushBack(1, "a") + lm.PushBack(2, "b") + + lm.RemoveHead() + assert.Equal(t, lm.HeadNode().Data.Value, "a") + assert.NotEqual(t, lm.HeadNode().Data.Value, "-") + }) + + t.Run("Should updates v with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + lm.PushBack(1, "a") + + lm.PushBack(1, "b") + n := lm.GetNode(1) + assert.Equal(t, n.Data.Key, 1) + assert.Equal(t, n.Data.Value, "b") + + lm.PushFront(1, "c") + n = lm.GetNode(1) + assert.Equal(t, n.Data.Key, 1) + assert.Equal(t, n.Data.Value, "c") + }) + + t.Run("Should not prunes oldest item with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + + lm.PushBack(1, "a") + lm.PushBack(2, "b") + lm.PushBack(3, "c") + lm.PushBack(4, "d") + + n := lm.GetNode(1) + assert.Equal(t, n.Data.Key, 1) + assert.Equal(t, n.Data.Value, "a") + + lm.PushBack(5, "e") + + n = lm.GetNode(1) + assert.NotNil(t, n) + }) + + t.Run("Should prunes by changing capacity with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + + lm.PushBack(1, "a") + lm.PushBack(2, "b") + lm.PushBack(3, "c") + lm.PushBack(4, "d") + + lm.SetCapacity(6) + + n := lm.GetNode(2) + assert.Equal(t, n.Data.Key, 2) + assert.Equal(t, n.Data.Value, "b") + + lm.SetCapacity(2) + assert.True(t, lm.Full()) + + n = lm.GetNode(2) + assert.Nil(t, n) + }) + + t.Run("Test PushBack and should not prune with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + + lm.PushBack(1, "a") // This item should be pruned + lm.PushBack(2, "b") + lm.PushBack(3, "c") + lm.PushBack(4, "d") + + n := lm.TailNode() + assert.Equal(t, n.Data.Key, 4) + assert.Equal(t, n.Data.Value, "d") + }) + + t.Run("Test PushFront and prune with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + + lm.PushFront(1, "a") + lm.PushFront(2, "b") + lm.PushFront(3, "c") + lm.PushFront(4, "d") // This item should be pruned + + n := lm.TailNode() + assert.Equal(t, n.Data.Key, 1) + assert.Equal(t, n.Data.Value, "a") + }) + + t.Run("Delete first with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + + lm.PushBack(1, "a") + lm.PushBack(2, "b") + lm.PushBack(3, "c") + + lm.Remove(1) + + assert.Equal(t, lm.HeadNode().Data.Key, 2) + assert.Equal(t, lm.HeadNode().Data.Value, "b") + }) + + t.Run("Delete last with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + + lm.PushBack(1, "a") + lm.PushBack(2, "b") + lm.PushBack(3, "c") + + lm.Remove(3) + + assert.Equal(t, lm.TailNode().Data.Key, 2) + assert.Equal(t, lm.TailNode().Data.Value, "b") + }) + + t.Run("Test Has function with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + + lm.PushBack(1, "a") + + assert.True(t, lm.Has(1)) + assert.False(t, lm.Has(2)) + }) + + t.Run("Test Clear with Zero Capacity", func(t *testing.T) { + lm := NewLinkedMap[int, string](0) + + lm.PushBack(1, "a") + lm.Clear() + assert.True(t, lm.Empty()) }) } From 459134948e83285ee759c1c8160c32b783bb8a92 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Thu, 28 Dec 2023 15:33:02 +0330 Subject: [PATCH 19/30] test: get more coverage from config/config.go --- config/config.go | 12 ++++++------ config/config_test.go | 35 +++++++++++++++++++++++++++++++++++ store/block.go | 6 +++--- store/config.go | 22 +++++++++++----------- store/store.go | 8 ++++---- store/store_test.go | 10 +++++----- 6 files changed, 64 insertions(+), 29 deletions(-) diff --git a/config/config.go b/config/config.go index bf48860fe..bce030788 100644 --- a/config/config.go +++ b/config/config.go @@ -83,8 +83,8 @@ func DefaultConfigMainnet(genParams *param.Params) *Config { // TO BE DEFINED // Store private configs - conf.Store.TransactionToLiveInterval = genParams.TransactionToLiveInterval - conf.Store.SortitionInterval = genParams.SortitionInterval + conf.Store.TxCacheSize = genParams.TransactionToLiveInterval + conf.Store.SortitionCacheSize = genParams.SortitionInterval conf.Store.AccountCacheSize = 1024 conf.Store.PublicKeyCacheSize = 1024 @@ -131,8 +131,8 @@ func DefaultConfigTestnet(genParams *param.Params) *Config { conf.Nanomsg.Listen = "tcp://127.0.0.1:40799" // Store private configs - conf.Store.TransactionToLiveInterval = genParams.TransactionToLiveInterval - conf.Store.SortitionInterval = genParams.SortitionInterval + conf.Store.TxCacheSize = genParams.TransactionToLiveInterval + conf.Store.SortitionCacheSize = genParams.SortitionInterval conf.Store.AccountCacheSize = 1024 conf.Store.PublicKeyCacheSize = 1024 @@ -159,8 +159,8 @@ func DefaultConfigLocalnet(genParams *param.Params) *Config { conf.Nanomsg.Listen = "tcp://127.0.0.1:0" // Store private configs - conf.Store.TransactionToLiveInterval = genParams.TransactionToLiveInterval - conf.Store.SortitionInterval = genParams.SortitionInterval + conf.Store.TxCacheSize = genParams.TransactionToLiveInterval + conf.Store.SortitionCacheSize = genParams.SortitionInterval conf.Store.AccountCacheSize = 1024 conf.Store.PublicKeyCacheSize = 1024 diff --git a/config/config_test.go b/config/config_test.go index 79c9653aa..348c4250a 100644 --- a/config/config_test.go +++ b/config/config_test.go @@ -19,6 +19,10 @@ func TestSaveMainnetConfig(t *testing.T) { assert.NoError(t, err) assert.NoError(t, conf.BasicCheck()) + assert.Equal(t, conf.Store.TxCacheSize, param.DefaultParams().TransactionToLiveInterval) + assert.Equal(t, conf.Store.SortitionCacheSize, param.DefaultParams().SortitionInterval) + assert.Equal(t, conf.Store.AccountCacheSize, 1024) + assert.Equal(t, conf.Store.PublicKeyCacheSize, 1024) } func TestSaveConfig(t *testing.T) { @@ -33,6 +37,10 @@ func TestSaveConfig(t *testing.T) { assert.NoError(t, conf.BasicCheck()) assert.Equal(t, conf.Network.NetworkName, "pactus-testnet-v2") assert.Equal(t, conf.Network.DefaultPort, 21777) + assert.Equal(t, conf.Store.TxCacheSize, param.DefaultParams().TransactionToLiveInterval) + assert.Equal(t, conf.Store.SortitionCacheSize, param.DefaultParams().SortitionInterval) + assert.Equal(t, conf.Store.AccountCacheSize, 1024) + assert.Equal(t, conf.Store.PublicKeyCacheSize, 1024) } func TestLocalnetConfig(t *testing.T) { @@ -43,6 +51,25 @@ func TestLocalnetConfig(t *testing.T) { assert.Empty(t, conf.Network.RelayAddrStrings) assert.Equal(t, conf.Network.NetworkName, "pactus-localnet") assert.Equal(t, conf.Network.DefaultPort, 21666) + assert.Equal(t, conf.Store.TxCacheSize, param.DefaultParams().TransactionToLiveInterval) + assert.Equal(t, conf.Store.SortitionCacheSize, param.DefaultParams().SortitionInterval) + assert.Equal(t, conf.Store.AccountCacheSize, 1024) + assert.Equal(t, conf.Store.PublicKeyCacheSize, 1024) +} + +func TestTestnetConfig(t *testing.T) { + conf := DefaultConfigTestnet(param.DefaultParams()) + + assert.NoError(t, conf.BasicCheck()) + assert.NotEmpty(t, conf.Network.ListenAddrStrings) + assert.NotEmpty(t, conf.Network.DefaultRelayAddrStrings) + assert.Empty(t, conf.Network.RelayAddrStrings) + assert.Equal(t, conf.Network.NetworkName, "pactus-testnet-v2") + assert.Equal(t, conf.Network.DefaultPort, 21777) + assert.Equal(t, conf.Store.TxCacheSize, param.DefaultParams().TransactionToLiveInterval) + assert.Equal(t, conf.Store.SortitionCacheSize, param.DefaultParams().SortitionInterval) + assert.Equal(t, conf.Store.AccountCacheSize, 1024) + assert.Equal(t, conf.Store.PublicKeyCacheSize, 1024) } func TestLoadFromFile(t *testing.T) { @@ -59,6 +86,10 @@ func TestLoadFromFile(t *testing.T) { conf, err := LoadFromFile(path, false, defConf) assert.NoError(t, err) assert.Equal(t, conf, defConf) + assert.Equal(t, conf.Store.TxCacheSize, param.DefaultParams().TransactionToLiveInterval) + assert.Equal(t, conf.Store.SortitionCacheSize, param.DefaultParams().SortitionInterval) + assert.Equal(t, conf.Store.AccountCacheSize, 1024) + assert.Equal(t, conf.Store.PublicKeyCacheSize, 1024) } func TestExampleConfig(t *testing.T) { @@ -83,6 +114,10 @@ func TestExampleConfig(t *testing.T) { defaultToml = strings.ReplaceAll(defaultToml, "\n\n", "\n") assert.Equal(t, defaultToml, exampleToml) + assert.Equal(t, defaultConf.Store.TxCacheSize, param.DefaultParams().TransactionToLiveInterval) + assert.Equal(t, defaultConf.Store.SortitionCacheSize, param.DefaultParams().SortitionInterval) + assert.Equal(t, defaultConf.Store.AccountCacheSize, 1024) + assert.Equal(t, defaultConf.Store.PublicKeyCacheSize, 1024) } func TestNodeConfigBasicCheck(t *testing.T) { diff --git a/store/block.go b/store/block.go index 1b11e1eff..8be811d69 100644 --- a/store/block.go +++ b/store/block.go @@ -28,11 +28,11 @@ type blockStore struct { sortitionInterval uint32 } -func newBlockStore(db *leveldb.DB, sortitionInterval uint32) *blockStore { +func newBlockStore(db *leveldb.DB, sortitionCacheSize uint32) *blockStore { return &blockStore{ db: db, - sortitionSeedCache: make([]*sortition.VerifiableSeed, 0, sortitionInterval), - sortitionInterval: sortitionInterval, + sortitionSeedCache: make([]*sortition.VerifiableSeed, 0, sortitionCacheSize), + sortitionInterval: sortitionCacheSize, } } diff --git a/store/config.go b/store/config.go index bf92bf873..aeffb3693 100644 --- a/store/config.go +++ b/store/config.go @@ -12,19 +12,19 @@ type Config struct { Path string `toml:"path"` // Private configs - TransactionToLiveInterval uint32 `toml:"-"` - SortitionInterval uint32 `toml:"-"` - AccountCacheSize int `toml:"-"` - PublicKeyCacheSize int `toml:"-"` + TxCacheSize uint32 `toml:"-"` + SortitionCacheSize uint32 `toml:"-"` + AccountCacheSize int `toml:"-"` + PublicKeyCacheSize int `toml:"-"` } func DefaultConfig() *Config { return &Config{ - Path: "data", - TransactionToLiveInterval: 8640, - SortitionInterval: 17, - AccountCacheSize: 1024, - PublicKeyCacheSize: 1024, + Path: "data", + TxCacheSize: 8640, + SortitionCacheSize: 17, + AccountCacheSize: 1024, + PublicKeyCacheSize: 1024, } } @@ -44,8 +44,8 @@ func (conf *Config) BasicCheck() error { if conf.AccountCacheSize == 0 || conf.PublicKeyCacheSize == 0 || - conf.SortitionInterval == 0 || - conf.TransactionToLiveInterval == 0 { + conf.SortitionCacheSize == 0 || + conf.TxCacheSize == 0 { return errors.Errorf(errors.ErrInvalidConfig, "private configs is not valid") } return nil diff --git a/store/store.go b/store/store.go index a815d62e1..7a6473136 100644 --- a/store/store.go +++ b/store/store.go @@ -93,8 +93,8 @@ func NewStore(conf *Config) (Store, error) { db: db, pubKeyCache: pubKeyCache, batch: new(leveldb.Batch), - blockStore: newBlockStore(db, conf.SortitionInterval), - txStore: newTxStore(db, conf.TransactionToLiveInterval), + blockStore: newBlockStore(db, conf.SortitionCacheSize), + txStore: newTxStore(db, conf.TxCacheSize), accountStore: newAccountStore(db, conf.AccountCacheSize), validatorStore: newValidatorStore(db), } @@ -106,8 +106,8 @@ func NewStore(conf *Config) (Store, error) { currentHeight := lc.Height() startHeight := uint32(1) - if currentHeight > conf.TransactionToLiveInterval { - startHeight = currentHeight - conf.TransactionToLiveInterval + if currentHeight > conf.TxCacheSize { + startHeight = currentHeight - conf.TxCacheSize } for i := startHeight; i < currentHeight+1; i++ { diff --git a/store/store_test.go b/store/store_test.go index 814cf13be..312d3732c 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -27,11 +27,11 @@ func setup(t *testing.T) *testData { params := param.DefaultParams() conf := &Config{ - Path: util.TempDirPath(), - TransactionToLiveInterval: params.TransactionToLiveInterval, - SortitionInterval: params.SortitionInterval, - AccountCacheSize: 1024, - PublicKeyCacheSize: 1024, + Path: util.TempDirPath(), + TxCacheSize: params.TransactionToLiveInterval, + SortitionCacheSize: params.SortitionInterval, + AccountCacheSize: 1024, + PublicKeyCacheSize: 1024, } s, err := NewStore(conf) require.NoError(t, err) From d297c9ad28c5838f893a2cface9818743e0c9d67 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Fri, 29 Dec 2023 18:54:32 +0330 Subject: [PATCH 20/30] fix: clone account in store/account iterates --- store/account.go | 2 +- store/config.go | 15 ++++----------- 2 files changed, 5 insertions(+), 12 deletions(-) diff --git a/store/account.go b/store/account.go index cd8d48ba9..58d0f5d42 100644 --- a/store/account.go +++ b/store/account.go @@ -81,7 +81,7 @@ func (as *accountStore) iterateAccounts(consumer func(crypto.Address, *account.A var addr crypto.Address copy(addr[:], key[1:]) - stopped := consumer(addr, acc.Clone()) + stopped := consumer(addr, acc) if stopped { return } diff --git a/store/config.go b/store/config.go index aeffb3693..df4c09c64 100644 --- a/store/config.go +++ b/store/config.go @@ -21,10 +21,10 @@ type Config struct { func DefaultConfig() *Config { return &Config{ Path: "data", - TxCacheSize: 8640, - SortitionCacheSize: 17, - AccountCacheSize: 1024, - PublicKeyCacheSize: 1024, + TxCacheSize: 0, + SortitionCacheSize: 0, + AccountCacheSize: 0, + PublicKeyCacheSize: 0, } } @@ -41,12 +41,5 @@ func (conf *Config) BasicCheck() error { if !util.IsValidDirPath(conf.Path) { return errors.Errorf(errors.ErrInvalidConfig, "path is not valid") } - - if conf.AccountCacheSize == 0 || - conf.PublicKeyCacheSize == 0 || - conf.SortitionCacheSize == 0 || - conf.TxCacheSize == 0 { - return errors.Errorf(errors.ErrInvalidConfig, "private configs is not valid") - } return nil } From 4f0b61c73efc13973c69ffb0ac44530ac964297a Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Fri, 29 Dec 2023 18:58:06 +0330 Subject: [PATCH 21/30] test: refined account deep copy --- store/account_test.go | 2 ++ 1 file changed, 2 insertions(+) diff --git a/store/account_test.go b/store/account_test.go index aa54b72f0..2d6f7a09f 100644 --- a/store/account_test.go +++ b/store/account_test.go @@ -140,6 +140,8 @@ func TestAccountDeepCopy(t *testing.T) { acc2, _ := td.store.Account(addr) acc2.AddToBalance(1) + accCache, _ := td.store.accountStore.addrCache.Get(addr) + assert.NotEqual(t, accCache.Hash(), acc2.Hash()) expectedAcc, _ := td.store.accountStore.addrCache.Get(addr) assert.NotEqual(t, expectedAcc.Hash(), acc2.Hash()) From bfb39917632c8a9fff7748b17d07d89f33c9412e Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Fri, 29 Dec 2023 19:01:15 +0330 Subject: [PATCH 22/30] refactor: rename addrCache to accInCache --- store/account.go | 20 ++++++++++---------- store/account_test.go | 4 ++-- 2 files changed, 12 insertions(+), 12 deletions(-) diff --git a/store/account.go b/store/account.go index 58d0f5d42..0a029c3d0 100644 --- a/store/account.go +++ b/store/account.go @@ -10,9 +10,9 @@ import ( ) type accountStore struct { - db *leveldb.DB - addrCache *lru.Cache[crypto.Address, *account.Account] - total int32 + db *leveldb.DB + accInCache *lru.Cache[crypto.Address, *account.Account] + total int32 } func accountKey(addr crypto.Address) []byte { return append(accountPrefix, addr.Bytes()...) } @@ -32,14 +32,14 @@ func newAccountStore(db *leveldb.DB, cacheSize int) *accountStore { iter.Release() return &accountStore{ - db: db, - total: total, - addrCache: addrLruCache, + db: db, + total: total, + accInCache: addrLruCache, } } func (as *accountStore) hasAccount(addr crypto.Address) bool { - ok := as.addrCache.Contains(addr) + ok := as.accInCache.Contains(addr) if !ok { ok = tryHas(as.db, accountKey(addr)) } @@ -47,7 +47,7 @@ func (as *accountStore) hasAccount(addr crypto.Address) bool { } func (as *accountStore) account(addr crypto.Address) (*account.Account, error) { - acc, ok := as.addrCache.Get(addr) + acc, ok := as.accInCache.Get(addr) if ok { return acc.Clone(), nil } @@ -62,7 +62,7 @@ func (as *accountStore) account(addr crypto.Address) (*account.Account, error) { return nil, err } - as.addrCache.Add(addr, acc) + as.accInCache.Add(addr, acc) return acc, nil } @@ -100,7 +100,7 @@ func (as *accountStore) updateAccount(batch *leveldb.Batch, addr crypto.Address, if !as.hasAccount(addr) { as.total++ } - as.addrCache.Add(addr, acc) + as.accInCache.Add(addr, acc) batch.Put(accountKey(addr), data) } diff --git a/store/account_test.go b/store/account_test.go index 2d6f7a09f..4c0aa1b3c 100644 --- a/store/account_test.go +++ b/store/account_test.go @@ -140,9 +140,9 @@ func TestAccountDeepCopy(t *testing.T) { acc2, _ := td.store.Account(addr) acc2.AddToBalance(1) - accCache, _ := td.store.accountStore.addrCache.Get(addr) + accCache, _ := td.store.accountStore.accInCache.Get(addr) assert.NotEqual(t, accCache.Hash(), acc2.Hash()) - expectedAcc, _ := td.store.accountStore.addrCache.Get(addr) + expectedAcc, _ := td.store.accountStore.accInCache.Get(addr) assert.NotEqual(t, expectedAcc.Hash(), acc2.Hash()) } From d6d19b8cc4091fd4668d3d21b43516c10c35fc33 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Fri, 29 Dec 2023 20:07:02 +0330 Subject: [PATCH 23/30] feat: implement pair slice --- store/tx.go | 2 +- txpool/pool.go | 10 ++-- util/linkedmap/linkedmap.go | 4 +- util/linkedmap/linkedmap_test.go | 62 ++++++++++---------- util/pairslice/pairslice.go | 70 ++++++++++++++++++++++ util/pairslice/pairslice_test.go | 99 ++++++++++++++++++++++++++++++++ 6 files changed, 208 insertions(+), 39 deletions(-) create mode 100644 util/pairslice/pairslice.go create mode 100644 util/pairslice/pairslice_test.go diff --git a/store/tx.go b/store/tx.go index 1c7c7a8de..d252c6a76 100644 --- a/store/tx.go +++ b/store/tx.go @@ -27,7 +27,7 @@ type txStore struct { func newTxStore(db *leveldb.DB, txCacheSize uint32) *txStore { return &txStore{ db: db, - txCache: linkedmap.NewLinkedMap[tx.ID, uint32](0), + txCache: linkedmap.New[tx.ID, uint32](0), txCacheSize: txCacheSize, } } diff --git a/txpool/pool.go b/txpool/pool.go index 6f63154d4..79d159d43 100644 --- a/txpool/pool.go +++ b/txpool/pool.go @@ -29,11 +29,11 @@ type txPool struct { func NewTxPool(conf *Config, broadcastCh chan message.Message) TxPool { pending := make(map[payload.Type]*linkedmap.LinkedMap[tx.ID, *tx.Tx]) - pending[payload.TypeTransfer] = linkedmap.NewLinkedMap[tx.ID, *tx.Tx](conf.sendPoolSize()) - pending[payload.TypeBond] = linkedmap.NewLinkedMap[tx.ID, *tx.Tx](conf.bondPoolSize()) - pending[payload.TypeUnbond] = linkedmap.NewLinkedMap[tx.ID, *tx.Tx](conf.unbondPoolSize()) - pending[payload.TypeWithdraw] = linkedmap.NewLinkedMap[tx.ID, *tx.Tx](conf.withdrawPoolSize()) - pending[payload.TypeSortition] = linkedmap.NewLinkedMap[tx.ID, *tx.Tx](conf.sortitionPoolSize()) + pending[payload.TypeTransfer] = linkedmap.New[tx.ID, *tx.Tx](conf.sendPoolSize()) + pending[payload.TypeBond] = linkedmap.New[tx.ID, *tx.Tx](conf.bondPoolSize()) + pending[payload.TypeUnbond] = linkedmap.New[tx.ID, *tx.Tx](conf.unbondPoolSize()) + pending[payload.TypeWithdraw] = linkedmap.New[tx.ID, *tx.Tx](conf.withdrawPoolSize()) + pending[payload.TypeSortition] = linkedmap.New[tx.ID, *tx.Tx](conf.sortitionPoolSize()) pool := &txPool{ config: conf, diff --git a/util/linkedmap/linkedmap.go b/util/linkedmap/linkedmap.go index 3257e22d8..bd0cdd42e 100644 --- a/util/linkedmap/linkedmap.go +++ b/util/linkedmap/linkedmap.go @@ -15,8 +15,8 @@ type LinkedMap[K comparable, V any] struct { capacity int } -// NewLinkedMap creates a new LinkedMap with the specified capacity. -func NewLinkedMap[K comparable, V any](capacity int) *LinkedMap[K, V] { +// New creates a new LinkedMap with the specified capacity. +func New[K comparable, V any](capacity int) *LinkedMap[K, V] { return &LinkedMap[K, V]{ list: ll.New[Pair[K, V]](), hashmap: make(map[K]*ll.Element[Pair[K, V]]), diff --git a/util/linkedmap/linkedmap_test.go b/util/linkedmap/linkedmap_test.go index 3dd8e453e..887d002c6 100644 --- a/util/linkedmap/linkedmap_test.go +++ b/util/linkedmap/linkedmap_test.go @@ -8,7 +8,7 @@ import ( func TestLinkedMap(t *testing.T) { t.Run("Test FirstNode", func(t *testing.T) { - lm := NewLinkedMap[int, string](4) + lm := New[int, string](4) assert.Nil(t, lm.HeadNode()) lm.PushFront(3, "c") @@ -20,7 +20,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Test LastNode", func(t *testing.T) { - lm := NewLinkedMap[int, string](4) + lm := New[int, string](4) assert.Nil(t, lm.TailNode()) lm.PushBack(1, "a") @@ -32,7 +32,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Test Get", func(t *testing.T) { - lm := NewLinkedMap[int, string](4) + lm := New[int, string](4) lm.PushBack(2, "b") lm.PushBack(1, "a") @@ -46,7 +46,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Test Remove", func(t *testing.T) { - lm := NewLinkedMap[int, string](4) + lm := New[int, string](4) lm.PushBack(0, "-") lm.PushBack(2, "b") @@ -56,7 +56,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Test RemoveTail", func(t *testing.T) { - lm := NewLinkedMap[int, string](4) + lm := New[int, string](4) lm.PushBack(0, "-") lm.PushBack(1, "a") lm.PushBack(2, "b") @@ -67,7 +67,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Test RemoveHead", func(t *testing.T) { - lm := NewLinkedMap[int, string](4) + lm := New[int, string](4) lm.PushBack(0, "-") lm.PushBack(1, "a") lm.PushBack(2, "b") @@ -78,7 +78,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Should updates v", func(t *testing.T) { - lm := NewLinkedMap[int, string](4) + lm := New[int, string](4) lm.PushBack(1, "a") lm.PushBack(1, "b") @@ -93,7 +93,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Should prunes oldest item", func(t *testing.T) { - lm := NewLinkedMap[int, string](4) + lm := New[int, string](4) lm.PushBack(1, "a") lm.PushBack(2, "b") @@ -111,7 +111,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Should prunes by changing capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](4) + lm := New[int, string](4) lm.PushBack(1, "a") lm.PushBack(2, "b") @@ -132,7 +132,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Test PushBack and prune", func(t *testing.T) { - lm := NewLinkedMap[int, string](3) + lm := New[int, string](3) lm.PushBack(1, "a") // This item should be pruned lm.PushBack(2, "b") @@ -145,7 +145,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Test PushFront and prune", func(t *testing.T) { - lm := NewLinkedMap[int, string](3) + lm := New[int, string](3) lm.PushFront(1, "a") lm.PushFront(2, "b") @@ -158,7 +158,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Delete first ", func(t *testing.T) { - lm := NewLinkedMap[int, string](3) + lm := New[int, string](3) lm.PushBack(1, "a") lm.PushBack(2, "b") @@ -171,7 +171,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Delete last", func(t *testing.T) { - lm := NewLinkedMap[int, string](3) + lm := New[int, string](3) lm.PushBack(1, "a") lm.PushBack(2, "b") @@ -184,7 +184,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Test Has function", func(t *testing.T) { - lm := NewLinkedMap[int, string](2) + lm := New[int, string](2) lm.PushBack(1, "a") @@ -193,7 +193,7 @@ func TestLinkedMap(t *testing.T) { }) t.Run("Test Clear", func(t *testing.T) { - lm := NewLinkedMap[int, string](2) + lm := New[int, string](2) lm.PushBack(1, "a") lm.Clear() @@ -204,12 +204,12 @@ func TestLinkedMap(t *testing.T) { func TestCapacity(t *testing.T) { t.Run("Check Capacity", func(t *testing.T) { capacity := 100 - lm := NewLinkedMap[int, string](capacity) + lm := New[int, string](capacity) assert.Equal(t, lm.Capacity(), capacity) }) t.Run("Test FirstNode with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) assert.Nil(t, lm.HeadNode()) lm.PushFront(3, "c") @@ -221,7 +221,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Test LastNode with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) assert.Nil(t, lm.TailNode()) lm.PushBack(1, "a") @@ -233,7 +233,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Test Get with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushBack(2, "b") lm.PushBack(1, "a") @@ -247,7 +247,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Test Remove with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushBack(0, "-") lm.PushBack(2, "b") @@ -257,7 +257,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Test RemoveTail with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushBack(0, "-") lm.PushBack(1, "a") lm.PushBack(2, "b") @@ -268,7 +268,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Test RemoveHead with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushBack(0, "-") lm.PushBack(1, "a") lm.PushBack(2, "b") @@ -279,7 +279,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Should updates v with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushBack(1, "a") lm.PushBack(1, "b") @@ -294,7 +294,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Should not prunes oldest item with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushBack(1, "a") lm.PushBack(2, "b") @@ -312,7 +312,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Should prunes by changing capacity with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushBack(1, "a") lm.PushBack(2, "b") @@ -333,7 +333,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Test PushBack and should not prune with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushBack(1, "a") // This item should be pruned lm.PushBack(2, "b") @@ -346,7 +346,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Test PushFront and prune with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushFront(1, "a") lm.PushFront(2, "b") @@ -359,7 +359,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Delete first with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushBack(1, "a") lm.PushBack(2, "b") @@ -372,7 +372,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Delete last with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushBack(1, "a") lm.PushBack(2, "b") @@ -385,7 +385,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Test Has function with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushBack(1, "a") @@ -394,7 +394,7 @@ func TestCapacity(t *testing.T) { }) t.Run("Test Clear with Zero Capacity", func(t *testing.T) { - lm := NewLinkedMap[int, string](0) + lm := New[int, string](0) lm.PushBack(1, "a") lm.Clear() diff --git a/util/pairslice/pairslice.go b/util/pairslice/pairslice.go new file mode 100644 index 000000000..8c5213824 --- /dev/null +++ b/util/pairslice/pairslice.go @@ -0,0 +1,70 @@ +package pairslice + +import "log" + +type Pair[K comparable, V any] struct { + FirstElement K + SecondElement V +} + +type PairSlice[K comparable, V any] struct { + pairs []*Pair[K, V] + index int + capacity int +} + +func New[K comparable, V any](capacity int) *PairSlice[K, V] { + return &PairSlice[K, V]{ + pairs: make([]*Pair[K, V], capacity), + capacity: capacity, + } +} + +func (ps *PairSlice[K, V]) Append(firstElement K, secondElement V) { + ps.pairs[ps.index] = &Pair[K, V]{firstElement, secondElement} + log.Printf("memory address: %p", ps.pairs) + ps.index++ + ps.resetIndex() +} + +func (ps *PairSlice[K, V]) Pop() { + ps.pairs = ps.pairs[1:] + ps.index-- + ps.resetIndex() +} + +func (ps *PairSlice[K, V]) Has(index int) bool { + if index >= ps.capacity { + return false + } + + return ps.pairs[index] != nil +} + +func (ps *PairSlice[K, V]) Get(index int) *Pair[K, V] { + if index >= ps.capacity { + return nil + } + return ps.pairs[index] +} + +func (ps *PairSlice[K, V]) First() *Pair[K, V] { + return ps.Get(0) +} + +func (ps *PairSlice[K, V]) Last() *Pair[K, V] { + if ps.index == 0 { + return ps.Get(len(ps.pairs) - 1) + } + return ps.Get(ps.index - 1) +} + +func (ps *PairSlice[K, V]) resetIndex() { + if ps.index < 0 { + ps.index = 0 + } + + if ps.index == ps.capacity { + ps.index = 0 + } +} diff --git a/util/pairslice/pairslice_test.go b/util/pairslice/pairslice_test.go new file mode 100644 index 000000000..b5befb964 --- /dev/null +++ b/util/pairslice/pairslice_test.go @@ -0,0 +1,99 @@ +package pairslice + +import ( + "testing" + + "github.com/stretchr/testify/assert" +) + +func TestNew(t *testing.T) { + ps := New[int, string](10) + + assert.NotNil(t, ps) + assert.Equal(t, 10, len(ps.pairs)) + assert.Equal(t, 0, ps.index) + assert.Equal(t, 10, ps.capacity) +} + +func TestPairSlice(t *testing.T) { + t.Run("Test Append", func(t *testing.T) { + ps := New[int, string](4) + + ps.Append(0, "a") + ps.Append(1, "b") + ps.Append(2, "c") + + assert.Equal(t, &Pair[int, string]{2, "c"}, ps.pairs[2]) + }) + + t.Run("Test Pop", func(t *testing.T) { + ps := New[int, string](4) + + ps.Append(0, "a") + ps.Append(1, "b") + ps.Append(2, "c") + + ps.Pop() + + assert.Equal(t, &Pair[int, string]{1, "b"}, ps.pairs[0]) + assert.Equal(t, &Pair[int, string]{2, "c"}, ps.pairs[1]) + assert.Nil(t, ps.pairs[2]) + }) + + t.Run("Test Has", func(t *testing.T) { + ps := New[int, string](4) + + ps.Append(0, "a") + ps.Append(1, "b") + ps.Append(2, "c") + + assert.True(t, ps.Has(0)) + assert.False(t, ps.Has(3)) + assert.False(t, ps.Has(10)) + }) + + t.Run("Test Get", func(t *testing.T) { + ps := New[int, string](4) + + ps.Append(0, "a") + ps.Append(1, "b") + ps.Append(2, "c") + + assert.Equal(t, &Pair[int, string]{0, "a"}, ps.Get(0)) + assert.Equal(t, &Pair[int, string]{2, "c"}, ps.Get(2)) + assert.Nil(t, ps.Get(10)) + }) + + t.Run("Test First", func(t *testing.T) { + ps := New[int, string](4) + + ps.Append(0, "a") + ps.Append(1, "b") + ps.Append(2, "c") + + assert.Equal(t, &Pair[int, string]{0, "a"}, ps.First()) + }) + + t.Run("Test Last", func(t *testing.T) { + ps := New[int, string](4) + + ps.Append(0, "a") + ps.Append(1, "b") + ps.Append(2, "c") + + assert.Equal(t, &Pair[int, string]{2, "c"}, ps.Last()) + }) + + t.Run("Test Last with Pop", func(t *testing.T) { + ps := New[int, string](4) + + ps.Append(0, "a") + ps.Append(1, "b") + ps.Append(2, "c") + ps.Append(3, "d") + + ps.Pop() + + assert.Equal(t, &Pair[int, string]{3, "d"}, ps.Last()) + }) +} From ef3285379c58bf018b23cb8c6cd7ba15751bdd3e Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Fri, 29 Dec 2023 20:40:57 +0330 Subject: [PATCH 24/30] docs: add comment for pairslice methods --- util/pairslice/pairslice.go | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/util/pairslice/pairslice.go b/util/pairslice/pairslice.go index 8c5213824..48e044fe9 100644 --- a/util/pairslice/pairslice.go +++ b/util/pairslice/pairslice.go @@ -13,6 +13,7 @@ type PairSlice[K comparable, V any] struct { capacity int } +// New creates a new instance of PairSlice with a specified capacity. func New[K comparable, V any](capacity int) *PairSlice[K, V] { return &PairSlice[K, V]{ pairs: make([]*Pair[K, V], capacity), @@ -20,6 +21,8 @@ func New[K comparable, V any](capacity int) *PairSlice[K, V] { } } +// Append adds the Pair to the end of the slice. If the capacity is full, +// it automatically removes the first element and appends the new Pair to the end of the PairSlice. func (ps *PairSlice[K, V]) Append(firstElement K, secondElement V) { ps.pairs[ps.index] = &Pair[K, V]{firstElement, secondElement} log.Printf("memory address: %p", ps.pairs) @@ -27,12 +30,14 @@ func (ps *PairSlice[K, V]) Append(firstElement K, secondElement V) { ps.resetIndex() } +// Pop removes the first element from PairSlice. func (ps *PairSlice[K, V]) Pop() { ps.pairs = ps.pairs[1:] ps.index-- ps.resetIndex() } +// Has checks the index exists or not. func (ps *PairSlice[K, V]) Has(index int) bool { if index >= ps.capacity { return false @@ -41,6 +46,7 @@ func (ps *PairSlice[K, V]) Has(index int) bool { return ps.pairs[index] != nil } +// Get returns the Pair at the specified index. If the index doesn't exist, it returns nil. func (ps *PairSlice[K, V]) Get(index int) *Pair[K, V] { if index >= ps.capacity { return nil @@ -48,10 +54,12 @@ func (ps *PairSlice[K, V]) Get(index int) *Pair[K, V] { return ps.pairs[index] } +// First returns the first Pair in the PairSlice. func (ps *PairSlice[K, V]) First() *Pair[K, V] { return ps.Get(0) } +// Last returns the last Pair in the PairSlice. func (ps *PairSlice[K, V]) Last() *Pair[K, V] { if ps.index == 0 { return ps.Get(len(ps.pairs) - 1) From c6c05bc8bb8cb5c388940cb2b33e63ed1b2b8795 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Fri, 29 Dec 2023 22:59:36 +0330 Subject: [PATCH 25/30] feat: implement tripleslice --- store/block.go | 34 ++++---- store/block_test.go | 5 +- store/store.go | 2 +- util/pairslice/pairslice.go | 78 ------------------ util/tripleslice/tripleslice.go | 81 +++++++++++++++++++ .../tripleslice_test.go} | 35 +++++--- 6 files changed, 124 insertions(+), 111 deletions(-) delete mode 100644 util/pairslice/pairslice.go create mode 100644 util/tripleslice/tripleslice.go rename util/{pairslice/pairslice_test.go => tripleslice/tripleslice_test.go} (60%) diff --git a/store/block.go b/store/block.go index 8be811d69..74639266b 100644 --- a/store/block.go +++ b/store/block.go @@ -10,6 +10,7 @@ import ( "github.com/pactus-project/pactus/util" "github.com/pactus-project/pactus/util/encoding" "github.com/pactus-project/pactus/util/logger" + "github.com/pactus-project/pactus/util/tripleslice" "github.com/syndtr/goleveldb/leveldb" ) @@ -24,15 +25,15 @@ func blockHashKey(h hash.Hash) []byte { type blockStore struct { db *leveldb.DB - sortitionSeedCache []*sortition.VerifiableSeed - sortitionInterval uint32 + sortitionSeedCache *tripleslice.TripleSlice[uint32, *sortition.VerifiableSeed] + sortitionCacheSize uint32 } func newBlockStore(db *leveldb.DB, sortitionCacheSize uint32) *blockStore { return &blockStore{ db: db, - sortitionSeedCache: make([]*sortition.VerifiableSeed, 0, sortitionCacheSize), - sortitionInterval: sortitionCacheSize, + sortitionSeedCache: tripleslice.New[uint32, *sortition.VerifiableSeed](int(sortitionCacheSize)), + sortitionCacheSize: sortitionCacheSize, } } @@ -99,7 +100,7 @@ func (bs *blockStore) saveBlock(batch *leveldb.Batch, height uint32, blk *block. batch.Put(blockHashKey, util.Uint32ToSlice(height)) sortitionSeed := blk.Header().SortitionSeed() - bs.saveToCache(sortitionSeed) + bs.saveToCache(height, sortitionSeed) return regs } @@ -121,19 +122,15 @@ func (bs *blockStore) blockHeight(h hash.Hash) uint32 { return util.SliceToUint32(data) } -func (bs *blockStore) sortitionSeed(blockHeight, currentHeight uint32) *sortition.VerifiableSeed { +func (bs *blockStore) sortitionSeed(_, currentHeight uint32) *sortition.VerifiableSeed { + triple := bs.sortitionSeedCache.Last() + blockHeight := triple.FirstElement index := currentHeight - blockHeight - if index > bs.sortitionInterval { - return nil - } - if index != 0 { - index = uint32(len(bs.sortitionSeedCache)) - index - } else { - index = uint32(len(bs.sortitionSeedCache)) - 1 + if index == 0 { + return bs.sortitionSeedCache.Last().SecondElement } - - return bs.sortitionSeedCache[index] + return bs.sortitionSeedCache.Get(triple.ThirdElement).SecondElement } func (bs *blockStore) hasBlock(height uint32) bool { @@ -144,9 +141,6 @@ func (bs *blockStore) hasPublicKey(addr crypto.Address) bool { return tryHas(bs.db, publicKeyKey(addr)) } -func (bs *blockStore) saveToCache(sortitionSeed sortition.VerifiableSeed) { - bs.sortitionSeedCache = append(bs.sortitionSeedCache, &sortitionSeed) - if len(bs.sortitionSeedCache) > int(bs.sortitionInterval) { - bs.sortitionSeedCache = bs.sortitionSeedCache[1:] - } +func (bs *blockStore) saveToCache(blockHeight uint32, sortitionSeed sortition.VerifiableSeed) { + bs.sortitionSeedCache.Append(blockHeight, &sortitionSeed) } diff --git a/store/block_test.go b/store/block_test.go index 33c61c4d4..3359bfd3d 100644 --- a/store/block_test.go +++ b/store/block_test.go @@ -81,12 +81,11 @@ func TestBlockStore(t *testing.T) { } // check old sortition seed doesn't exist in cache - for _, seed := range td.store.blockStore.sortitionSeedCache { + for _, seed := range td.store.blockStore.sortitionSeedCache.All() { assert.NotEqual(t, &oldSortitionSeed, seed) } // last sortition seed should exist at last index of cache - assert.Equal(t, &lastSortitionSeed, td.store.blockStore. - sortitionSeedCache[len(td.store.blockStore.sortitionSeedCache)-1]) + assert.Equal(t, &lastSortitionSeed, td.store.blockStore.sortitionSeedCache.Last().SecondElement) }) } diff --git a/store/store.go b/store/store.go index 7a6473136..4fe69c57a 100644 --- a/store/store.go +++ b/store/store.go @@ -126,7 +126,7 @@ func NewStore(conf *Config) (Store, error) { } sortitionSeed := blk.Header().SortitionSeed() - s.blockStore.saveToCache(sortitionSeed) + s.blockStore.saveToCache(i, sortitionSeed) } return s, nil diff --git a/util/pairslice/pairslice.go b/util/pairslice/pairslice.go deleted file mode 100644 index 48e044fe9..000000000 --- a/util/pairslice/pairslice.go +++ /dev/null @@ -1,78 +0,0 @@ -package pairslice - -import "log" - -type Pair[K comparable, V any] struct { - FirstElement K - SecondElement V -} - -type PairSlice[K comparable, V any] struct { - pairs []*Pair[K, V] - index int - capacity int -} - -// New creates a new instance of PairSlice with a specified capacity. -func New[K comparable, V any](capacity int) *PairSlice[K, V] { - return &PairSlice[K, V]{ - pairs: make([]*Pair[K, V], capacity), - capacity: capacity, - } -} - -// Append adds the Pair to the end of the slice. If the capacity is full, -// it automatically removes the first element and appends the new Pair to the end of the PairSlice. -func (ps *PairSlice[K, V]) Append(firstElement K, secondElement V) { - ps.pairs[ps.index] = &Pair[K, V]{firstElement, secondElement} - log.Printf("memory address: %p", ps.pairs) - ps.index++ - ps.resetIndex() -} - -// Pop removes the first element from PairSlice. -func (ps *PairSlice[K, V]) Pop() { - ps.pairs = ps.pairs[1:] - ps.index-- - ps.resetIndex() -} - -// Has checks the index exists or not. -func (ps *PairSlice[K, V]) Has(index int) bool { - if index >= ps.capacity { - return false - } - - return ps.pairs[index] != nil -} - -// Get returns the Pair at the specified index. If the index doesn't exist, it returns nil. -func (ps *PairSlice[K, V]) Get(index int) *Pair[K, V] { - if index >= ps.capacity { - return nil - } - return ps.pairs[index] -} - -// First returns the first Pair in the PairSlice. -func (ps *PairSlice[K, V]) First() *Pair[K, V] { - return ps.Get(0) -} - -// Last returns the last Pair in the PairSlice. -func (ps *PairSlice[K, V]) Last() *Pair[K, V] { - if ps.index == 0 { - return ps.Get(len(ps.pairs) - 1) - } - return ps.Get(ps.index - 1) -} - -func (ps *PairSlice[K, V]) resetIndex() { - if ps.index < 0 { - ps.index = 0 - } - - if ps.index == ps.capacity { - ps.index = 0 - } -} diff --git a/util/tripleslice/tripleslice.go b/util/tripleslice/tripleslice.go new file mode 100644 index 000000000..cb2f92ce2 --- /dev/null +++ b/util/tripleslice/tripleslice.go @@ -0,0 +1,81 @@ +package tripleslice + +type Triple[K comparable, V any] struct { + FirstElement K + SecondElement V + ThirdElement int +} + +type TripleSlice[K comparable, V any] struct { + pairs []*Triple[K, V] + index int + capacity int +} + +// New creates a new instance of TripleSlice with a specified capacity. +func New[K comparable, V any](capacity int) *TripleSlice[K, V] { + return &TripleSlice[K, V]{ + pairs: make([]*Triple[K, V], capacity), + capacity: capacity, + } +} + +// Append adds the Triple to the end of the slice. If the capacity is full, +// it automatically removes the first element and appends the new Triple to the end of the TripleSlice. +func (ps *TripleSlice[K, V]) Append(firstElement K, secondElement V) { + ps.pairs[ps.index] = &Triple[K, V]{firstElement, secondElement, ps.index} + ps.index++ + ps.resetIndex() +} + +// Pop removes the first element from TripleSlice. +func (ps *TripleSlice[K, V]) Pop() { + ps.pairs = ps.pairs[1:] + ps.index-- + ps.resetIndex() +} + +// Has checks the index exists or not. +func (ps *TripleSlice[K, V]) Has(index int) bool { + if index >= ps.capacity { + return false + } + + return ps.pairs[index] != nil +} + +// Get returns the Triple at the specified index. If the index doesn't exist, it returns nil. +func (ps *TripleSlice[K, V]) Get(index int) *Triple[K, V] { + if index >= ps.capacity { + return nil + } + return ps.pairs[index] +} + +// First returns the first Triple in the TripleSlice. +func (ps *TripleSlice[K, V]) First() *Triple[K, V] { + return ps.Get(0) +} + +// Last returns the last Triple in the TripleSlice. +func (ps *TripleSlice[K, V]) Last() *Triple[K, V] { + if ps.index == 0 { + return ps.Get(len(ps.pairs) - 1) + } + return ps.Get(ps.index - 1) +} + +// All returns whole pairs of slice. +func (ps *TripleSlice[K, V]) All() []*Triple[K, V] { + return ps.pairs +} + +func (ps *TripleSlice[K, V]) resetIndex() { + if ps.index < 0 { + ps.index = 0 + } + + if ps.index == ps.capacity { + ps.index = 0 + } +} diff --git a/util/pairslice/pairslice_test.go b/util/tripleslice/tripleslice_test.go similarity index 60% rename from util/pairslice/pairslice_test.go rename to util/tripleslice/tripleslice_test.go index b5befb964..0520a6abb 100644 --- a/util/pairslice/pairslice_test.go +++ b/util/tripleslice/tripleslice_test.go @@ -1,4 +1,4 @@ -package pairslice +package tripleslice import ( "testing" @@ -23,7 +23,7 @@ func TestPairSlice(t *testing.T) { ps.Append(1, "b") ps.Append(2, "c") - assert.Equal(t, &Pair[int, string]{2, "c"}, ps.pairs[2]) + assert.Equal(t, &Triple[int, string]{2, "c", 2}, ps.pairs[2]) }) t.Run("Test Pop", func(t *testing.T) { @@ -35,8 +35,8 @@ func TestPairSlice(t *testing.T) { ps.Pop() - assert.Equal(t, &Pair[int, string]{1, "b"}, ps.pairs[0]) - assert.Equal(t, &Pair[int, string]{2, "c"}, ps.pairs[1]) + assert.Equal(t, &Triple[int, string]{1, "b", 1}, ps.pairs[0]) + assert.Equal(t, &Triple[int, string]{2, "c", 2}, ps.pairs[1]) assert.Nil(t, ps.pairs[2]) }) @@ -59,8 +59,8 @@ func TestPairSlice(t *testing.T) { ps.Append(1, "b") ps.Append(2, "c") - assert.Equal(t, &Pair[int, string]{0, "a"}, ps.Get(0)) - assert.Equal(t, &Pair[int, string]{2, "c"}, ps.Get(2)) + assert.Equal(t, &Triple[int, string]{0, "a", 0}, ps.Get(0)) + assert.Equal(t, &Triple[int, string]{2, "c", 2}, ps.Get(2)) assert.Nil(t, ps.Get(10)) }) @@ -71,7 +71,7 @@ func TestPairSlice(t *testing.T) { ps.Append(1, "b") ps.Append(2, "c") - assert.Equal(t, &Pair[int, string]{0, "a"}, ps.First()) + assert.Equal(t, &Triple[int, string]{0, "a", 0}, ps.First()) }) t.Run("Test Last", func(t *testing.T) { @@ -81,7 +81,7 @@ func TestPairSlice(t *testing.T) { ps.Append(1, "b") ps.Append(2, "c") - assert.Equal(t, &Pair[int, string]{2, "c"}, ps.Last()) + assert.Equal(t, &Triple[int, string]{2, "c", 2}, ps.Last()) }) t.Run("Test Last with Pop", func(t *testing.T) { @@ -94,6 +94,23 @@ func TestPairSlice(t *testing.T) { ps.Pop() - assert.Equal(t, &Pair[int, string]{3, "d"}, ps.Last()) + assert.Equal(t, &Triple[int, string]{3, "d", 3}, ps.Last()) + }) + + t.Run("Test All", func(t *testing.T) { + ps := New[int, string](4) + + ps.Append(0, "a") + ps.Append(1, "b") + ps.Append(2, "c") + + all := ps.All() + + assert.Equal(t, all[0].FirstElement, 0) + assert.Equal(t, all[1].FirstElement, 1) + assert.Equal(t, all[2].FirstElement, 2) + assert.Equal(t, all[0].SecondElement, "a") + assert.Equal(t, all[1].SecondElement, "b") + assert.Equal(t, all[2].SecondElement, "c") }) } From 9a2a1dca9061201bf2b09b7712f2c1c29d6f4326 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sat, 30 Dec 2023 20:55:38 +0330 Subject: [PATCH 26/30] feat: implement sortition seed --- sandbox/sandbox.go | 2 +- store/account_test.go | 10 +-- store/block.go | 28 ++++--- store/block_test.go | 51 ++++++------ store/interface.go | 2 +- store/mock.go | 2 +- store/store.go | 4 +- store/store_test.go | 39 +++++---- store/validator_test.go | 14 ++-- util/pairslice/pairslice.go | 67 +++++++++++++++ .../pairslice_test.go} | 38 ++++----- util/tripleslice/tripleslice.go | 81 ------------------- 12 files changed, 167 insertions(+), 171 deletions(-) create mode 100644 util/pairslice/pairslice.go rename util/{tripleslice/tripleslice_test.go => pairslice/pairslice_test.go} (62%) delete mode 100644 util/tripleslice/tripleslice.go diff --git a/sandbox/sandbox.go b/sandbox/sandbox.go index ce1d56d81..4919788e3 100644 --- a/sandbox/sandbox.go +++ b/sandbox/sandbox.go @@ -280,7 +280,7 @@ func (sb *sandbox) VerifyProof(blockHeight uint32, proof sortition.Proof, val *v sb.lk.RLock() defer sb.lk.RUnlock() - seed := sb.store.SortitionSeed(blockHeight, sb.height+1) + seed := sb.store.SortitionSeed(blockHeight) if seed == nil { return false } diff --git a/store/account_test.go b/store/account_test.go index 4c0aa1b3c..6d95cf2bb 100644 --- a/store/account_test.go +++ b/store/account_test.go @@ -11,7 +11,7 @@ import ( ) func TestAccountCounter(t *testing.T) { - td := setup(t) + td := setup(t, nil) num := td.RandInt32(1000) acc, addr := td.GenerateTestAccount(num) @@ -43,7 +43,7 @@ func TestAccountCounter(t *testing.T) { } func TestAccountBatchSaving(t *testing.T) { - td := setup(t) + td := setup(t, nil) total := td.RandInt32NonZero(100) t.Run("Add some accounts", func(t *testing.T) { @@ -63,7 +63,7 @@ func TestAccountBatchSaving(t *testing.T) { } func TestAccountByAddress(t *testing.T) { - td := setup(t) + td := setup(t, nil) total := td.RandInt32NonZero(100) var lastAddr crypto.Address @@ -103,7 +103,7 @@ func TestAccountByAddress(t *testing.T) { } func TestIterateAccounts(t *testing.T) { - td := setup(t) + td := setup(t, nil) total := td.RandInt32NonZero(100) hashes1 := []hash.Hash{} @@ -132,7 +132,7 @@ func TestIterateAccounts(t *testing.T) { } func TestAccountDeepCopy(t *testing.T) { - td := setup(t) + td := setup(t, nil) num := td.RandInt32(1000) acc1, addr := td.GenerateTestAccount(num) diff --git a/store/block.go b/store/block.go index 74639266b..5bda93358 100644 --- a/store/block.go +++ b/store/block.go @@ -10,7 +10,7 @@ import ( "github.com/pactus-project/pactus/util" "github.com/pactus-project/pactus/util/encoding" "github.com/pactus-project/pactus/util/logger" - "github.com/pactus-project/pactus/util/tripleslice" + "github.com/pactus-project/pactus/util/pairslice" "github.com/syndtr/goleveldb/leveldb" ) @@ -25,14 +25,14 @@ func blockHashKey(h hash.Hash) []byte { type blockStore struct { db *leveldb.DB - sortitionSeedCache *tripleslice.TripleSlice[uint32, *sortition.VerifiableSeed] + sortitionSeedCache *pairslice.PairSlice[uint32, *sortition.VerifiableSeed] sortitionCacheSize uint32 } func newBlockStore(db *leveldb.DB, sortitionCacheSize uint32) *blockStore { return &blockStore{ db: db, - sortitionSeedCache: tripleslice.New[uint32, *sortition.VerifiableSeed](int(sortitionCacheSize)), + sortitionSeedCache: pairslice.New[uint32, *sortition.VerifiableSeed](int(sortitionCacheSize)), sortitionCacheSize: sortitionCacheSize, } } @@ -122,15 +122,20 @@ func (bs *blockStore) blockHeight(h hash.Hash) uint32 { return util.SliceToUint32(data) } -func (bs *blockStore) sortitionSeed(_, currentHeight uint32) *sortition.VerifiableSeed { - triple := bs.sortitionSeedCache.Last() - blockHeight := triple.FirstElement - index := currentHeight - blockHeight +func (bs *blockStore) sortitionSeed(blockHeight uint32) *sortition.VerifiableSeed { + startHeight, _, _ := bs.sortitionSeedCache.First() - if index == 0 { - return bs.sortitionSeedCache.Last().SecondElement + if blockHeight < startHeight { + return nil } - return bs.sortitionSeedCache.Get(triple.ThirdElement).SecondElement + + index := blockHeight - startHeight + _, sortitionSeed, ok := bs.sortitionSeedCache.Get(int(index)) + if !ok { + return nil + } + + return sortitionSeed } func (bs *blockStore) hasBlock(height uint32) bool { @@ -143,4 +148,7 @@ func (bs *blockStore) hasPublicKey(addr crypto.Address) bool { func (bs *blockStore) saveToCache(blockHeight uint32, sortitionSeed sortition.VerifiableSeed) { bs.sortitionSeedCache.Append(blockHeight, &sortitionSeed) + if bs.sortitionSeedCache.Len() > int(bs.sortitionCacheSize) { + bs.sortitionSeedCache.RemoveFirst() + } } diff --git a/store/block_test.go b/store/block_test.go index 3359bfd3d..d5a0c2e1a 100644 --- a/store/block_test.go +++ b/store/block_test.go @@ -4,12 +4,11 @@ import ( "bytes" "testing" - "github.com/pactus-project/pactus/sortition" "github.com/stretchr/testify/assert" ) func TestBlockStore(t *testing.T) { - td := setup(t) + td := setup(t, nil) lastCert := td.store.LastCertificate() lastHeight := lastCert.Height() @@ -56,36 +55,34 @@ func TestBlockStore(t *testing.T) { }() td.store.SaveBlock(nextBlk, nextCert) }) +} - t.Run("Should not be old sortition seed in cache plus check last sortition seed be in cache", - func(t *testing.T) { - var oldSortitionSeed sortition.VerifiableSeed - var lastSortitionSeed sortition.VerifiableSeed - lastHeight = td.store.LastCertificate().Height() - generateBlkCount := 100 +func TestSortitionSeed(t *testing.T) { + conf := testConfig() + conf.SortitionCacheSize = 7 - for i := 0; i <= generateBlkCount; i++ { - lastHeight++ - nNextBlk, nNextCert := td.GenerateTestBlock(lastHeight) + td := setup(t, conf) + lastHeight := td.store.LastCertificate().Height() - td.store.SaveBlock(nNextBlk, nNextCert) - assert.NoError(t, td.store.WriteBatch()) + t.Run("Test height zero", func(t *testing.T) { + assert.Nil(t, td.store.SortitionSeed(0)) + }) - if i == 0 { - oldSortitionSeed = nNextBlk.Header().SortitionSeed() - } + t.Run("Test non existing height", func(t *testing.T) { + assert.Nil(t, td.store.SortitionSeed(lastHeight+1)) + }) - if i == generateBlkCount { - lastSortitionSeed = nNextBlk.Header().SortitionSeed() - } - } + t.Run("Test not cached height", func(t *testing.T) { + assert.Nil(t, td.store.SortitionSeed(3)) + }) - // check old sortition seed doesn't exist in cache - for _, seed := range td.store.blockStore.sortitionSeedCache.All() { - assert.NotEqual(t, &oldSortitionSeed, seed) - } + t.Run("OK", func(t *testing.T) { + rndInt := td.RandUint32(conf.SortitionCacheSize) + rndInt += lastHeight - conf.SortitionCacheSize - // last sortition seed should exist at last index of cache - assert.Equal(t, &lastSortitionSeed, td.store.blockStore.sortitionSeedCache.Last().SecondElement) - }) + committedBlk, _ := td.store.Block(rndInt) + blk, _ := committedBlk.ToBlock() + expectedSortition := blk.Header().SortitionSeed() + assert.Equal(t, &expectedSortition, td.store.SortitionSeed(rndInt)) + }) } diff --git a/store/interface.go b/store/interface.go index 38d9f2b0a..ee2f36b2a 100644 --- a/store/interface.go +++ b/store/interface.go @@ -82,7 +82,7 @@ type Reader interface { Block(height uint32) (*CommittedBlock, error) BlockHeight(h hash.Hash) uint32 BlockHash(height uint32) hash.Hash - SortitionSeed(currentHeight, height uint32) *sortition.VerifiableSeed + SortitionSeed(blockHeight uint32) *sortition.VerifiableSeed Transaction(id tx.ID) (*CommittedTx, error) AnyRecentTransaction(id tx.ID) bool PublicKey(addr crypto.Address) (*bls.PublicKey, error) diff --git a/store/mock.go b/store/mock.go index e47f5135f..53e54f7cd 100644 --- a/store/mock.go +++ b/store/mock.go @@ -66,7 +66,7 @@ func (m *MockStore) BlockHeight(h hash.Hash) uint32 { return 0 } -func (m *MockStore) SortitionSeed(blockHeight, _ uint32) *sortition.VerifiableSeed { +func (m *MockStore) SortitionSeed(blockHeight uint32) *sortition.VerifiableSeed { if blk, ok := m.Blocks[blockHeight]; ok { sortitionSeed := blk.Header().SortitionSeed() return &sortitionSeed diff --git a/store/store.go b/store/store.go index 4fe69c57a..27b4d9313 100644 --- a/store/store.go +++ b/store/store.go @@ -204,11 +204,11 @@ func (s *store) BlockHash(height uint32) hash.Hash { return hash.UndefHash } -func (s *store) SortitionSeed(blockHeight, currentHeight uint32) *sortition.VerifiableSeed { +func (s *store) SortitionSeed(blockHeight uint32) *sortition.VerifiableSeed { s.lk.Lock() defer s.lk.Unlock() - return s.blockStore.sortitionSeed(blockHeight, currentHeight) + return s.blockStore.sortitionSeed(blockHeight) } func (s *store) PublicKey(addr crypto.Address) (*bls.PublicKey, error) { diff --git a/store/store_test.go b/store/store_test.go index 312d3732c..52cfeb886 100644 --- a/store/store_test.go +++ b/store/store_test.go @@ -6,7 +6,6 @@ import ( "github.com/pactus-project/pactus/crypto/bls" "github.com/pactus-project/pactus/crypto/hash" "github.com/pactus-project/pactus/types/block" - "github.com/pactus-project/pactus/types/param" "github.com/pactus-project/pactus/types/tx" "github.com/pactus-project/pactus/util" "github.com/pactus-project/pactus/util/testsuite" @@ -20,20 +19,26 @@ type testData struct { store *store } -func setup(t *testing.T) *testData { +func testConfig() *Config { + return &Config{ + Path: util.TempDirPath(), + TxCacheSize: 1024, + SortitionCacheSize: 1024, + AccountCacheSize: 1024, + PublicKeyCacheSize: 1024, + } +} + +func setup(t *testing.T, config *Config) *testData { t.Helper() ts := testsuite.NewTestSuite(t) - params := param.DefaultParams() - conf := &Config{ - Path: util.TempDirPath(), - TxCacheSize: params.TransactionToLiveInterval, - SortitionCacheSize: params.SortitionInterval, - AccountCacheSize: 1024, - PublicKeyCacheSize: 1024, + if config == nil { + config = testConfig() } - s, err := NewStore(conf) + + s, err := NewStore(config) require.NoError(t, err) td := &testData{ @@ -51,7 +56,7 @@ func setup(t *testing.T) *testData { } func TestBlockHash(t *testing.T) { - td := setup(t) + td := setup(t, nil) sb, _ := td.store.Block(1) @@ -61,7 +66,7 @@ func TestBlockHash(t *testing.T) { } func TestBlockHeight(t *testing.T) { - td := setup(t) + td := setup(t, nil) sb, _ := td.store.Block(1) @@ -71,7 +76,7 @@ func TestBlockHeight(t *testing.T) { } func TestUnknownTransactionID(t *testing.T) { - td := setup(t) + td := setup(t, nil) trx, err := td.store.Transaction(td.RandHash()) assert.Error(t, err) @@ -79,7 +84,7 @@ func TestUnknownTransactionID(t *testing.T) { } func TestWriteAndClosePeacefully(t *testing.T) { - td := setup(t) + td := setup(t, nil) // After closing db, we should not crash assert.NoError(t, td.store.Close()) @@ -87,7 +92,7 @@ func TestWriteAndClosePeacefully(t *testing.T) { } func TestRetrieveBlockAndTransactions(t *testing.T) { - td := setup(t) + td := setup(t, nil) lastCert := td.store.LastCertificate() lastHeight := lastCert.Height() @@ -109,7 +114,7 @@ func TestRetrieveBlockAndTransactions(t *testing.T) { } func TestIndexingPublicKeys(t *testing.T) { - td := setup(t) + td := setup(t, nil) committedBlock, _ := td.store.Block(1) blk, _ := committedBlock.ToBlock() @@ -140,7 +145,7 @@ func TestIndexingPublicKeys(t *testing.T) { } func TestStrippedPublicKey(t *testing.T) { - td := setup(t) + td := setup(t, nil) // Find a public key that we have already indexed in the database. committedBlock1, _ := td.store.Block(1) diff --git a/store/validator_test.go b/store/validator_test.go index aab47d47c..9e823416f 100644 --- a/store/validator_test.go +++ b/store/validator_test.go @@ -11,7 +11,7 @@ import ( ) func TestValidatorCounter(t *testing.T) { - td := setup(t) + td := setup(t, nil) num := td.RandInt32(1000) val, _ := td.GenerateTestValidator(num) @@ -46,7 +46,7 @@ func TestValidatorCounter(t *testing.T) { } func TestValidatorBatchSaving(t *testing.T) { - td := setup(t) + td := setup(t, nil) total := td.RandInt32NonZero(100) t.Run("Add some validators", func(t *testing.T) { @@ -66,7 +66,7 @@ func TestValidatorBatchSaving(t *testing.T) { } func TestValidatorAddresses(t *testing.T) { - td := setup(t) + td := setup(t, nil) total := td.RandInt32NonZero(100) addrs1 := make([]crypto.Address, 0, total) @@ -82,7 +82,7 @@ func TestValidatorAddresses(t *testing.T) { } func TestValidatorByNumber(t *testing.T) { - td := setup(t) + td := setup(t, nil) total := td.RandInt32NonZero(100) t.Run("Add some validators", func(t *testing.T) { @@ -131,7 +131,7 @@ func TestValidatorByNumber(t *testing.T) { } func TestValidatorByAddress(t *testing.T) { - td := setup(t) + td := setup(t, nil) total := td.RandInt32NonZero(100) t.Run("Add some validators", func(t *testing.T) { @@ -172,7 +172,7 @@ func TestValidatorByAddress(t *testing.T) { } func TestIterateValidators(t *testing.T) { - td := setup(t) + td := setup(t, nil) total := td.RandInt32NonZero(100) hashes1 := []hash.Hash{} @@ -201,7 +201,7 @@ func TestIterateValidators(t *testing.T) { } func TestValidatorDeepCopy(t *testing.T) { - td := setup(t) + td := setup(t, nil) num := td.RandInt32NonZero(1000) val1, _ := td.GenerateTestValidator(num) diff --git a/util/pairslice/pairslice.go b/util/pairslice/pairslice.go new file mode 100644 index 000000000..f2a024985 --- /dev/null +++ b/util/pairslice/pairslice.go @@ -0,0 +1,67 @@ +package pairslice + +import ( + "golang.org/x/exp/slices" +) + +type Pair[K comparable, V any] struct { + First K + Second V +} + +type PairSlice[K comparable, V any] struct { + pairs []*Pair[K, V] + index int +} + +// New creates a new instance of PairSlice with a specified capacity. +func New[K comparable, V any](capacity int) *PairSlice[K, V] { + return &PairSlice[K, V]{ + pairs: make([]*Pair[K, V], 0, capacity), + } +} + +// Append adds the Pair to the end of the slice. If the capacity is full, +// it automatically removes the first element and appends the new Pair to the end of the PairSlice. +func (ps *PairSlice[K, V]) Append(first K, second V) { + ps.pairs = append(ps.pairs, &Pair[K, V]{first, second}) +} + +// RemoveFirst removes the first element from PairSlice. +func (ps *PairSlice[K, V]) RemoveFirst() { + ps.remove(0) +} + +// RemoveLast removes the first element from PairSlice. +func (ps *PairSlice[K, V]) RemoveLast() { + ps.remove(ps.Len() - 2) +} + +func (ps *PairSlice[K, V]) Len() int { + return len(ps.pairs) +} + +func (ps *PairSlice[K, V]) remove(index int) { + ps.pairs = slices.Delete(ps.pairs, index, index+1) +} + +// Get returns the Pair at the specified index. If the index doesn't exist, it returns nil. +func (ps *PairSlice[K, V]) Get(index int) (K, V, bool) { + if index >= len(ps.pairs) || index < 0 { + var first K + var second V + return first, second, false + } + pair := ps.pairs[index] + return pair.First, pair.Second, true +} + +// First returns the first Pair in the PairSlice. +func (ps *PairSlice[K, V]) First() (K, V, bool) { + return ps.Get(0) +} + +// Last returns the last Pair in the PairSlice. +func (ps *PairSlice[K, V]) Last() (K, V, bool) { + return ps.Get(ps.index - 1) +} diff --git a/util/tripleslice/tripleslice_test.go b/util/pairslice/pairslice_test.go similarity index 62% rename from util/tripleslice/tripleslice_test.go rename to util/pairslice/pairslice_test.go index 0520a6abb..3ebcd9b3a 100644 --- a/util/tripleslice/tripleslice_test.go +++ b/util/pairslice/pairslice_test.go @@ -1,4 +1,4 @@ -package tripleslice +package pairslice import ( "testing" @@ -23,20 +23,20 @@ func TestPairSlice(t *testing.T) { ps.Append(1, "b") ps.Append(2, "c") - assert.Equal(t, &Triple[int, string]{2, "c", 2}, ps.pairs[2]) + assert.Equal(t, &Pair[int, string]{2, "c", 2}, ps.pairs[2]) }) - t.Run("Test Pop", func(t *testing.T) { + t.Run("Test RemoveFirst", func(t *testing.T) { ps := New[int, string](4) ps.Append(0, "a") ps.Append(1, "b") ps.Append(2, "c") - ps.Pop() + ps.RemoveFirst() - assert.Equal(t, &Triple[int, string]{1, "b", 1}, ps.pairs[0]) - assert.Equal(t, &Triple[int, string]{2, "c", 2}, ps.pairs[1]) + assert.Equal(t, &Pair[int, string]{1, "b", 1}, ps.pairs[0]) + assert.Equal(t, &Pair[int, string]{2, "c", 2}, ps.pairs[1]) assert.Nil(t, ps.pairs[2]) }) @@ -59,8 +59,8 @@ func TestPairSlice(t *testing.T) { ps.Append(1, "b") ps.Append(2, "c") - assert.Equal(t, &Triple[int, string]{0, "a", 0}, ps.Get(0)) - assert.Equal(t, &Triple[int, string]{2, "c", 2}, ps.Get(2)) + assert.Equal(t, &Pair[int, string]{0, "a", 0}, ps.Get(0)) + assert.Equal(t, &Pair[int, string]{2, "c", 2}, ps.Get(2)) assert.Nil(t, ps.Get(10)) }) @@ -71,7 +71,7 @@ func TestPairSlice(t *testing.T) { ps.Append(1, "b") ps.Append(2, "c") - assert.Equal(t, &Triple[int, string]{0, "a", 0}, ps.First()) + assert.Equal(t, &Pair[int, string]{0, "a", 0}, ps.First()) }) t.Run("Test Last", func(t *testing.T) { @@ -81,10 +81,10 @@ func TestPairSlice(t *testing.T) { ps.Append(1, "b") ps.Append(2, "c") - assert.Equal(t, &Triple[int, string]{2, "c", 2}, ps.Last()) + assert.Equal(t, &Pair[int, string]{2, "c", 2}, ps.Last()) }) - t.Run("Test Last with Pop", func(t *testing.T) { + t.Run("Test Last with RemoveFirst", func(t *testing.T) { ps := New[int, string](4) ps.Append(0, "a") @@ -92,9 +92,9 @@ func TestPairSlice(t *testing.T) { ps.Append(2, "c") ps.Append(3, "d") - ps.Pop() + ps.RemoveFirst() - assert.Equal(t, &Triple[int, string]{3, "d", 3}, ps.Last()) + assert.Equal(t, &Pair[int, string]{3, "d", 3}, ps.Last()) }) t.Run("Test All", func(t *testing.T) { @@ -106,11 +106,11 @@ func TestPairSlice(t *testing.T) { all := ps.All() - assert.Equal(t, all[0].FirstElement, 0) - assert.Equal(t, all[1].FirstElement, 1) - assert.Equal(t, all[2].FirstElement, 2) - assert.Equal(t, all[0].SecondElement, "a") - assert.Equal(t, all[1].SecondElement, "b") - assert.Equal(t, all[2].SecondElement, "c") + assert.Equal(t, all[0].First, 0) + assert.Equal(t, all[1].First, 1) + assert.Equal(t, all[2].First, 2) + assert.Equal(t, all[0].Second, "a") + assert.Equal(t, all[1].Second, "b") + assert.Equal(t, all[2].Second, "c") }) } diff --git a/util/tripleslice/tripleslice.go b/util/tripleslice/tripleslice.go deleted file mode 100644 index cb2f92ce2..000000000 --- a/util/tripleslice/tripleslice.go +++ /dev/null @@ -1,81 +0,0 @@ -package tripleslice - -type Triple[K comparable, V any] struct { - FirstElement K - SecondElement V - ThirdElement int -} - -type TripleSlice[K comparable, V any] struct { - pairs []*Triple[K, V] - index int - capacity int -} - -// New creates a new instance of TripleSlice with a specified capacity. -func New[K comparable, V any](capacity int) *TripleSlice[K, V] { - return &TripleSlice[K, V]{ - pairs: make([]*Triple[K, V], capacity), - capacity: capacity, - } -} - -// Append adds the Triple to the end of the slice. If the capacity is full, -// it automatically removes the first element and appends the new Triple to the end of the TripleSlice. -func (ps *TripleSlice[K, V]) Append(firstElement K, secondElement V) { - ps.pairs[ps.index] = &Triple[K, V]{firstElement, secondElement, ps.index} - ps.index++ - ps.resetIndex() -} - -// Pop removes the first element from TripleSlice. -func (ps *TripleSlice[K, V]) Pop() { - ps.pairs = ps.pairs[1:] - ps.index-- - ps.resetIndex() -} - -// Has checks the index exists or not. -func (ps *TripleSlice[K, V]) Has(index int) bool { - if index >= ps.capacity { - return false - } - - return ps.pairs[index] != nil -} - -// Get returns the Triple at the specified index. If the index doesn't exist, it returns nil. -func (ps *TripleSlice[K, V]) Get(index int) *Triple[K, V] { - if index >= ps.capacity { - return nil - } - return ps.pairs[index] -} - -// First returns the first Triple in the TripleSlice. -func (ps *TripleSlice[K, V]) First() *Triple[K, V] { - return ps.Get(0) -} - -// Last returns the last Triple in the TripleSlice. -func (ps *TripleSlice[K, V]) Last() *Triple[K, V] { - if ps.index == 0 { - return ps.Get(len(ps.pairs) - 1) - } - return ps.Get(ps.index - 1) -} - -// All returns whole pairs of slice. -func (ps *TripleSlice[K, V]) All() []*Triple[K, V] { - return ps.pairs -} - -func (ps *TripleSlice[K, V]) resetIndex() { - if ps.index < 0 { - ps.index = 0 - } - - if ps.index == ps.capacity { - ps.index = 0 - } -} From c9d300c3da420b951e0a355781363b6c612dbf01 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sun, 31 Dec 2023 00:27:32 +0330 Subject: [PATCH 27/30] test: implement tests for pair slice --- util/pairslice/pairslice.go | 5 +- util/pairslice/pairslice_test.go | 127 +++++++++++++++++-------------- 2 files changed, 73 insertions(+), 59 deletions(-) diff --git a/util/pairslice/pairslice.go b/util/pairslice/pairslice.go index f2a024985..05f5d2904 100644 --- a/util/pairslice/pairslice.go +++ b/util/pairslice/pairslice.go @@ -11,7 +11,6 @@ type Pair[K comparable, V any] struct { type PairSlice[K comparable, V any] struct { pairs []*Pair[K, V] - index int } // New creates a new instance of PairSlice with a specified capacity. @@ -34,7 +33,7 @@ func (ps *PairSlice[K, V]) RemoveFirst() { // RemoveLast removes the first element from PairSlice. func (ps *PairSlice[K, V]) RemoveLast() { - ps.remove(ps.Len() - 2) + ps.remove(ps.Len() - 1) } func (ps *PairSlice[K, V]) Len() int { @@ -63,5 +62,5 @@ func (ps *PairSlice[K, V]) First() (K, V, bool) { // Last returns the last Pair in the PairSlice. func (ps *PairSlice[K, V]) Last() (K, V, bool) { - return ps.Get(ps.index - 1) + return ps.Get(ps.Len() - 1) } diff --git a/util/pairslice/pairslice_test.go b/util/pairslice/pairslice_test.go index 3ebcd9b3a..92315adfb 100644 --- a/util/pairslice/pairslice_test.go +++ b/util/pairslice/pairslice_test.go @@ -10,107 +10,122 @@ func TestNew(t *testing.T) { ps := New[int, string](10) assert.NotNil(t, ps) - assert.Equal(t, 10, len(ps.pairs)) - assert.Equal(t, 0, ps.index) - assert.Equal(t, 10, ps.capacity) + assert.Equal(t, 10, cap(ps.pairs)) + assert.Equal(t, 0, len(ps.pairs)) } func TestPairSlice(t *testing.T) { + t.Run("Test Append", func(t *testing.T) { ps := New[int, string](4) - ps.Append(0, "a") - ps.Append(1, "b") - ps.Append(2, "c") + ps.Append(1, "a") + ps.Append(2, "b") + ps.Append(3, "c") - assert.Equal(t, &Pair[int, string]{2, "c", 2}, ps.pairs[2]) + assert.Equal(t, 3, ps.Len()) + assert.Equal(t, ps.pairs[2].First, 3) + assert.Equal(t, ps.pairs[2].Second, "c") }) t.Run("Test RemoveFirst", func(t *testing.T) { ps := New[int, string](4) - ps.Append(0, "a") - ps.Append(1, "b") - ps.Append(2, "c") + ps.Append(1, "a") + ps.Append(2, "b") + ps.Append(3, "c") ps.RemoveFirst() - assert.Equal(t, &Pair[int, string]{1, "b", 1}, ps.pairs[0]) - assert.Equal(t, &Pair[int, string]{2, "c", 2}, ps.pairs[1]) - assert.Nil(t, ps.pairs[2]) + assert.Equal(t, ps.pairs[0].First, 2) + assert.Equal(t, ps.pairs[0].Second, "b") }) - t.Run("Test Has", func(t *testing.T) { + t.Run("Test RemoveLast", func(t *testing.T) { ps := New[int, string](4) - ps.Append(0, "a") - ps.Append(1, "b") - ps.Append(2, "c") + ps.Append(1, "a") + ps.Append(2, "b") + ps.Append(3, "c") + ps.Append(4, "d") + + ps.RemoveLast() - assert.True(t, ps.Has(0)) - assert.False(t, ps.Has(3)) - assert.False(t, ps.Has(10)) + assert.Equal(t, ps.pairs[2].First, 3) + assert.Equal(t, ps.pairs[2].Second, "c") }) - t.Run("Test Get", func(t *testing.T) { + t.Run("Test Len", func(t *testing.T) { ps := New[int, string](4) - ps.Append(0, "a") - ps.Append(1, "b") - ps.Append(2, "c") + ps.Append(1, "a") + ps.Append(2, "b") - assert.Equal(t, &Pair[int, string]{0, "a", 0}, ps.Get(0)) - assert.Equal(t, &Pair[int, string]{2, "c", 2}, ps.Get(2)) - assert.Nil(t, ps.Get(10)) + assert.Equal(t, 2, ps.Len()) }) - t.Run("Test First", func(t *testing.T) { + t.Run("Test Remove", func(t *testing.T) { ps := New[int, string](4) - ps.Append(0, "a") - ps.Append(1, "b") - ps.Append(2, "c") + ps.Append(1, "a") + ps.Append(2, "b") + ps.Append(3, "c") + ps.Append(4, "d") - assert.Equal(t, &Pair[int, string]{0, "a", 0}, ps.First()) + ps.remove(1) + + assert.Equal(t, ps.pairs[1].First, 3) + assert.Equal(t, ps.pairs[1].Second, "c") }) - t.Run("Test Last", func(t *testing.T) { + t.Run("Test Get", func(t *testing.T) { ps := New[int, string](4) - ps.Append(0, "a") - ps.Append(1, "b") - ps.Append(2, "c") + ps.Append(1, "a") + ps.Append(2, "b") + ps.Append(3, "c") + ps.Append(4, "d") - assert.Equal(t, &Pair[int, string]{2, "c", 2}, ps.Last()) + first, second, _ := ps.Get(2) + assert.Equal(t, ps.pairs[2].First, first) + assert.Equal(t, ps.pairs[2].Second, second) }) - t.Run("Test Last with RemoveFirst", func(t *testing.T) { + t.Run("Test Get negative index or bigger than len", func(t *testing.T) { ps := New[int, string](4) - ps.Append(0, "a") - ps.Append(1, "b") - ps.Append(2, "c") - ps.Append(3, "d") + ps.Append(1, "a") + ps.Append(4, "d") - ps.RemoveFirst() - - assert.Equal(t, &Pair[int, string]{3, "d", 3}, ps.Last()) + _, _, result1 := ps.Get(-1) + _, _, result2 := ps.Get(10) + assert.False(t, result1) + assert.False(t, result2) }) - t.Run("Test All", func(t *testing.T) { + t.Run("Test First", func(t *testing.T) { ps := New[int, string](4) - ps.Append(0, "a") - ps.Append(1, "b") - ps.Append(2, "c") + ps.Append(1, "a") + ps.Append(2, "b") + ps.Append(3, "c") + ps.Append(4, "d") + + first, second, _ := ps.First() + assert.Equal(t, ps.pairs[0].First, first) + assert.Equal(t, ps.pairs[0].Second, second) + }) + + t.Run("Test Last", func(t *testing.T) { + ps := New[int, string](4) - all := ps.All() + ps.Append(1, "a") + ps.Append(2, "b") + ps.Append(3, "c") + ps.Append(4, "d") - assert.Equal(t, all[0].First, 0) - assert.Equal(t, all[1].First, 1) - assert.Equal(t, all[2].First, 2) - assert.Equal(t, all[0].Second, "a") - assert.Equal(t, all[1].Second, "b") - assert.Equal(t, all[2].Second, "c") + first, second, _ := ps.Last() + assert.Equal(t, ps.pairs[3].First, first) + assert.Equal(t, ps.pairs[3].Second, second) }) } From 254ff409aa1315192d13306c2ebdbe0eac26c85b Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sun, 31 Dec 2023 00:31:55 +0330 Subject: [PATCH 28/30] docs: write docs for pairslice --- util/pairslice/pairslice.go | 17 ++++++++++------- 1 file changed, 10 insertions(+), 7 deletions(-) diff --git a/util/pairslice/pairslice.go b/util/pairslice/pairslice.go index 05f5d2904..65b176080 100644 --- a/util/pairslice/pairslice.go +++ b/util/pairslice/pairslice.go @@ -4,11 +4,13 @@ import ( "golang.org/x/exp/slices" ) +// Pair represents a key-value pair. type Pair[K comparable, V any] struct { First K Second V } +// PairSlice represents a slice of key-value pairs. type PairSlice[K comparable, V any] struct { pairs []*Pair[K, V] } @@ -20,8 +22,7 @@ func New[K comparable, V any](capacity int) *PairSlice[K, V] { } } -// Append adds the Pair to the end of the slice. If the capacity is full, -// it automatically removes the first element and appends the new Pair to the end of the PairSlice. +// Append adds the first and second to the end of the slice. func (ps *PairSlice[K, V]) Append(first K, second V) { ps.pairs = append(ps.pairs, &Pair[K, V]{first, second}) } @@ -31,22 +32,24 @@ func (ps *PairSlice[K, V]) RemoveFirst() { ps.remove(0) } -// RemoveLast removes the first element from PairSlice. +// RemoveLast removes the last element from PairSlice. func (ps *PairSlice[K, V]) RemoveLast() { ps.remove(ps.Len() - 1) } +// Len returns the number of elements in the PairSlice. func (ps *PairSlice[K, V]) Len() int { return len(ps.pairs) } +// remove removes the element at the specified index from PairSlice. func (ps *PairSlice[K, V]) remove(index int) { ps.pairs = slices.Delete(ps.pairs, index, index+1) } -// Get returns the Pair at the specified index. If the index doesn't exist, it returns nil. +// Get returns the properties at the specified index. If the index is out of bounds, it returns false. func (ps *PairSlice[K, V]) Get(index int) (K, V, bool) { - if index >= len(ps.pairs) || index < 0 { + if index < 0 || index >= len(ps.pairs) { var first K var second V return first, second, false @@ -55,12 +58,12 @@ func (ps *PairSlice[K, V]) Get(index int) (K, V, bool) { return pair.First, pair.Second, true } -// First returns the first Pair in the PairSlice. +// First returns the first properties in the PairSlice. If the PairSlice is empty, it returns false. func (ps *PairSlice[K, V]) First() (K, V, bool) { return ps.Get(0) } -// Last returns the last Pair in the PairSlice. +// Last returns the last properties in the PairSlice. If the PairSlice is empty, it returns false. func (ps *PairSlice[K, V]) Last() (K, V, bool) { return ps.Get(ps.Len() - 1) } From 296a5b40214fe48d0df4865d78a4f0eb8fa4ffb0 Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sun, 31 Dec 2023 00:37:56 +0330 Subject: [PATCH 29/30] fix: lint issue --- util/pairslice/pairslice_test.go | 1 - 1 file changed, 1 deletion(-) diff --git a/util/pairslice/pairslice_test.go b/util/pairslice/pairslice_test.go index 92315adfb..94820dce1 100644 --- a/util/pairslice/pairslice_test.go +++ b/util/pairslice/pairslice_test.go @@ -15,7 +15,6 @@ func TestNew(t *testing.T) { } func TestPairSlice(t *testing.T) { - t.Run("Test Append", func(t *testing.T) { ps := New[int, string](4) From 2e6d417126eed2150939c64702c44590e71169dc Mon Sep 17 00:00:00 2001 From: amirvalhalla Date: Sun, 31 Dec 2023 12:10:47 +0330 Subject: [PATCH 30/30] fix: clone account for cache --- store/account.go | 2 +- store/tx.go | 12 ++++++------ 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/store/account.go b/store/account.go index 0a029c3d0..453328794 100644 --- a/store/account.go +++ b/store/account.go @@ -62,7 +62,7 @@ func (as *accountStore) account(addr crypto.Address) (*account.Account, error) { return nil, err } - as.accInCache.Add(addr, acc) + as.accInCache.Add(addr, acc.Clone()) return acc, nil } diff --git a/store/tx.go b/store/tx.go index d252c6a76..16e28cbe4 100644 --- a/store/tx.go +++ b/store/tx.go @@ -20,14 +20,14 @@ func txKey(id tx.ID) []byte { return append(txPrefix, id.Bytes()...) } type txStore struct { db *leveldb.DB - txCache *linkedmap.LinkedMap[tx.ID, uint32] + txIDCache *linkedmap.LinkedMap[tx.ID, uint32] txCacheSize uint32 } func newTxStore(db *leveldb.DB, txCacheSize uint32) *txStore { return &txStore{ db: db, - txCache: linkedmap.New[tx.ID, uint32](0), + txIDCache: linkedmap.New[tx.ID, uint32](0), txCacheSize: txCacheSize, } } @@ -51,18 +51,18 @@ func (ts *txStore) saveTxs(batch *leveldb.Batch, txs block.Txs, regs []blockRegi func (ts *txStore) pruneCache(currentHeight uint32) { for { - head := ts.txCache.HeadNode() + head := ts.txIDCache.HeadNode() txHeight := head.Data.Value if currentHeight-txHeight <= ts.txCacheSize { break } - ts.txCache.RemoveHead() + ts.txIDCache.RemoveHead() } } func (ts *txStore) hasTX(id tx.ID) bool { - return ts.txCache.Has(id) + return ts.txIDCache.Has(id) } func (ts *txStore) tx(id tx.ID) (*blockRegion, error) { @@ -79,5 +79,5 @@ func (ts *txStore) tx(id tx.ID) (*blockRegion, error) { } func (ts *txStore) saveToCache(id tx.ID, height uint32) { - ts.txCache.PushBack(id, height) + ts.txIDCache.PushBack(id, height) }