From e6dab903dcf18b00a6f316b10333b284676f2625 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Wed, 7 Dec 2022 17:18:23 +0800 Subject: [PATCH 1/4] make LazyLoadVersion do the same thing as LoadVersion --- CHANGELOG.md | 1 + mutable_tree.go | 40 ++++++++++++++++++++++++++++++++++------ nodedb.go | 15 +++++++++++++++ 3 files changed, 50 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dad0076bf..0ced4f1af 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - [#586](https://github.com/cosmos/iavl/pull/586) Remove the `RangeProof` and refactor the ics23_proof to use the internal methods. - [#640](https://github.com/cosmos/iavl/pull/640) commit `NodeDB` batch in `LoadVersionForOverwriting`. +- [#638](https://github.com/cosmos/iavl/pull/638) Make LazyLoadVersion check the opts.InitialVersion, add API `LazyLoadVersionForOverwriting`. ## 0.19.4 (October 28, 2022) diff --git a/mutable_tree.go b/mutable_tree.go index 4924264cd..e5e812979 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -462,17 +462,26 @@ func (tree *MutableTree) Load() (int64, error) { } // LazyLoadVersion attempts to lazy load only the specified target version -// without loading previous roots/versions. Lazy loading should be used in cases -// where only reads are expected. Any writes to a lazy loaded tree may result in -// unexpected behavior. If the targetVersion is non-positive, the latest version +// without loading previous roots/versions. If the targetVersion is non-positive, the latest version // will be loaded by default. If the latest version is non-positive, this method // performs a no-op. Otherwise, if the root does not exist, an error will be // returned. func (tree *MutableTree) LazyLoadVersion(targetVersion int64) (int64, error) { + firstVersion, err := tree.ndb.firstVersion() + if err != nil { + return 0, err + } + latestVersion, err := tree.ndb.getLatestVersion() if err != nil { return 0, err } + + if firstVersion > 0 && firstVersion < int64(tree.ndb.opts.InitialVersion) { + return latestVersion, fmt.Errorf("initial version set to %v, but found earlier version %v", + tree.ndb.opts.InitialVersion, firstVersion) + } + if latestVersion < targetVersion { return latestVersion, fmt.Errorf("wanted to load target %d but only found up to %d", targetVersion, latestVersion) } @@ -613,10 +622,18 @@ func (tree *MutableTree) LoadVersion(targetVersion int64) (int64, error) { return latestVersion, nil } -// LoadVersionForOverwriting attempts to load a tree at a previously committed +// loadVersionForOverwriting attempts to load a tree at a previously committed // version, or the latest version below it. Any versions greater than targetVersion will be deleted. -func (tree *MutableTree) LoadVersionForOverwriting(targetVersion int64) (int64, error) { - latestVersion, err := tree.LoadVersion(targetVersion) +func (tree *MutableTree) loadVersionForOverwriting(targetVersion int64, lazy bool) (int64, error) { + var ( + latestVersion int64 + err error + ) + if lazy { + latestVersion, err = tree.LazyLoadVersion(targetVersion) + } else { + latestVersion, err = tree.LoadVersion(targetVersion) + } if err != nil { return latestVersion, err } @@ -652,6 +669,17 @@ func (tree *MutableTree) LoadVersionForOverwriting(targetVersion int64) (int64, return latestVersion, nil } +// LoadVersionForOverwriting attempts to load a tree at a previously committed +// version, or the latest version below it. Any versions greater than targetVersion will be deleted. +func (tree *MutableTree) LoadVersionForOverwriting(targetVersion int64) (int64, error) { + return tree.loadVersionForOverwriting(targetVersion, false) +} + +// LazyLoadVersionForOverwriting is the lazy version of `LoadVersionForOverwriting`. +func (tree *MutableTree) LazyLoadVersionForOverwriting(targetVersion int64) (int64, error) { + return tree.loadVersionForOverwriting(targetVersion, true) +} + // Returns true if the tree may be auto-upgraded, false otherwise // An example of when an upgrade may be performed is when we are enaling fast storage for the first time or // need to overwrite fast nodes due to mismatch with live state. diff --git a/nodedb.go b/nodedb.go index 2d00d01ac..89a8465f3 100644 --- a/nodedb.go +++ b/nodedb.go @@ -767,6 +767,21 @@ func (ndb *nodeDB) getPreviousVersion(version int64) (int64, error) { return 0, nil } +// firstVersion returns the first version in the iavl tree, returns 0 if it's empty. +func (ndb *nodeDB) firstVersion() (int64, error) { + itr, err := dbm.IteratePrefix(ndb.db, rootKeyFormat.Key()) + if err != nil { + return 0, err + } + defer itr.Close() + if itr.Valid() { + var version int64 + rootKeyFormat.Scan(itr.Key(), &version) + return version, nil + } + return 0, nil +} + // deleteRoot deletes the root entry from disk, but not the node it points to. func (ndb *nodeDB) deleteRoot(version int64, checkLatestVersion bool) error { latestVersion, err := ndb.getLatestVersion() From cf257af53adfb3494988cddef6e20bcddf401740 Mon Sep 17 00:00:00 2001 From: yihuang Date: Mon, 16 Jan 2023 16:20:32 +0800 Subject: [PATCH 2/4] Update CHANGELOG.md --- CHANGELOG.md | 2 ++ 1 file changed, 2 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0ced4f1af..4faa2f480 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,8 @@ - [#586](https://github.com/cosmos/iavl/pull/586) Remove the `RangeProof` and refactor the ics23_proof to use the internal methods. - [#640](https://github.com/cosmos/iavl/pull/640) commit `NodeDB` batch in `LoadVersionForOverwriting`. +- [#636](https://github.com/cosmos/iavl/pull/636) Speed up rollback method: `LoadVersionForOverwriting`. +- [#654](https://github.com/cosmos/iavl/pull/654) Add API `TraverseStateChanges` to extract state changes from iavl versions. - [#638](https://github.com/cosmos/iavl/pull/638) Make LazyLoadVersion check the opts.InitialVersion, add API `LazyLoadVersionForOverwriting`. ## 0.19.4 (October 28, 2022) From 8a822342ad02c6fefcf7a7f74ecf165d6314bf19 Mon Sep 17 00:00:00 2001 From: HuangYi Date: Tue, 17 Jan 2023 10:31:53 +0800 Subject: [PATCH 3/4] add unit test --- mutable_tree_test.go | 12 ++++++++++++ 1 file changed, 12 insertions(+) diff --git a/mutable_tree_test.go b/mutable_tree_test.go index c6862e0ec..f8afce61a 100644 --- a/mutable_tree_test.go +++ b/mutable_tree_test.go @@ -385,12 +385,20 @@ func TestMutableTree_InitialVersion(t *testing.T) { require.NoError(t, err) assert.EqualValues(t, 10, version) + // Check `LazyLoadVersion` behaviors the same as `LoadVersion` + version, err = tree.LazyLoadVersion(0) + require.NoError(t, err) + assert.EqualValues(t, 10, version) + // Reloading the tree with an initial version beyond the lowest should error tree, err = NewMutableTreeWithOpts(memDB, 0, &Options{InitialVersion: 10}, false) require.NoError(t, err) _, err = tree.Load() require.Error(t, err) + _, err = tree.LazyLoadVersion(0) + require.Error(t, err) + // Reloading the tree with a lower initial version is fine, and new versions can be produced tree, err = NewMutableTreeWithOpts(memDB, 0, &Options{InitialVersion: 3}, false) require.NoError(t, err) @@ -398,6 +406,10 @@ func TestMutableTree_InitialVersion(t *testing.T) { require.NoError(t, err) assert.EqualValues(t, 10, version) + version, err = tree.LazyLoadVersion(0) + require.NoError(t, err) + assert.EqualValues(t, 10, version) + _, err = tree.Set([]byte("c"), []byte{0x03}) require.NoError(t, err) _, version, err = tree.SaveVersion() From fc765fedea07339f40e8f9407fc30ea1c3405a0b Mon Sep 17 00:00:00 2001 From: HuangYi Date: Wed, 18 Jan 2023 09:36:14 +0800 Subject: [PATCH 4/4] rename --- mutable_tree.go | 2 +- nodedb.go | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/mutable_tree.go b/mutable_tree.go index c2e0d8794..ef1d732a7 100644 --- a/mutable_tree.go +++ b/mutable_tree.go @@ -467,7 +467,7 @@ func (tree *MutableTree) Load() (int64, error) { // performs a no-op. Otherwise, if the root does not exist, an error will be // returned. func (tree *MutableTree) LazyLoadVersion(targetVersion int64) (int64, error) { - firstVersion, err := tree.ndb.firstVersion() + firstVersion, err := tree.ndb.getFirstVersion() if err != nil { return 0, err } diff --git a/nodedb.go b/nodedb.go index 27af4c260..e49ddb78b 100644 --- a/nodedb.go +++ b/nodedb.go @@ -761,8 +761,8 @@ func (ndb *nodeDB) getPreviousVersion(version int64) (int64, error) { return 0, nil } -// firstVersion returns the first version in the iavl tree, returns 0 if it's empty. -func (ndb *nodeDB) firstVersion() (int64, error) { +// getFirstVersion returns the first version in the iavl tree, returns 0 if it's empty. +func (ndb *nodeDB) getFirstVersion() (int64, error) { itr, err := dbm.IteratePrefix(ndb.db, rootKeyFormat.Key()) if err != nil { return 0, err