From 4fd49eddd19050abd0552c4adcacc79357eab5bb Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 2 Jul 2018 14:43:50 -0700 Subject: [PATCH 1/3] 0.9.1 -- compute hash from rangeproof --- CHANGELOG.md | 6 ++++ proof_path.go | 30 ++++++++++++++--- proof_range.go | 89 +++++++++++++++++++++++++++++++------------------- proof_test.go | 4 --- version.go | 2 +- 5 files changed, 88 insertions(+), 43 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index dbf8404bd..5bf969b49 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,11 @@ # Changelog +## 0.9.1 (July 1, 2018) + +IMPROVEMENTS + +- RangeProof.ComputeRootHash() to compute root rather than provide as in Verify(hash) + ## 0.9.0 (July 1, 2018) BREAKING CHANGES diff --git a/proof_path.go b/proof_path.go index 45db016cc..dbde9fa10 100644 --- a/proof_path.go +++ b/proof_path.go @@ -28,8 +28,19 @@ func (pwl pathWithLeaf) StringIndented(indent string) string { indent) } +// `verify` checks that the leaf node's hash + the inner nodes merkle-izes to +// the given root. If it returns an error, it means the leafHash or the +// PathToLeaf is incorrect. func (pwl pathWithLeaf) verify(root []byte) cmn.Error { - return pwl.Path.verify(pwl.Leaf.Hash(), root) + leafHash := pwl.Leaf.Hash() + return pwl.Path.verify(leafHash, root) +} + +// `computeRootHash` computes the root hash with leaf node. +// Does not verify the root hash. +func (pwl pathWithLeaf) computeRootHash() []byte { + leafHash := pwl.Leaf.Hash() + return pwl.Path.computeRootHash(leafHash) } //---------------------------------------- @@ -62,9 +73,9 @@ func (pl PathToLeaf) StringIndented(indent string) string { indent) } -// verify checks that the leaf node's hash + the inner nodes merkle-izes to the -// given root. If it returns an error, it means the leafHash or the PathToLeaf -// is incorrect. +// `verify` checks that the leaf node's hash + the inner nodes merkle-izes to +// the given root. If it returns an error, it means the leafHash or the +// PathToLeaf is incorrect. func (pl PathToLeaf) verify(leafHash []byte, root []byte) cmn.Error { hash := leafHash for i := len(pl) - 1; i >= 0; i-- { @@ -77,6 +88,17 @@ func (pl PathToLeaf) verify(leafHash []byte, root []byte) cmn.Error { return nil } +// `computeRootHash` computes the root hash assuming some leaf hash. +// Does not verify the root hash. +func (pl PathToLeaf) computeRootHash(leafHash []byte) []byte { + hash := leafHash + for i := len(pl) - 1; i >= 0; i-- { + pin := pl[i] + hash = pin.Hash(hash) + } + return hash +} + func (pl PathToLeaf) isLeftmost() bool { for _, node := range pl { if len(node.Left) > 0 { diff --git a/proof_range.go b/proof_range.go index 6d2b95a3c..c5377b0db 100644 --- a/proof_range.go +++ b/proof_range.go @@ -13,7 +13,6 @@ import ( type RangeProof struct { // You don't need the right path because // it can be derived from what we have. - RootHash cmn.HexBytes `json:"root_hash"` LeftPath PathToLeaf `json:"left_path"` InnerNodes []PathToLeaf `json:"inner_nodes"` Leaves []proofLeafNode `json:"leaves"` @@ -49,7 +48,6 @@ func (proof *RangeProof) StringIndented(indent string) string { lstrs = append(lstrs, leaf.StringIndented(indent+" ")) } return fmt.Sprintf(`RangeProof{ -%s RootHash: %X %s LeftPath: %v %s InnerNodes: %s %v @@ -57,7 +55,6 @@ func (proof *RangeProof) StringIndented(indent string) string { %s %v %s (treeEnd): %v %s}`, - indent, proof.RootHash, indent, proof.LeftPath.StringIndented(indent+" "), indent, indent, strings.Join(istrs, "\n"+indent+" "), @@ -161,7 +158,33 @@ func (proof *RangeProof) Verify(root []byte) error { if proof == nil { return cmn.ErrorWrap(ErrInvalidProof, "proof is nil") } - treeEnd, err := proof._verify(root) + err := proof.verify(root) + return err +} + +func (proof *RangeProof) verify(root []byte) (err error) { + rootHash, err := proof.computeRootHash() + if err != nil { + return err + } else if !bytes.Equal(rootHash, root) { + return cmn.ErrorWrap(ErrInvalidRoot, "root hash doesn't match") + } + return nil +} + +// ComputeRootHash computes the root hash with leaves. +// Returns nil if error or proof is nil. +// Does not verify the root hash. +func (proof *RangeProof) ComputeRootHash() []byte { + if proof == nil { + return nil + } + rootHash, _ := proof.computeRootHash() + return rootHash +} + +func (proof *RangeProof) computeRootHash() (rootHash []byte, err error) { + rootHash, treeEnd, err := proof._computeRootHash() if err == nil { if treeEnd { proof.treeEnd = 1 // memoize @@ -169,18 +192,15 @@ func (proof *RangeProof) Verify(root []byte) error { proof.treeEnd = -1 // memoize } } - return err + return rootHash, err } -func (proof *RangeProof) _verify(root []byte) (treeEnd bool, err error) { - if !bytes.Equal(proof.RootHash, root) { - return false, cmn.ErrorWrap(ErrInvalidRoot, "root hash doesn't match") - } +func (proof *RangeProof) _computeRootHash() (rootHash []byte, treeEnd bool, err error) { if len(proof.Leaves) == 0 { - return false, cmn.ErrorWrap(ErrInvalidProof, "no leaves") + return nil, false, cmn.ErrorWrap(ErrInvalidProof, "no leaves") } if len(proof.InnerNodes)+1 != len(proof.Leaves) { - return false, cmn.ErrorWrap(ErrInvalidProof, "InnerNodes vs Leaves length mismatch, leaves should be 1 more.") + return nil, false, cmn.ErrorWrap(ErrInvalidProof, "InnerNodes vs Leaves length mismatch, leaves should be 1 more.") } // Start from the left path and prove each leaf. @@ -188,28 +208,27 @@ func (proof *RangeProof) _verify(root []byte) (treeEnd bool, err error) { // shared across recursive calls var leaves = proof.Leaves var innersq = proof.InnerNodes - var VERIFY func(path PathToLeaf, root []byte, rightmost bool) (treeEnd bool, done bool, err error) + var COMPUTEHASH func(path PathToLeaf, rightmost bool) (hash []byte, treeEnd bool, done bool, err error) + // rightmost: is the root a rightmost child of the tree? // treeEnd: true iff the last leaf is the last item of the tree. - // NOTE: root doesn't necessarily mean root of the tree here. - VERIFY = func(path PathToLeaf, root []byte, rightmost bool) (treeEnd bool, done bool, err error) { + // Returns the (possibly intermediate, possibly root) hash. + COMPUTEHASH = func(path PathToLeaf, rightmost bool) (hash []byte, treeEnd bool, done bool, err error) { // Pop next leaf. nleaf, rleaves := leaves[0], leaves[1:] leaves = rleaves - // Verify leaf with path. - if err := (pathWithLeaf{ + // Compute hash. + hash = (pathWithLeaf{ Path: path, Leaf: nleaf, - }).verify(root); err != nil { - return false, false, err.Trace(0, "verifying some path to a leaf") - } + }).computeRootHash() // If we don't have any leaves left, we're done. if len(leaves) == 0 { rightmost = rightmost && path.isRightmost() - return rightmost, true, nil + return hash, rightmost, true, nil } // Prove along path (until we run out of leaves). @@ -231,31 +250,35 @@ func (proof *RangeProof) _verify(root []byte) (treeEnd bool, err error) { innersq = rinnersq // Recursively verify inners against remaining leaves. - treeEnd, done, err := VERIFY(inners, lpath.Right, rightmost && rpath.isRightmost()) + derivedRoot, treeEnd, done, err := COMPUTEHASH(inners, rightmost && rpath.isRightmost()) if err != nil { - return treeEnd, false, cmn.ErrorWrap(err, "recursive VERIFY call") - } else if done { - return treeEnd, true, nil + return nil, treeEnd, false, cmn.ErrorWrap(err, "recursive COMPUTEHASH call") + } + if !bytes.Equal(derivedRoot, lpath.Right) { + return nil, treeEnd, false, cmn.ErrorWrap(ErrInvalidRoot, "intermediate root hash %X doesn't match, got %X", lpath.Right, derivedRoot) + } + if done { + return hash, treeEnd, true, nil } } - // We're not done yet. No error, not done either. Technically if - // rightmost, we know there's an error "left over leaves -- malformed - // proof", but we return that at the top level, below. - return false, false, nil + // We're not done yet (leaves left over). No error, not done either. + // Technically if rightmost, we know there's an error "left over leaves + // -- malformed proof", but we return that at the top level, below. + return hash, false, false, nil } // Verify! path := proof.LeftPath - treeEnd, done, err := VERIFY(path, root, true) + rootHash, treeEnd, done, err := COMPUTEHASH(path, true) if err != nil { - return treeEnd, cmn.ErrorWrap(err, "root VERIFY call") + return nil, treeEnd, cmn.ErrorWrap(err, "root COMPUTEHASH call") } else if !done { - return treeEnd, cmn.ErrorWrap(ErrInvalidProof, "left over leaves -- malformed proof") + return nil, treeEnd, cmn.ErrorWrap(ErrInvalidProof, "left over leaves -- malformed proof") } // Ok! - return treeEnd, nil + return rootHash, treeEnd, nil } /////////////////////////////////////////////////////////////////////////////// @@ -309,7 +332,6 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr } if _stop { return &RangeProof{ - RootHash: t.root.hash, LeftPath: path, Leaves: leaves, }, keys, values, nil @@ -398,7 +420,6 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr ) return &RangeProof{ - RootHash: t.root.hash, LeftPath: path, InnerNodes: innersq, Leaves: leaves, diff --git a/proof_test.go b/proof_test.go index 75e65e3a1..64522a9e0 100644 --- a/proof_test.go +++ b/proof_test.go @@ -223,10 +223,6 @@ func verifyProof(t *testing.T, proof *RangeProof, root []byte) { proofBytes, badProofBytes) } } - - // targetted changes fails... - proof.RootHash = test.MutateByteSlice(proof.RootHash) - assert.Error(t, proof.Verify(root)) } //---------------------------------------- diff --git a/version.go b/version.go index c2e85365f..4c734cb35 100644 --- a/version.go +++ b/version.go @@ -1,3 +1,3 @@ package iavl -const Version = "0.9.0" +const Version = "0.9.1" From 48ba5981c11926ad1634d3706902443a844d4e96 Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 2 Jul 2018 14:58:07 -0700 Subject: [PATCH 2/3] Require Verify() before Verify*() --- CHANGELOG.md | 1 + proof_range.go | 41 ++++++++++++++++++++++++++++------------- proof_test.go | 4 ++++ 3 files changed, 33 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5bf969b49..af276a7f5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ IMPROVEMENTS - RangeProof.ComputeRootHash() to compute root rather than provide as in Verify(hash) +- RangeProof.Verify\*() first require .Verify(root), which memoizes ## 0.9.0 (July 1, 2018) diff --git a/proof_range.go b/proof_range.go index c5377b0db..26656f0b2 100644 --- a/proof_range.go +++ b/proof_range.go @@ -16,8 +16,12 @@ type RangeProof struct { LeftPath PathToLeaf `json:"left_path"` InnerNodes []PathToLeaf `json:"inner_nodes"` Leaves []proofLeafNode `json:"leaves"` - // temporary - treeEnd int // 0 if not set, 1 if true, -1 if false. + + // memoize + rootVerified bool + rootHash []byte // valid iff rootVerified is true + treeEnd bool // valid iff rootVerified is true + } func (proof *RangeProof) Keys() (keys [][]byte) { @@ -53,6 +57,8 @@ func (proof *RangeProof) StringIndented(indent string) string { %s %v %s Leaves: %s %v +%s (rootVerified): %v +%s (rootHash): %X %s (treeEnd): %v %s}`, indent, proof.LeftPath.StringIndented(indent+" "), @@ -60,6 +66,8 @@ func (proof *RangeProof) StringIndented(indent string) string { indent, strings.Join(istrs, "\n"+indent+" "), indent, indent, strings.Join(lstrs, "\n"+indent+" "), + indent, proof.rootVerified, + indent, proof.rootHash, indent, proof.treeEnd, indent) } @@ -82,6 +90,9 @@ func (proof *RangeProof) VerifyItem(key, value []byte) error { if proof == nil { return cmn.ErrorWrap(ErrInvalidProof, "proof is nil") } + if !proof.rootVerified { + return cmn.NewError("must call Verify(root) first.") + } i := sort.Search(len(leaves), func(i int) bool { return bytes.Compare(key, leaves[i].Key) <= 0 }) @@ -102,7 +113,7 @@ func (proof *RangeProof) VerifyAbsence(key []byte) error { if proof == nil { return cmn.ErrorWrap(ErrInvalidProof, "proof is nil") } - if proof.treeEnd == 0 { + if !proof.rootVerified { return cmn.NewError("must call Verify(root) first.") } cmp := bytes.Compare(key, proof.Leaves[0].Key) @@ -141,7 +152,7 @@ func (proof *RangeProof) VerifyAbsence(key []byte) error { } // It's still a valid proof if our last leaf is the rightmost child. - if proof.treeEnd == 1 { + if proof.treeEnd { return nil // OK! } @@ -163,11 +174,18 @@ func (proof *RangeProof) Verify(root []byte) error { } func (proof *RangeProof) verify(root []byte) (err error) { - rootHash, err := proof.computeRootHash() - if err != nil { - return err - } else if !bytes.Equal(rootHash, root) { + rootHash := proof.rootHash + if rootHash == nil { + derivedHash, err := proof.computeRootHash() + if err != nil { + return err + } + rootHash = derivedHash + } + if !bytes.Equal(rootHash, root) { return cmn.ErrorWrap(ErrInvalidRoot, "root hash doesn't match") + } else { + proof.rootVerified = true } return nil } @@ -186,11 +204,8 @@ func (proof *RangeProof) ComputeRootHash() []byte { func (proof *RangeProof) computeRootHash() (rootHash []byte, err error) { rootHash, treeEnd, err := proof._computeRootHash() if err == nil { - if treeEnd { - proof.treeEnd = 1 // memoize - } else { - proof.treeEnd = -1 // memoize - } + proof.rootHash = rootHash // memoize + proof.treeEnd = treeEnd // memoize } return rootHash, err } diff --git a/proof_test.go b/proof_test.go index 64522a9e0..77f423464 100644 --- a/proof_test.go +++ b/proof_test.go @@ -25,6 +25,8 @@ func TestTreeGetWithProof(t *testing.T) { require.NoError(err) require.NotEmpty(val) require.NotNil(proof) + err = proof.VerifyItem(key, val) + require.Error(err, "%+v", err) // Verifying item before calling Verify(root) err = proof.Verify(root) require.NoError(err, "%+v", err) err = proof.VerifyItem(key, val) @@ -35,6 +37,8 @@ func TestTreeGetWithProof(t *testing.T) { require.NoError(err) require.Empty(val) require.NotNil(proof) + err = proof.VerifyAbsence(key) + require.Error(err, "%+v", err) // Verifying absence before calling Verify(root) err = proof.Verify(root) require.NoError(err, "%+v", err) err = proof.VerifyAbsence(key) From 647806e556151653152675015ed2fa96f5b6c91e Mon Sep 17 00:00:00 2001 From: Jae Kwon Date: Mon, 2 Jul 2018 15:10:30 -0700 Subject: [PATCH 3/3] Review fixes from #75 --- proof_range.go | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) diff --git a/proof_range.go b/proof_range.go index 26656f0b2..0eefbd101 100644 --- a/proof_range.go +++ b/proof_range.go @@ -24,6 +24,13 @@ type RangeProof struct { } +// Keys returns all the keys in the RangeProof. NOTE: The keys here may +// include more keys than provided by tree.GetRangeWithProof or +// VersionedTree.GetVersionedRangeWithProof. The keys returned there are only +// in the provided [startKey,endKey){limit} range. The keys returned here may +// include extra keys, such as: +// - the key before startKey if startKey is provided and doesn't exist; +// - the key after a queried key with tree.GetWithProof, when the key is absent. func (proof *RangeProof) Keys() (keys [][]byte) { if proof == nil { return nil @@ -81,7 +88,6 @@ func (proof *RangeProof) LeftIndex() int64 { return proof.LeftPath.Index() } -// `i` is the relative index within this RangeProof. // Also see LeftIndex(). // Verify that a key has some value. // Does not assume that the proof itself is valid, call Verify() first. @@ -302,7 +308,7 @@ func (proof *RangeProof) _computeRootHash() (rootHash []byte, treeEnd bool, err // If keyStart or keyEnd don't exist, the leaf before keyStart // or after keyEnd will also be included, but not be included in values. // If keyEnd-1 exists, no later leaves will be included. -// If keyStart <= keyEnd and both not nil, panics. +// If keyStart >= keyEnd and both not nil, panics. // Limit is never exceeded. func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangeProof, keys, values [][]byte, err error) { if keyStart != nil && keyEnd != nil && bytes.Compare(keyStart, keyEnd) >= 0 {