0.9.1 -- compute hash from rangeproof

CHANGELOG.md

@@ -1,5 +1,12 @@
 # Changelog
 
+## 0.9.1 (July 1, 2018)
+
+IMPROVEMENTS
+
+- RangeProof.ComputeRootHash() to compute root rather than provide as in Verify(hash)
+- RangeProof.Verify\*() first require .Verify(root), which memoizes
+
 ## 0.9.0 (July 1, 2018)
 
 BREAKING CHANGES

proof_path.go

@@ -28,8 +28,19 @@ func (pwl pathWithLeaf) StringIndented(indent string) string {
 		indent)
 }
 
+// `verify` checks that the leaf node's hash + the inner nodes merkle-izes to
+// the given root. If it returns an error, it means the leafHash or the
+// PathToLeaf is incorrect.
 func (pwl pathWithLeaf) verify(root []byte) cmn.Error {
-	return pwl.Path.verify(pwl.Leaf.Hash(), root)
+	leafHash := pwl.Leaf.Hash()
+	return pwl.Path.verify(leafHash, root)
+}
+
+// `computeRootHash` computes the root hash with leaf node.
+// Does not verify the root hash.
+func (pwl pathWithLeaf) computeRootHash() []byte {
+	leafHash := pwl.Leaf.Hash()
+	return pwl.Path.computeRootHash(leafHash)
 }
 
 //----------------------------------------
@@ -62,9 +73,9 @@ func (pl PathToLeaf) StringIndented(indent string) string {
 		indent)
 }
 
-// verify checks that the leaf node's hash + the inner nodes merkle-izes to the
-// given root. If it returns an error, it means the leafHash or the PathToLeaf
-// is incorrect.
+// `verify` checks that the leaf node's hash + the inner nodes merkle-izes to
+// the given root. If it returns an error, it means the leafHash or the
+// PathToLeaf is incorrect.
 func (pl PathToLeaf) verify(leafHash []byte, root []byte) cmn.Error {
 	hash := leafHash
 	for i := len(pl) - 1; i >= 0; i-- {
@@ -77,6 +88,17 @@ func (pl PathToLeaf) verify(leafHash []byte, root []byte) cmn.Error {
 	return nil
 }
 
+// `computeRootHash` computes the root hash assuming some leaf hash.
+// Does not verify the root hash.
+func (pl PathToLeaf) computeRootHash(leafHash []byte) []byte {
+	hash := leafHash
+	for i := len(pl) - 1; i >= 0; i-- {
+		pin := pl[i]
+		hash = pin.Hash(hash)
+	}
+	return hash
+}
+
 func (pl PathToLeaf) isLeftmost() bool {
 	for _, node := range pl {
 		if len(node.Left) > 0 {

proof_range.go

@@ -13,14 +13,24 @@ import (
 type RangeProof struct {
 	// You don't need the right path because
 	// it can be derived from what we have.
-	RootHash   cmn.HexBytes    `json:"root_hash"`
 	LeftPath   PathToLeaf      `json:"left_path"`
 	InnerNodes []PathToLeaf    `json:"inner_nodes"`
 	Leaves     []proofLeafNode `json:"leaves"`
-	// temporary
-	treeEnd int // 0 if not set, 1 if true, -1 if false.
+
+	// memoize
+	rootVerified bool
+	rootHash     []byte // valid iff rootVerified is true
+	treeEnd      bool   // valid iff rootVerified is true
+
 }
 
+// Keys returns all the keys in the RangeProof.  NOTE: The keys here may
+// include more keys than provided by tree.GetRangeWithProof or
+// VersionedTree.GetVersionedRangeWithProof.  The keys returned there are only
+// in the provided [startKey,endKey){limit} range.  The keys returned here may
+// include extra keys, such as:
+// - the key before startKey if startKey is provided and doesn't exist;
+// - the key after a queried key with tree.GetWithProof, when the key is absent.
 func (proof *RangeProof) Keys() (keys [][]byte) {
 	if proof == nil {
 		return nil
@@ -49,20 +59,22 @@ func (proof *RangeProof) StringIndented(indent string) string {
 		lstrs = append(lstrs, leaf.StringIndented(indent+"    "))
 	}
 	return fmt.Sprintf(`RangeProof{
-%s  RootHash: %X
 %s  LeftPath: %v
 %s  InnerNodes:
 %s    %v
 %s  Leaves:
 %s    %v
+%s  (rootVerified): %v
+%s  (rootHash): %X
 %s  (treeEnd): %v
 %s}`,
-		indent, proof.RootHash,
 		indent, proof.LeftPath.StringIndented(indent+"  "),
 		indent,
 		indent, strings.Join(istrs, "\n"+indent+"    "),
 		indent,
 		indent, strings.Join(lstrs, "\n"+indent+"    "),
+		indent, proof.rootVerified,
+		indent, proof.rootHash,
 		indent, proof.treeEnd,
 		indent)
 }
@@ -76,7 +88,6 @@ func (proof *RangeProof) LeftIndex() int64 {
 	return proof.LeftPath.Index()
 }
 
-// `i` is the relative index within this RangeProof.
 // Also see LeftIndex().
 // Verify that a key has some value.
 // Does not assume that the proof itself is valid, call Verify() first.
@@ -85,6 +96,9 @@ func (proof *RangeProof) VerifyItem(key, value []byte) error {
 	if proof == nil {
 		return cmn.ErrorWrap(ErrInvalidProof, "proof is nil")
 	}
+	if !proof.rootVerified {
+		return cmn.NewError("must call Verify(root) first.")
+	}
 	i := sort.Search(len(leaves), func(i int) bool {
 		return bytes.Compare(key, leaves[i].Key) <= 0
 	})
@@ -105,7 +119,7 @@ func (proof *RangeProof) VerifyAbsence(key []byte) error {
 	if proof == nil {
 		return cmn.ErrorWrap(ErrInvalidProof, "proof is nil")
 	}
-	if proof.treeEnd == 0 {
+	if !proof.rootVerified {
 		return cmn.NewError("must call Verify(root) first.")
 	}
 	cmp := bytes.Compare(key, proof.Leaves[0].Key)
@@ -144,7 +158,7 @@ func (proof *RangeProof) VerifyAbsence(key []byte) error {
 	}
 
 	// It's still a valid proof if our last leaf is the rightmost child.
-	if proof.treeEnd == 1 {
+	if proof.treeEnd {
 		return nil // OK!
 	}
 
@@ -161,55 +175,81 @@ func (proof *RangeProof) Verify(root []byte) error {
 	if proof == nil {
 		return cmn.ErrorWrap(ErrInvalidProof, "proof is nil")
 	}
-	treeEnd, err := proof._verify(root)
-	if err == nil {
-		if treeEnd {
-			proof.treeEnd = 1 // memoize
-		} else {
-			proof.treeEnd = -1 // memoize
+	err := proof.verify(root)
+	return err
+}
+
+func (proof *RangeProof) verify(root []byte) (err error) {
+	rootHash := proof.rootHash
+	if rootHash == nil {
+		derivedHash, err := proof.computeRootHash()
+		if err != nil {
+			return err
 		}
+		rootHash = derivedHash
 	}
-	return err
+	if !bytes.Equal(rootHash, root) {
+		return cmn.ErrorWrap(ErrInvalidRoot, "root hash doesn't match")
+	} else {
+		proof.rootVerified = true
+	}
+	return nil
 }
 
-func (proof *RangeProof) _verify(root []byte) (treeEnd bool, err error) {
-	if !bytes.Equal(proof.RootHash, root) {
-		return false, cmn.ErrorWrap(ErrInvalidRoot, "root hash doesn't match")
+// ComputeRootHash computes the root hash with leaves.
+// Returns nil if error or proof is nil.
+// Does not verify the root hash.
+func (proof *RangeProof) ComputeRootHash() []byte {
+	if proof == nil {
+		return nil
+	}
+	rootHash, _ := proof.computeRootHash()
+	return rootHash
+}
+
+func (proof *RangeProof) computeRootHash() (rootHash []byte, err error) {
+	rootHash, treeEnd, err := proof._computeRootHash()
+	if err == nil {
+		proof.rootHash = rootHash // memoize
+		proof.treeEnd = treeEnd   // memoize
 	}
+	return rootHash, err
+}
+
+func (proof *RangeProof) _computeRootHash() (rootHash []byte, treeEnd bool, err error) {
 	if len(proof.Leaves) == 0 {
-		return false, cmn.ErrorWrap(ErrInvalidProof, "no leaves")
+		return nil, false, cmn.ErrorWrap(ErrInvalidProof, "no leaves")
 	}
 	if len(proof.InnerNodes)+1 != len(proof.Leaves) {
-		return false, cmn.ErrorWrap(ErrInvalidProof, "InnerNodes vs Leaves length mismatch, leaves should be 1 more.")
+		return nil, false, cmn.ErrorWrap(ErrInvalidProof, "InnerNodes vs Leaves length mismatch, leaves should be 1 more.")
 	}
 
 	// Start from the left path and prove each leaf.
 
 	// shared across recursive calls
 	var leaves = proof.Leaves
 	var innersq = proof.InnerNodes
-	var VERIFY func(path PathToLeaf, root []byte, rightmost bool) (treeEnd bool, done bool, err error)
+	var COMPUTEHASH func(path PathToLeaf, rightmost bool) (hash []byte, treeEnd bool, done bool, err error)
+
 	// rightmost: is the root a rightmost child of the tree?
 	// treeEnd: true iff the last leaf is the last item of the tree.
-	// NOTE: root doesn't necessarily mean root of the tree here.
-	VERIFY = func(path PathToLeaf, root []byte, rightmost bool) (treeEnd bool, done bool, err error) {
+	// Returns the (possibly intermediate, possibly root) hash.
+	COMPUTEHASH = func(path PathToLeaf, rightmost bool) (hash []byte, treeEnd bool, done bool, err error) {
 
 		// Pop next leaf.
 		nleaf, rleaves := leaves[0], leaves[1:]
 		leaves = rleaves
 
-		// Verify leaf with path.
-		if err := (pathWithLeaf{
+		// Compute hash.
+		hash = (pathWithLeaf{
 			Path: path,
 			Leaf: nleaf,
-		}).verify(root); err != nil {
-			return false, false, err.Trace(0, "verifying some path to a leaf")
-		}
+		}).computeRootHash()
 
 		// If we don't have any leaves left, we're done.
 		if len(leaves) == 0 {
 			rightmost = rightmost && path.isRightmost()
-			return rightmost, true, nil
+			return hash, rightmost, true, nil
 		}
 
 		// Prove along path (until we run out of leaves).
@@ -231,31 +271,35 @@ func (proof *RangeProof) _verify(root []byte) (treeEnd bool, err error) {
 			innersq = rinnersq
 
 			// Recursively verify inners against remaining leaves.
-			treeEnd, done, err := VERIFY(inners, lpath.Right, rightmost && rpath.isRightmost())
+			derivedRoot, treeEnd, done, err := COMPUTEHASH(inners, rightmost && rpath.isRightmost())
 			if err != nil {
-				return treeEnd, false, cmn.ErrorWrap(err, "recursive VERIFY call")
-			} else if done {
-				return treeEnd, true, nil
+				return nil, treeEnd, false, cmn.ErrorWrap(err, "recursive COMPUTEHASH call")
+			}
+			if !bytes.Equal(derivedRoot, lpath.Right) {
+				return nil, treeEnd, false, cmn.ErrorWrap(ErrInvalidRoot, "intermediate root hash %X doesn't match, got %X", lpath.Right, derivedRoot)
+			}
+			if done {
+				return hash, treeEnd, true, nil
 			}
 		}
 
-		// We're not done yet. No error, not done either.  Technically if
-		// rightmost, we know there's an error "left over leaves -- malformed
-		// proof", but we return that at the top level, below.
-		return false, false, nil
+		// We're not done yet (leaves left over). No error, not done either.
+		// Technically if rightmost, we know there's an error "left over leaves
+		// -- malformed proof", but we return that at the top level, below.
+		return hash, false, false, nil
 	}
 
 	// Verify!
 	path := proof.LeftPath
-	treeEnd, done, err := VERIFY(path, root, true)
+	rootHash, treeEnd, done, err := COMPUTEHASH(path, true)
 	if err != nil {
-		return treeEnd, cmn.ErrorWrap(err, "root VERIFY call")
+		return nil, treeEnd, cmn.ErrorWrap(err, "root COMPUTEHASH call")
 	} else if !done {
-		return treeEnd, cmn.ErrorWrap(ErrInvalidProof, "left over leaves -- malformed proof")
+		return nil, treeEnd, cmn.ErrorWrap(ErrInvalidProof, "left over leaves -- malformed proof")
 	}
 
 	// Ok!
-	return treeEnd, nil
+	return rootHash, treeEnd, nil
 }
 
 ///////////////////////////////////////////////////////////////////////////////
@@ -264,7 +308,7 @@ func (proof *RangeProof) _verify(root []byte) (treeEnd bool, err error) {
 // If keyStart or keyEnd don't exist, the leaf before keyStart
 // or after keyEnd will also be included, but not be included in values.
 // If keyEnd-1 exists, no later leaves will be included.
-// If keyStart <= keyEnd and both not nil, panics.
+// If keyStart >= keyEnd and both not nil, panics.
 // Limit is never exceeded.
 func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangeProof, keys, values [][]byte, err error) {
 	if keyStart != nil && keyEnd != nil && bytes.Compare(keyStart, keyEnd) >= 0 {
@@ -309,7 +353,6 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr
 	}
 	if _stop {
 		return &RangeProof{
-			RootHash: t.root.hash,
 			LeftPath: path,
 			Leaves:   leaves,
 		}, keys, values, nil
@@ -398,7 +441,6 @@ func (t *Tree) getRangeProof(keyStart, keyEnd []byte, limit int) (proof *RangePr
 	)
 
 	return &RangeProof{
-		RootHash:   t.root.hash,
 		LeftPath:   path,
 		InnerNodes: innersq,
 		Leaves:     leaves,

proof_test.go

@@ -25,6 +25,8 @@ func TestTreeGetWithProof(t *testing.T) {
 	require.NoError(err)
 	require.NotEmpty(val)
 	require.NotNil(proof)
+	err = proof.VerifyItem(key, val)
+	require.Error(err, "%+v", err) // Verifying item before calling Verify(root)
 	err = proof.Verify(root)
 	require.NoError(err, "%+v", err)
 	err = proof.VerifyItem(key, val)
@@ -35,6 +37,8 @@ func TestTreeGetWithProof(t *testing.T) {
 	require.NoError(err)
 	require.Empty(val)
 	require.NotNil(proof)
+	err = proof.VerifyAbsence(key)
+	require.Error(err, "%+v", err) // Verifying absence before calling Verify(root)
 	err = proof.Verify(root)
 	require.NoError(err, "%+v", err)
 	err = proof.VerifyAbsence(key)
@@ -223,10 +227,6 @@ func verifyProof(t *testing.T, proof *RangeProof, root []byte) {
 				proofBytes, badProofBytes)
 		}
 	}
-
-	// targetted changes fails...
-	proof.RootHash = test.MutateByteSlice(proof.RootHash)
-	assert.Error(t, proof.Verify(root))
 }
 
 //----------------------------------------

version.go

@@ -1,3 +1,3 @@
 package iavl
 
-const Version = "0.9.0"
+const Version = "0.9.1"