From e66b8ef2129e35c21bd601f557776209265f498f Mon Sep 17 00:00:00 2001 From: Roy Crihfield Date: Tue, 22 Feb 2022 20:22:06 +0800 Subject: [PATCH] feat: ADR-040: ICS-23 proofs for SMT store (#10015) ## Description Implements [ICS-23](https://github.com/cosmos/ibc/tree/master/spec/core/ics-023-vector-commitments) conformant proofs for the SMT-based KV store and defines the proof spec as part of [ADR-040](https://github.com/cosmos/cosmos-sdk/blob/eb7d939f86c6cd7b4218492364cdda3f649f06b5/docs/architecture/adr-040-storage-and-smt-state-commitments.md). Closes: https://github.com/vulcanize/cosmos-sdk/issues/8 --- ### Author Checklist *All items are required. Please add a note to the item if the item is not applicable and please add links to any relevant follow up issues.* I have... - [x] included the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title - [ ] added `!` to the type prefix if API or client breaking change - [x] targeted the correct branch (see [PR Targeting](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#pr-targeting)) - [x] provided a link to the relevant issue or specification - [ ] followed the guidelines for [building modules](https://github.com/cosmos/cosmos-sdk/blob/master/docs/building-modules) - n/a - [x] included the necessary unit and integration [tests](https://github.com/cosmos/cosmos-sdk/blob/master/CONTRIBUTING.md#testing) - [x] added a changelog entry to `CHANGELOG.md` - [x] included comments for [documenting Go code](https://blog.golang.org/godoc) - [ ] updated the relevant documentation or specification - [ ] reviewed "Files changed" and left comments if necessary - [ ] confirmed all CI checks have passed ### Reviewers Checklist *All items are required. Please add a note if the item is not applicable and please add your handle next to the items reviewed if you only reviewed selected items.* I have... - [ ] confirmed the correct [type prefix](https://github.com/commitizen/conventional-commit-types/blob/v3.0.0/index.json) in the PR title - [ ] confirmed `!` in the type prefix if API or client breaking change - [ ] confirmed all author checklist items have been addressed - [ ] reviewed state machine logic - [ ] reviewed API design and naming - [ ] reviewed documentation is accurate - [ ] reviewed tests and test coverage - [ ] manually tested (if applicable) --- CHANGELOG.md | 1 + go.mod | 4 +- go.sum | 3 +- store/internal/proofs/create.go | 8 +- store/rootmulti/proof.go | 8 -- store/types/commit_info.go | 26 +------ store/types/proof.go | 45 +++++++++++- store/v2/multi/cache_store.go | 2 +- store/v2/multi/doc.go | 2 +- store/v2/multi/proof.go | 52 +++++++++++++ store/v2/multi/proof_test.go | 125 ++++++++++++++++++++++++++++++++ store/v2/multi/store.go | 19 +++-- store/v2/multi/store_test.go | 2 +- store/v2/multi/sub_store.go | 2 +- store/v2/multi/test_util.go | 2 +- store/v2/multi/view_store.go | 19 ++++- store/v2/smt/ics23.go | 124 +++++++++++++++++++++++++++++++ store/v2/smt/ics23_test.go | 108 +++++++++++++++++++++++++++ store/v2/smt/proof_test.go | 3 +- store/v2/smt/store.go | 49 ++++++++++--- store/v2/smt/store_test.go | 14 ++-- store/v2/types.go | 7 ++ 22 files changed, 552 insertions(+), 73 deletions(-) create mode 100644 store/v2/multi/proof.go create mode 100644 store/v2/multi/proof_test.go create mode 100644 store/v2/smt/ics23.go create mode 100644 store/v2/smt/ics23_test.go diff --git a/CHANGELOG.md b/CHANGELOG.md index 58b074e11d50..8db6795f3cc5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,6 +64,7 @@ Ref: https://keepachangelog.com/en/1.0.0/ * (gov) [\#11036](https://github.com/cosmos/cosmos-sdk/pull/11036) Add in-place migrations for 0.43->0.46. Add a `migrate v0.46` CLI command for v0.43->0.46 JSON genesis migration. * [\#11006](https://github.com/cosmos/cosmos-sdk/pull/11006) Add `debug pubkey-raw` command to allow inspecting of pubkeys in legacy bech32 format * (x/authz) [\#10714](https://github.com/cosmos/cosmos-sdk/pull/10714) Add support for pruning expired authorizations +* [\#10015](https://github.com/cosmos/cosmos-sdk/pull/10015) ADR-040: ICS-23 proofs for SMT store ### API Breaking Changes diff --git a/go.mod b/go.mod index c28098cdb385..83f9c9fb6b33 100644 --- a/go.mod +++ b/go.mod @@ -9,7 +9,7 @@ require ( github.com/btcsuite/btcd v0.22.0-beta github.com/cockroachdb/apd/v2 v2.0.2 github.com/coinbase/rosetta-sdk-go v0.7.2 - github.com/confio/ics23/go v0.6.6 + github.com/confio/ics23/go v0.7.0-rc github.com/cosmos/btcutil v1.0.4 github.com/cosmos/cosmos-proto v1.0.0-alpha7 github.com/cosmos/cosmos-sdk/api v0.1.0-alpha4 @@ -18,6 +18,7 @@ require ( github.com/cosmos/go-bip39 v1.0.0 github.com/cosmos/iavl v0.17.3 github.com/cosmos/ledger-cosmos-go v0.11.1 + github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/gogo/gateway v1.1.0 github.com/gogo/protobuf v1.3.3 github.com/golang/mock v1.6.0 @@ -72,7 +73,6 @@ require ( github.com/cosmos/ledger-go v0.9.2 // indirect github.com/danieljoos/wincred v1.0.2 // indirect github.com/davecgh/go-spew v1.1.1 // indirect - github.com/desertbit/timer v0.0.0-20180107155436-c41aec40b27f // indirect github.com/dgraph-io/badger/v2 v2.2007.4 // indirect github.com/dgraph-io/ristretto v0.1.0 // indirect github.com/dgryski/go-farm v0.0.0-20200201041132-a6ae2369ad13 // indirect diff --git a/go.sum b/go.sum index fa856da667ab..ece955f4d984 100644 --- a/go.sum +++ b/go.sum @@ -259,8 +259,9 @@ github.com/cockroachdb/datadriven v0.0.0-20190809214429-80d97fb3cbaa/go.mod h1:z github.com/codahale/hdrhistogram v0.0.0-20161010025455-3a0bb77429bd/go.mod h1:sE/e/2PUdi/liOCUjSTXgM1o87ZssimdTWN964YiIeI= github.com/coinbase/rosetta-sdk-go v0.7.2 h1:uCNrASIyt7rV9bA3gzPG3JDlxVP5v/zLgi01GWngncM= github.com/coinbase/rosetta-sdk-go v0.7.2/go.mod h1:wk9dvjZFSZiWSNkFuj3dMleTA1adLFotg5y71PhqKB4= -github.com/confio/ics23/go v0.6.6 h1:pkOy18YxxJ/r0XFDCnrl4Bjv6h4LkBSpLS6F38mrKL8= github.com/confio/ics23/go v0.6.6/go.mod h1:E45NqnlpxGnpfTWL/xauN7MRwEE28T4Dd4uraToOaKg= +github.com/confio/ics23/go v0.7.0-rc h1:cH2I3xkPE6oD4tP5pmZDAfYq8V7VeXCr98X1MpARTaI= +github.com/confio/ics23/go v0.7.0-rc/go.mod h1:E45NqnlpxGnpfTWL/xauN7MRwEE28T4Dd4uraToOaKg= github.com/consensys/bavard v0.1.8-0.20210406032232-f3452dc9b572/go.mod h1:Bpd0/3mZuaj6Sj+PqrmIquiOKy397AKGThQPaGzNXAQ= github.com/consensys/gnark-crypto v0.4.1-0.20210426202927-39ac3d4b3f1f/go.mod h1:815PAHg3wvysy0SyIqanF8gZ0Y1wjk/hrDHD/iT88+Q= github.com/containerd/console v1.0.2/go.mod h1:ytZPjGgY2oeTkAONYafi2kSj0aYggsf8acV1PGKCbzQ= diff --git a/store/internal/proofs/create.go b/store/internal/proofs/create.go index 01851d1dea09..a202408324a9 100644 --- a/store/internal/proofs/create.go +++ b/store/internal/proofs/create.go @@ -16,7 +16,7 @@ var ( ) /* -CreateMembershipProof will produce a CommitmentProof that the given key (and queries value) exists in the iavl tree. +CreateMembershipProof will produce a CommitmentProof that the given key (and queries value) exists in the map. If the key doesn't exist in the tree, this will return an error. */ func CreateMembershipProof(data map[string][]byte, key []byte) (*ics23.CommitmentProof, error) { @@ -36,7 +36,7 @@ func CreateMembershipProof(data map[string][]byte, key []byte) (*ics23.Commitmen } /* -CreateNonMembershipProof will produce a CommitmentProof that the given key doesn't exist in the iavl tree. +CreateNonMembershipProof will produce a CommitmentProof that the given key doesn't exist in the map. If the key exists in the tree, this will return an error. */ func CreateNonMembershipProof(data map[string][]byte, key []byte) (*ics23.CommitmentProof, error) { @@ -94,8 +94,8 @@ func createExistenceProof(data map[string][]byte, key []byte) (*ics23.ExistenceP return nil, fmt.Errorf("cannot make existence proof if key is not in map") } - _, ics23, _ := sdkmaps.ProofsFromMap(data) - proof := ics23[string(key)] + _, proofs, _ := sdkmaps.ProofsFromMap(data) + proof := proofs[string(key)] if proof == nil { return nil, fmt.Errorf("returned no proof for key") } diff --git a/store/rootmulti/proof.go b/store/rootmulti/proof.go index d71e8c1adc4f..fc8925b7f20d 100644 --- a/store/rootmulti/proof.go +++ b/store/rootmulti/proof.go @@ -4,7 +4,6 @@ import ( "github.com/tendermint/tendermint/crypto/merkle" storetypes "github.com/cosmos/cosmos-sdk/store/types" - "github.com/cosmos/cosmos-sdk/store/v2/smt" ) // RequireProof returns whether proof is required for the subpath. @@ -26,10 +25,3 @@ func DefaultProofRuntime() (prt *merkle.ProofRuntime) { prt.RegisterOpDecoder(storetypes.ProofOpSimpleMerkleCommitment, storetypes.CommitmentOpDecoder) return } - -// SMTProofRuntime returns a ProofRuntime for sparse merkle trees. -func SMTProofRuntime() (prt *merkle.ProofRuntime) { - prt = merkle.NewProofRuntime() - prt.RegisterOpDecoder(smt.ProofType, smt.ProofDecoder) - return prt -} diff --git a/store/types/commit_info.go b/store/types/commit_info.go index e713040739f8..a811b6d89be1 100644 --- a/store/types/commit_info.go +++ b/store/types/commit_info.go @@ -1,13 +1,9 @@ package types import ( - fmt "fmt" - - ics23 "github.com/confio/ics23/go" tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" sdkmaps "github.com/cosmos/cosmos-sdk/store/internal/maps" - sdkproofs "github.com/cosmos/cosmos-sdk/store/internal/proofs" ) // GetHash returns the GetHash from the CommitID. @@ -42,27 +38,11 @@ func (ci CommitInfo) Hash() []byte { } func (ci CommitInfo) ProofOp(storeName string) tmcrypto.ProofOp { - cmap := ci.toMap() - _, proofs, _ := sdkmaps.ProofsFromMap(cmap) - - proof := proofs[storeName] - if proof == nil { - panic(fmt.Sprintf("ProofOp for %s but not registered store name", storeName)) - } - - // convert merkle.SimpleProof to CommitmentProof - existProof, err := sdkproofs.ConvertExistenceProof(proof, []byte(storeName), cmap[storeName]) + ret, err := ProofOpFromMap(ci.toMap(), storeName) if err != nil { - panic(fmt.Errorf("could not convert simple proof to existence proof: %w", err)) + panic(err) } - - commitmentProof := &ics23.CommitmentProof{ - Proof: &ics23.CommitmentProof_Exist{ - Exist: existProof, - }, - } - - return NewSimpleMerkleCommitmentOp([]byte(storeName), commitmentProof).ProofOp() + return ret } func (ci CommitInfo) CommitID() CommitID { diff --git a/store/types/proof.go b/store/types/proof.go index db8f673f46cd..f2d254267143 100644 --- a/store/types/proof.go +++ b/store/types/proof.go @@ -1,16 +1,21 @@ package types import ( + "fmt" + ics23 "github.com/confio/ics23/go" "github.com/tendermint/tendermint/crypto/merkle" tmmerkle "github.com/tendermint/tendermint/proto/tendermint/crypto" + sdkmaps "github.com/cosmos/cosmos-sdk/store/internal/maps" + sdkproofs "github.com/cosmos/cosmos-sdk/store/internal/proofs" sdkerrors "github.com/cosmos/cosmos-sdk/types/errors" ) const ( ProofOpIAVLCommitment = "ics23:iavl" ProofOpSimpleMerkleCommitment = "ics23:simple" + ProofOpSMTCommitment = "ics23:smt" ) // CommitmentOp implements merkle.ProofOperator by wrapping an ics23 CommitmentProof @@ -46,6 +51,15 @@ func NewSimpleMerkleCommitmentOp(key []byte, proof *ics23.CommitmentProof) Commi } } +func NewSmtCommitmentOp(key []byte, proof *ics23.CommitmentProof) CommitmentOp { + return CommitmentOp{ + Type: ProofOpSMTCommitment, + Spec: ics23.SmtSpec, + Key: key, + Proof: proof, + } +} + // CommitmentOpDecoder takes a merkle.ProofOp and attempts to decode it into a CommitmentOp ProofOperator // The proofOp.Data is just a marshalled CommitmentProof. The Key of the CommitmentOp is extracted // from the unmarshalled proof. @@ -56,8 +70,10 @@ func CommitmentOpDecoder(pop tmmerkle.ProofOp) (merkle.ProofOperator, error) { spec = ics23.IavlSpec case ProofOpSimpleMerkleCommitment: spec = ics23.TendermintSpec + case ProofOpSMTCommitment: + spec = ics23.SmtSpec default: - return nil, sdkerrors.Wrapf(ErrInvalidProof, "unexpected ProofOp.Type; got %s, want supported ics23 subtypes 'ProofOpIAVLCommitment' or 'ProofOpSimpleMerkleCommitment'", pop.Type) + return nil, sdkerrors.Wrapf(ErrInvalidProof, "unexpected ProofOp.Type; got %s, want supported ics23 subtypes 'ProofOpSimpleMerkleCommitment', 'ProofOpIAVLCommitment', or 'ProofOpSMTCommitment'", pop.Type) } proof := &ics23.CommitmentProof{} @@ -129,3 +145,30 @@ func (op CommitmentOp) ProofOp() tmmerkle.ProofOp { Data: bz, } } + +// ProofOpFromMap generates a single proof from a map and converts it to a ProofOp. +func ProofOpFromMap(cmap map[string][]byte, storeName string) (ret tmmerkle.ProofOp, err error) { + _, proofs, _ := sdkmaps.ProofsFromMap(cmap) + + proof := proofs[storeName] + if proof == nil { + err = fmt.Errorf("ProofOp for %s but not registered store name", storeName) + return + } + + // convert merkle.SimpleProof to CommitmentProof + existProof, err := sdkproofs.ConvertExistenceProof(proof, []byte(storeName), cmap[storeName]) + if err != nil { + err = fmt.Errorf("could not convert simple proof to existence proof: %w", err) + return + } + + commitmentProof := &ics23.CommitmentProof{ + Proof: &ics23.CommitmentProof_Exist{ + Exist: existProof, + }, + } + + ret = NewSimpleMerkleCommitmentOp([]byte(storeName), commitmentProof).ProofOp() + return +} diff --git a/store/v2/multi/cache_store.go b/store/v2/multi/cache_store.go index 58e261c2020f..3fcf7170bc19 100644 --- a/store/v2/multi/cache_store.go +++ b/store/v2/multi/cache_store.go @@ -1,4 +1,4 @@ -package root +package multi import ( "github.com/cosmos/cosmos-sdk/store/cachekv" diff --git a/store/v2/multi/doc.go b/store/v2/multi/doc.go index 76469ab11a69..a977d4d5bd75 100644 --- a/store/v2/multi/doc.go +++ b/store/v2/multi/doc.go @@ -16,4 +16,4 @@ // of a key's (non)existence within the substore SMT, and a proof of the substore's existence within the // MultiStore (using the Merkle map proof spec (TendermintSpec)). -package root +package multi diff --git a/store/v2/multi/proof.go b/store/v2/multi/proof.go new file mode 100644 index 000000000000..25fc2b8e381b --- /dev/null +++ b/store/v2/multi/proof.go @@ -0,0 +1,52 @@ +package multi + +import ( + "crypto/sha256" + + "github.com/tendermint/tendermint/crypto/merkle" + tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" + + types "github.com/cosmos/cosmos-sdk/store/v2" + "github.com/cosmos/cosmos-sdk/store/v2/smt" +) + +// DefaultProofRuntime returns a ProofRuntime supporting SMT and simple merkle proofs. +func DefaultProofRuntime() (prt *merkle.ProofRuntime) { + prt = merkle.NewProofRuntime() + prt.RegisterOpDecoder(types.ProofOpSMTCommitment, types.CommitmentOpDecoder) + prt.RegisterOpDecoder(types.ProofOpSimpleMerkleCommitment, types.CommitmentOpDecoder) + return prt +} + +// Prove commitment of key within an smt store and return ProofOps +func proveKey(s *smt.Store, key []byte) (*tmcrypto.ProofOps, error) { + var ret tmcrypto.ProofOps + keyProof, err := s.GetProofICS23(key) + if err != nil { + return nil, err + } + hkey := sha256.Sum256(key) + ret.Ops = append(ret.Ops, types.NewSmtCommitmentOp(hkey[:], keyProof).ProofOp()) + return &ret, nil +} + +// GetProof returns ProofOps containing: a proof for the given key within this substore; +// and a proof of the substore's existence within the MultiStore. +func (s *viewSubstore) GetProof(key []byte) (*tmcrypto.ProofOps, error) { + ret, err := proveKey(s.stateCommitmentStore, key) + if err != nil { + return nil, err + } + + // Prove commitment of substore within root store + storeHashes, err := s.root.getMerkleRoots() + if err != nil { + return nil, err + } + storeProof, err := types.ProofOpFromMap(storeHashes, s.name) + if err != nil { + return nil, err + } + ret.Ops = append(ret.Ops, storeProof) + return ret, nil +} diff --git a/store/v2/multi/proof_test.go b/store/v2/multi/proof_test.go new file mode 100644 index 000000000000..8f7d05de2366 --- /dev/null +++ b/store/v2/multi/proof_test.go @@ -0,0 +1,125 @@ +package multi + +import ( + "crypto/sha256" + "testing" + + "github.com/stretchr/testify/require" + abci "github.com/tendermint/tendermint/abci/types" + + "github.com/cosmos/cosmos-sdk/db/memdb" + "github.com/cosmos/cosmos-sdk/store/v2/smt" +) + +// We hash keys produce SMT paths, so reflect that here +func keyPath(prefix, key string) string { + hashed := sha256.Sum256([]byte(key)) + return prefix + string(hashed[:]) +} + +func TestVerifySMTStoreProof(t *testing.T) { + // Create main tree for testing. + txn := memdb.NewDB().ReadWriter() + store := smt.NewStore(txn) + store.Set([]byte("MYKEY"), []byte("MYVALUE")) + root := store.Root() + + res, err := proveKey(store, []byte("MYKEY")) + require.NoError(t, err) + + // Verify good proof. + prt := DefaultProofRuntime() + err = prt.VerifyValue(res, root, keyPath("/", "MYKEY"), []byte("MYVALUE")) + require.NoError(t, err) + + // Fail to verify bad proofs. + err = prt.VerifyValue(res, root, keyPath("/", "MYKEY_NOT"), []byte("MYVALUE")) + require.Error(t, err) + + err = prt.VerifyValue(res, root, keyPath("/", "MYKEY/MYKEY"), []byte("MYVALUE")) + require.Error(t, err) + + err = prt.VerifyValue(res, root, keyPath("", "MYKEY"), []byte("MYVALUE")) + require.Error(t, err) + + err = prt.VerifyValue(res, root, keyPath("/", "MYKEY"), []byte("MYVALUE_NOT")) + require.Error(t, err) + + err = prt.VerifyValue(res, root, keyPath("/", "MYKEY"), []byte(nil)) + require.Error(t, err) +} + +func TestVerifyMultiStoreQueryProof(t *testing.T) { + db := memdb.NewDB() + store, err := NewStore(db, simpleStoreConfig(t)) + require.NoError(t, err) + + substore := store.GetKVStore(skey_1) + substore.Set([]byte("MYKEY"), []byte("MYVALUE")) + cid := store.Commit() + + res := store.Query(abci.RequestQuery{ + Path: "/store1/key", // required path to get key/value+proof + Data: []byte("MYKEY"), + Prove: true, + }) + require.NotNil(t, res.ProofOps) + + // Verify good proofs. + prt := DefaultProofRuntime() + err = prt.VerifyValue(res.ProofOps, cid.Hash, keyPath("/store1/", "MYKEY"), []byte("MYVALUE")) + require.NoError(t, err) + + err = prt.VerifyValue(res.ProofOps, cid.Hash, keyPath("/store1/", "MYKEY"), []byte("MYVALUE")) + require.NoError(t, err) + + // Fail to verify bad proofs. + err = prt.VerifyValue(res.ProofOps, cid.Hash, keyPath("/store1/", "MYKEY_NOT"), []byte("MYVALUE")) + require.Error(t, err) + + err = prt.VerifyValue(res.ProofOps, cid.Hash, keyPath("/store1/MYKEY/", "MYKEY"), []byte("MYVALUE")) + require.Error(t, err) + + err = prt.VerifyValue(res.ProofOps, cid.Hash, keyPath("store1/", "MYKEY"), []byte("MYVALUE")) + require.Error(t, err) + + err = prt.VerifyValue(res.ProofOps, cid.Hash, keyPath("/", "MYKEY"), []byte("MYVALUE")) + require.Error(t, err) + + err = prt.VerifyValue(res.ProofOps, cid.Hash, keyPath("/store1/", "MYKEY"), []byte("MYVALUE_NOT")) + require.Error(t, err) + + err = prt.VerifyValue(res.ProofOps, cid.Hash, keyPath("/store1/", "MYKEY"), []byte(nil)) + require.Error(t, err) +} + +func TestVerifyMultiStoreQueryProofAbsence(t *testing.T) { + db := memdb.NewDB() + store, err := NewStore(db, simpleStoreConfig(t)) + require.NoError(t, err) + + substore := store.GetKVStore(skey_1) + substore.Set([]byte("MYKEY"), []byte("MYVALUE")) + cid := store.Commit() + + res := store.Query(abci.RequestQuery{ + Path: "/store1/key", // required path to get key/value+proof + Data: []byte("MYABSENTKEY"), + Prove: true, + }) + require.NotNil(t, res.ProofOps) + + // Verify good proof. + prt := DefaultProofRuntime() + err = prt.VerifyAbsence(res.ProofOps, cid.Hash, keyPath("/store1/", "MYABSENTKEY")) + require.NoError(t, err) + + // Fail to verify bad proofs. + prt = DefaultProofRuntime() + err = prt.VerifyAbsence(res.ProofOps, cid.Hash, keyPath("/", "MYABSENTKEY")) + require.Error(t, err) + + prt = DefaultProofRuntime() + err = prt.VerifyValue(res.ProofOps, cid.Hash, keyPath("/store1/", "MYABSENTKEY"), []byte("")) + require.Error(t, err) +} diff --git a/store/v2/multi/store.go b/store/v2/multi/store.go index 5ad1e33fa6af..552baa48bef7 100644 --- a/store/v2/multi/store.go +++ b/store/v2/multi/store.go @@ -1,4 +1,4 @@ -package root +package multi import ( "errors" @@ -43,8 +43,7 @@ var ( substoreMerkleRootKey = []byte{0} // Key for root hashes of Merkle trees dataPrefix = []byte{1} // Prefix for state mappings indexPrefix = []byte{2} // Prefix for Store reverse index - merkleNodePrefix = []byte{3} // Prefix for Merkle tree nodes - merkleValuePrefix = []byte{4} // Prefix for Merkle value mappings + smtPrefix = []byte{3} // Prefix for SMT data ErrVersionDoesNotExist = errors.New("version does not exist") ErrMaximumHeight = errors.New("maximum block height reached") @@ -125,6 +124,8 @@ type viewStore struct { } type viewSubstore struct { + root *viewStore + name string dataBucket dbm.DBReader indexBucket dbm.DBReader stateCommitmentStore *smt.Store @@ -492,9 +493,8 @@ func (rs *Store) getSubstore(key string) (*substore, error) { if rootHash != nil { stateCommitmentStore = loadSMT(stateCommitmentRW, rootHash) } else { - merkleNodes := prefixdb.NewPrefixReadWriter(stateCommitmentRW, merkleNodePrefix) - merkleValues := prefixdb.NewPrefixReadWriter(stateCommitmentRW, merkleValuePrefix) - stateCommitmentStore = smt.NewStore(merkleNodes, merkleValues) + smtdb := prefixdb.NewPrefixReadWriter(stateCommitmentRW, smtPrefix) + stateCommitmentStore = smt.NewStore(smtdb) } return &substore{ @@ -771,7 +771,7 @@ func (rs *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { break } // TODO: actual IBC compatible proof. This is a placeholder so unit tests can pass - res.ProofOps, err = substore.stateCommitmentStore.GetProof([]byte(storeName + string(res.Key))) + res.ProofOps, err = substore.GetProof(res.Key) if err != nil { return sdkerrors.QueryResult(fmt.Errorf("Merkle proof creation failed for key: %v", res.Key), false) //nolint: stylecheck // proper name } @@ -806,9 +806,8 @@ func (rs *Store) Query(req abci.RequestQuery) (res abci.ResponseQuery) { } func loadSMT(stateCommitmentTxn dbm.DBReadWriter, root []byte) *smt.Store { - merkleNodes := prefixdb.NewPrefixReadWriter(stateCommitmentTxn, merkleNodePrefix) - merkleValues := prefixdb.NewPrefixReadWriter(stateCommitmentTxn, merkleValuePrefix) - return smt.LoadStore(merkleNodes, merkleValues, root) + smtdb := prefixdb.NewPrefixReadWriter(stateCommitmentTxn, smtPrefix) + return smt.LoadStore(smtdb, root) } // Returns closest index and whether it's a match diff --git a/store/v2/multi/store_test.go b/store/v2/multi/store_test.go index 49904829aee1..fdf9202f0fbd 100644 --- a/store/v2/multi/store_test.go +++ b/store/v2/multi/store_test.go @@ -1,4 +1,4 @@ -package root +package multi import ( "bytes" diff --git a/store/v2/multi/sub_store.go b/store/v2/multi/sub_store.go index e11e8b0d5440..613b7ac2c412 100644 --- a/store/v2/multi/sub_store.go +++ b/store/v2/multi/sub_store.go @@ -1,4 +1,4 @@ -package root +package multi import ( "crypto/sha256" diff --git a/store/v2/multi/test_util.go b/store/v2/multi/test_util.go index 777e59cc2b06..cc03639906d9 100644 --- a/store/v2/multi/test_util.go +++ b/store/v2/multi/test_util.go @@ -1,4 +1,4 @@ -package root +package multi import ( "bytes" diff --git a/store/v2/multi/view_store.go b/store/v2/multi/view_store.go index 85eaeb6e7cfe..79e080e05cb5 100644 --- a/store/v2/multi/view_store.go +++ b/store/v2/multi/view_store.go @@ -1,4 +1,4 @@ -package root +package multi import ( "errors" @@ -84,6 +84,21 @@ func (st *viewSubstore) CacheWrapWithListeners(storeKey types.StoreKey, listener return cachekv.NewStore(listenkv.NewStore(st, storeKey, listeners)) } +func (s *viewStore) getMerkleRoots() (ret map[string][]byte, err error) { + ret = map[string][]byte{} + for key, _ := range s.schema { + sub, has := s.substoreCache[key] + if !has { + sub, err = s.getSubstore(key) + if err != nil { + return + } + } + ret[key] = sub.stateCommitmentStore.Root() + } + return +} + func (store *Store) getView(version int64) (ret *viewStore, err error) { stateView, err := store.stateDB.ReaderAt(uint64(version)) if err != nil { @@ -154,6 +169,8 @@ func (vs *viewStore) getSubstore(key string) (*viewSubstore, error) { return nil, err } return &viewSubstore{ + root: vs, + name: key, dataBucket: prefixdb.NewPrefixReader(stateR, dataPrefix), indexBucket: prefixdb.NewPrefixReader(stateR, indexPrefix), stateCommitmentStore: loadSMT(dbm.ReaderAsReadWriter(stateCommitmentR), rootHash), diff --git a/store/v2/smt/ics23.go b/store/v2/smt/ics23.go new file mode 100644 index 000000000000..31d78f993dc0 --- /dev/null +++ b/store/v2/smt/ics23.go @@ -0,0 +1,124 @@ +// Here we implement proof generation according to the ICS-23 specification: +// https://github.com/cosmos/ibc/tree/master/spec/core/ics-023-vector-commitments + +package smt + +import ( + "crypto/sha256" + "fmt" + + dbm "github.com/cosmos/cosmos-sdk/db" + + ics23 "github.com/confio/ics23/go" +) + +func createIcs23Proof(store *Store, key []byte) (*ics23.CommitmentProof, error) { + ret := &ics23.CommitmentProof{} + path := sha256.Sum256(key) + has, err := store.tree.Has(key) + if err != nil { + return nil, err + } + if has { // Membership proof + value, err := store.values.Get(path[:]) + if err != nil { + return nil, err + } + if value == nil { + return nil, fmt.Errorf("value not found for: %v", key) + } + proof, err := store.tree.Prove(key) + if err != nil { + return nil, err + } + ret.Proof = &ics23.CommitmentProof_Exist{&ics23.ExistenceProof{ + Key: path[:], + Value: value, + Leaf: ics23.SmtSpec.LeafSpec, + Path: convertInnerOps(path[:], proof.SideNodes), + }} + } else { // Non-membership + nonexist, err := toNonExistenceProof(store, path) + if err != nil { + return nil, err + } + ret.Proof = &ics23.CommitmentProof_Nonexist{nonexist} + } + return ret, nil +} + +func toNonExistenceProof(store *Store, path [32]byte) (*ics23.NonExistenceProof, error) { + // Seek to our neighbors via the backing DB + getNext := func(it dbm.Iterator) (*ics23.ExistenceProof, error) { + defer it.Close() + if it.Next() { + value, err := store.values.Get(it.Key()) + if err != nil { + return nil, err + } + if value == nil { + return nil, fmt.Errorf("value not found for: %v", it.Value()) + } + proof, err := store.tree.Prove(it.Value()) // pass the preimage to Prove + if err != nil { + return nil, err + } + return &ics23.ExistenceProof{ + Key: it.Key(), + Value: value, + Leaf: ics23.SmtSpec.LeafSpec, + Path: convertInnerOps(it.Key(), proof.SideNodes), + }, nil + } + return nil, nil + } + var lproof, rproof *ics23.ExistenceProof + it, err := store.preimages.ReverseIterator(nil, path[:]) + if err != nil { + return nil, err + } + lproof, err = getNext(it) + if err != nil { + return nil, err + } + it, err = store.preimages.Iterator(path[:], nil) + if err != nil { + return nil, err + } + rproof, err = getNext(it) + if err != nil { + return nil, err + } + return &ics23.NonExistenceProof{ + Key: path[:], + Left: lproof, + Right: rproof, + }, nil +} + +func convertInnerOps(path []byte, sideNodes [][]byte) []*ics23.InnerOp { + depth := len(sideNodes) + inners := make([]*ics23.InnerOp, 0, depth) + for i := 0; i < len(sideNodes); i++ { + op := &ics23.InnerOp{ + Hash: ics23.HashOp_SHA256, + Prefix: []byte{1}, + } + if getBitAtFromMSB(path[:], depth-1-i) == 1 { + // right child is on path + op.Prefix = append(op.Prefix, sideNodes[i]...) + } else { + op.Suffix = sideNodes[i] + } + inners = append(inners, op) + } + return inners +} + +// getBitAtFromMSB gets the bit at an offset from the most significant bit +func getBitAtFromMSB(data []byte, position int) int { + if int(data[position/8])&(1<<(8-1-uint(position)%8)) > 0 { + return 1 + } + return 0 +} diff --git a/store/v2/smt/ics23_test.go b/store/v2/smt/ics23_test.go new file mode 100644 index 000000000000..6382bcb2f5c3 --- /dev/null +++ b/store/v2/smt/ics23_test.go @@ -0,0 +1,108 @@ +package smt_test + +import ( + "crypto/sha256" + "testing" + + ics23 "github.com/confio/ics23/go" + "github.com/stretchr/testify/assert" + + "github.com/cosmos/cosmos-sdk/db/memdb" + store "github.com/cosmos/cosmos-sdk/store/v2/smt" +) + +func TestProofICS23(t *testing.T) { + txn := memdb.NewDB().ReadWriter() + s := store.NewStore(txn) + // pick keys whose hashes begin with different bits + key00 := []byte("foo") // 00101100 = sha256(foo)[0] + key01 := []byte("bill") // 01100010 + key10 := []byte("baz") // 10111010 + key11 := []byte("bar") // 11111100 + path00 := sha256.Sum256(key00) + path01 := sha256.Sum256(key01) + path10 := sha256.Sum256(key10) + val1 := []byte("0") + val2 := []byte("1") + + s.Set(key01, val1) + + // Membership + proof, err := s.GetProofICS23(key01) + assert.NoError(t, err) + nonexist := proof.GetNonexist() + assert.Nil(t, nonexist) + exist := proof.GetExist() + assert.NotNil(t, exist) + assert.Equal(t, 0, len(exist.Path)) + assert.NoError(t, exist.Verify(ics23.SmtSpec, s.Root(), path01[:], val1)) + + // Non-membership + proof, err = s.GetProofICS23(key00) // When leaf is leftmost node + assert.NoError(t, err) + nonexist = proof.GetNonexist() + assert.NotNil(t, nonexist) + assert.Nil(t, nonexist.Left) + assert.Equal(t, path00[:], nonexist.Key) + assert.NotNil(t, nonexist.Right) + assert.Equal(t, 0, len(nonexist.Right.Path)) + assert.NoError(t, nonexist.Verify(ics23.SmtSpec, s.Root(), path00[:])) + + proof, err = s.GetProofICS23(key10) // When rightmost + assert.NoError(t, err) + nonexist = proof.GetNonexist() + assert.NotNil(t, nonexist) + assert.NotNil(t, nonexist.Left) + assert.Equal(t, 0, len(nonexist.Left.Path)) + assert.Nil(t, nonexist.Right) + assert.NoError(t, nonexist.Verify(ics23.SmtSpec, s.Root(), path10[:])) + badNonexist := nonexist + + s.Set(key11, val2) + + proof, err = s.GetProofICS23(key10) // In between two keys + assert.NoError(t, err) + nonexist = proof.GetNonexist() + assert.NotNil(t, nonexist) + assert.Equal(t, path10[:], nonexist.Key) + assert.NotNil(t, nonexist.Left) + assert.Equal(t, 1, len(nonexist.Left.Path)) + assert.NotNil(t, nonexist.Right) + assert.Equal(t, 1, len(nonexist.Right.Path)) + assert.NoError(t, nonexist.Verify(ics23.SmtSpec, s.Root(), path10[:])) + + // Make sure proofs work with a loaded store + root := s.Root() + s = store.LoadStore(txn, root) + proof, err = s.GetProofICS23(key10) + assert.NoError(t, err) + nonexist = proof.GetNonexist() + assert.Equal(t, path10[:], nonexist.Key) + assert.NotNil(t, nonexist.Left) + assert.Equal(t, 1, len(nonexist.Left.Path)) + assert.NotNil(t, nonexist.Right) + assert.Equal(t, 1, len(nonexist.Right.Path)) + assert.NoError(t, nonexist.Verify(ics23.SmtSpec, s.Root(), path10[:])) + + // Invalid proofs should fail to verify + badExist := exist // expired proof + assert.Error(t, badExist.Verify(ics23.SmtSpec, s.Root(), path01[:], val1)) + + badExist = nonexist.Left + badExist.Key = key01 // .Key must contain key path + assert.Error(t, badExist.Verify(ics23.SmtSpec, s.Root(), path01[:], val1)) + + badExist = nonexist.Left + badExist.Path[0].Prefix = []byte{0} // wrong inner node prefix + assert.Error(t, badExist.Verify(ics23.SmtSpec, s.Root(), path01[:], val1)) + + badExist = nonexist.Left + badExist.Path = []*ics23.InnerOp{} // empty path + assert.Error(t, badExist.Verify(ics23.SmtSpec, s.Root(), path01[:], val1)) + + assert.Error(t, badNonexist.Verify(ics23.SmtSpec, s.Root(), path10[:])) + + badNonexist = nonexist + badNonexist.Key = key10 + assert.Error(t, badNonexist.Verify(ics23.SmtSpec, s.Root(), path10[:])) +} diff --git a/store/v2/smt/proof_test.go b/store/v2/smt/proof_test.go index 94e93edb85ac..ee84b57fbda0 100644 --- a/store/v2/smt/proof_test.go +++ b/store/v2/smt/proof_test.go @@ -14,7 +14,8 @@ import ( func TestProofOpInterface(t *testing.T) { hasher := sha256.New() - tree := smt.NewSparseMerkleTree(memdb.NewDB().ReadWriter(), memdb.NewDB().ReadWriter(), hasher) + nodes, values := memdb.NewDB(), memdb.NewDB() + tree := smt.NewSparseMerkleTree(nodes.ReadWriter(), values.ReadWriter(), hasher) key := []byte("foo") value := []byte("bar") root, err := tree.Update(key, value) diff --git a/store/v2/smt/store.go b/store/v2/smt/store.go index b63d0e65ecd5..c10f59f00262 100644 --- a/store/v2/smt/store.go +++ b/store/v2/smt/store.go @@ -5,8 +5,10 @@ import ( "errors" dbm "github.com/cosmos/cosmos-sdk/db" + "github.com/cosmos/cosmos-sdk/db/prefix" "github.com/cosmos/cosmos-sdk/store/types" + ics23 "github.com/confio/ics23/go" "github.com/lazyledger/smt" tmcrypto "github.com/tendermint/tendermint/proto/tendermint/crypto" ) @@ -17,32 +19,52 @@ var ( ) var ( + nodesPrefix = []byte{0} + valuesPrefix = []byte{1} + preimagesPrefix = []byte{2} + errKeyEmpty = errors.New("key is empty or nil") errValueNil = errors.New("value is nil") ) // Store Implements types.KVStore and CommitKVStore. type Store struct { - tree *smt.SparseMerkleTree + tree *smt.SparseMerkleTree + values dbm.DBReadWriter + // Map hashed keys back to preimage + preimages dbm.DBReadWriter } // An smt.MapStore that wraps Get to raise smt.InvalidKeyError; // smt.SparseMerkleTree expects this error to be returned when a key is not found type dbMapStore struct{ dbm.DBReadWriter } -func NewStore(nodes, values dbm.DBReadWriter) *Store { +func NewStore(db dbm.DBReadWriter) *Store { + nodes := prefix.NewPrefixReadWriter(db, nodesPrefix) + values := prefix.NewPrefixReadWriter(db, valuesPrefix) + preimages := prefix.NewPrefixReadWriter(db, preimagesPrefix) return &Store{ - tree: smt.NewSparseMerkleTree(dbMapStore{nodes}, dbMapStore{values}, sha256.New()), + tree: smt.NewSparseMerkleTree(dbMapStore{nodes}, dbMapStore{values}, sha256.New()), + values: values, + preimages: preimages, } } -func LoadStore(nodes, values dbm.DBReadWriter, root []byte) *Store { +func LoadStore(db dbm.DBReadWriter, root []byte) *Store { + nodes := prefix.NewPrefixReadWriter(db, nodesPrefix) + values := prefix.NewPrefixReadWriter(db, valuesPrefix) + preimages := prefix.NewPrefixReadWriter(db, preimagesPrefix) return &Store{ - tree: smt.ImportSparseMerkleTree(dbMapStore{nodes}, dbMapStore{values}, sha256.New(), root), + tree: smt.ImportSparseMerkleTree(dbMapStore{nodes}, dbMapStore{values}, sha256.New(), root), + values: values, + preimages: preimages, } } func (s *Store) GetProof(key []byte) (*tmcrypto.ProofOps, error) { + if len(key) == 0 { + return nil, errKeyEmpty + } proof, err := s.tree.Prove(key) if err != nil { return nil, err @@ -51,11 +73,15 @@ func (s *Store) GetProof(key []byte) (*tmcrypto.ProofOps, error) { return &tmcrypto.ProofOps{Ops: []tmcrypto.ProofOp{op.ProofOp()}}, nil } +func (s *Store) GetProofICS23(key []byte) (*ics23.CommitmentProof, error) { + return createIcs23Proof(s, key) +} + func (s *Store) Root() []byte { return s.tree.Root() } // BasicKVStore interface below: -// Get returns nil iff key doesn't exist. Panics on nil key. +// Get returns nil iff key doesn't exist. Panics on nil or empty key. func (s *Store) Get(key []byte) []byte { if len(key) == 0 { panic(errKeyEmpty) @@ -67,7 +93,7 @@ func (s *Store) Get(key []byte) []byte { return val } -// Has checks if a key exists. Panics on nil key. +// Has checks if a key exists. Panics on nil or empty key. func (s *Store) Has(key []byte) bool { if len(key) == 0 { panic(errKeyEmpty) @@ -91,6 +117,8 @@ func (s *Store) Set(key []byte, value []byte) { if err != nil { panic(err) } + path := sha256.Sum256(key) + s.preimages.Set(path[:], key) } // Delete deletes the key. Panics on nil key. @@ -98,10 +126,9 @@ func (s *Store) Delete(key []byte) { if len(key) == 0 { panic(errKeyEmpty) } - _, err := s.tree.Delete(key) - if err != nil { - panic(err) - } + _, _ = s.tree.Delete(key) + path := sha256.Sum256(key) + s.preimages.Delete(path[:]) } func (ms dbMapStore) Get(key []byte) ([]byte, error) { diff --git a/store/v2/smt/store_test.go b/store/v2/smt/store_test.go index 9a7c9192a072..e02ede93fcc9 100644 --- a/store/v2/smt/store_test.go +++ b/store/v2/smt/store_test.go @@ -10,8 +10,8 @@ import ( ) func TestGetSetHasDelete(t *testing.T) { - nodes, values := memdb.NewDB(), memdb.NewDB() - s := store.NewStore(nodes.ReadWriter(), values.ReadWriter()) + db := memdb.NewDB() + s := store.NewStore(db.ReadWriter()) s.Set([]byte("foo"), []byte("bar")) assert.Equal(t, []byte("bar"), s.Get([]byte("foo"))) @@ -29,16 +29,18 @@ func TestGetSetHasDelete(t *testing.T) { } func TestLoadStore(t *testing.T) { - nodes, values := memdb.NewDB(), memdb.NewDB() - nmap, vmap := nodes.ReadWriter(), values.ReadWriter() - s := store.NewStore(nmap, vmap) + db := memdb.NewDB() + txn := db.ReadWriter() + s := store.NewStore(txn) s.Set([]byte{0}, []byte{0}) s.Set([]byte{1}, []byte{1}) s.Delete([]byte{1}) root := s.Root() - s = store.LoadStore(nmap, vmap, root) + s = store.LoadStore(txn, root) assert.Equal(t, []byte{0}, s.Get([]byte{0})) assert.False(t, s.Has([]byte{1})) + s.Set([]byte{2}, []byte{2}) + assert.NotEqual(t, root, s.Root()) } diff --git a/store/v2/types.go b/store/v2/types.go index 6975cbdc49c3..5aacfc2d4133 100644 --- a/store/v2/types.go +++ b/store/v2/types.go @@ -56,6 +56,13 @@ var ( KVStoreReversePrefixIterator = v1.KVStoreReversePrefixIterator NewStoreKVPairWriteListener = v1.NewStoreKVPairWriteListener + + ProofOpSMTCommitment = v1.ProofOpSMTCommitment + ProofOpSimpleMerkleCommitment = v1.ProofOpSimpleMerkleCommitment + + CommitmentOpDecoder = v1.CommitmentOpDecoder + ProofOpFromMap = v1.ProofOpFromMap + NewSmtCommitmentOp = v1.NewSmtCommitmentOp ) // BasicMultiStore defines a minimal interface for accessing root state.