diff --git a/Makefile b/Makefile index 003c72b11..0d59b64d4 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,15 @@ CWD ?= CURRENT_WORKING_DIRECTIONRY_NOT_SUPPLIED # `VERBOSE_TEST="" make test_persistence` is an easy way to run the same tests without verbose output VERBOSE_TEST ?= -v +# Detect OS using the $(shell uname -s) command +ifeq ($(shell uname -s),Darwin) + # Add macOS-specific commands here + SEDI = sed -i '' +else ifeq ($(shell uname -s),Linux) + # Add Linux-specific commands here + SEDI = sed -i +endif + .SILENT: .PHONY: list ## List all make targets @@ -309,6 +318,9 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the $(PROTOC_SHARED) -I=./p2p/types/proto --go_out=./p2p/types ./p2p/types/proto/*.proto # IBC + @if test ! -e "./ibc/types/proto/proofs.proto"; then \ + make download_ics23_proto; \ + fi $(PROTOC_SHARED) -I=./ibc/types/proto --go_out=./ibc/types ./ibc/types/proto/*.proto # echo "View generated proto files by running: make protogen_show" @@ -316,6 +328,16 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the # CONSIDERATION: Some proto files contain unused gRPC services so we may need to add the following # if/when we decide to include it: `grpc--go-grpc_opt=paths=source_relative --go-grpc_out=./output/path` +.PHONY: download_ics23_proto +download_ics23_proto: + echo "Downloading cosmos/ics23 proto definitions..."; \ + curl -s -o ./ibc/types/proto/proofs.proto https://raw.githubusercontent.com/cosmos/ics23/master/proto/cosmos/ics23/v1/proofs.proto; \ + $(SEDI) \ + -e '/^package/{N;d;}' \ + -e 's@github.com/.*"@github.com/pokt-network/pocket/ibc/types"@g' \ + ./ibc/types/proto/proofs.proto && \ + awk 'BEGIN { print "// ===== !! THIS IS CLONED FROM cosmos/ics23 !! =====\n" } { print }' ./ibc/types/proto/proofs.proto > tmpfile && mv tmpfile ./ibc/types/proto/proofs.proto; \ + .PHONY: protogen_docker_m1 ## TECHDEBT: Test, validate & update. protogen_docker_m1: docker_check diff --git a/ibc/types/proofs.go b/ibc/types/proofs.go new file mode 100644 index 000000000..cf25da5ab --- /dev/null +++ b/ibc/types/proofs.go @@ -0,0 +1,197 @@ +package types + +import ics23 "github.com/cosmos/ics23/go" + +// Copy of ics23.SmtSpec +// Ref: https://github.com/cosmos/ics23/blob/daa1760cb80f8607494ecf9e40482e66717a24e0/go/proof.go#L47 +var SmtSpec = &ProofSpec{ + LeafSpec: &LeafOp{ + Hash: HashOp_SHA256, + PrehashKey: HashOp_SHA256, + PrehashValue: HashOp_SHA256, + Length: LengthOp_NO_PREFIX, + Prefix: []byte{0}, + }, + InnerSpec: &InnerSpec{ + ChildOrder: []int32{0, 1}, + ChildSize: 32, + MinPrefixLength: 1, + MaxPrefixLength: 1, + EmptyChild: make([]byte, 32), + Hash: HashOp_SHA256, + }, + MaxDepth: 256, + PrehashKeyBeforeComparison: true, +} + +func (p *ProofSpec) ConvertToIcs23ProofSpec() *ics23.ProofSpec { + if p == nil { + return nil + } + ics := new(ics23.ProofSpec) + ics.LeafSpec = p.LeafSpec.convertToIcs23LeafOp() + ics.InnerSpec = p.InnerSpec.convertToIcs23InnerSpec() + ics.MaxDepth = p.MaxDepth + ics.MinDepth = p.MinDepth + ics.PrehashKeyBeforeComparison = p.PrehashKeyBeforeComparison + return ics +} + +func ConvertFromIcs23ProofSpec(p *ics23.ProofSpec) *ProofSpec { + if p == nil { + return nil + } + spc := new(ProofSpec) + spc.LeafSpec = convertFromIcs23LeafOp(p.LeafSpec) + spc.InnerSpec = convertFromIcs23InnerSpec(p.InnerSpec) + spc.MaxDepth = p.MaxDepth + spc.MinDepth = p.MinDepth + spc.PrehashKeyBeforeComparison = p.PrehashKeyBeforeComparison + return spc +} + +func (l *LeafOp) convertToIcs23LeafOp() *ics23.LeafOp { + if l == nil { + return nil + } + ics := new(ics23.LeafOp) + ics.Hash = l.Hash.convertToIcs23HashOp() + ics.PrehashKey = l.PrehashKey.convertToIcs23HashOp() + ics.PrehashValue = l.PrehashValue.convertToIcs23HashOp() + ics.Length = l.Length.convertToIcs23LenthOp() + ics.Prefix = l.Prefix + return ics +} + +func convertFromIcs23LeafOp(l *ics23.LeafOp) *LeafOp { + if l == nil { + return nil + } + op := new(LeafOp) + op.Hash = convertFromIcs23HashOp(l.Hash) + op.PrehashKey = convertFromIcs23HashOp(l.PrehashKey) + op.PrehashValue = convertFromIcs23HashOp(l.PrehashValue) + op.Length = convertFromIcs23LengthOp(l.Length) + op.Prefix = l.Prefix + return op +} + +func (i *InnerSpec) convertToIcs23InnerSpec() *ics23.InnerSpec { + if i == nil { + return nil + } + ics := new(ics23.InnerSpec) + ics.ChildOrder = i.ChildOrder + ics.ChildSize = i.ChildSize + ics.MinPrefixLength = i.MinPrefixLength + ics.MaxPrefixLength = i.MaxPrefixLength + ics.EmptyChild = i.EmptyChild + ics.Hash = i.Hash.convertToIcs23HashOp() + return ics +} + +func convertFromIcs23InnerSpec(i *ics23.InnerSpec) *InnerSpec { + if i == nil { + return nil + } + spec := new(InnerSpec) + spec.ChildOrder = i.ChildOrder + spec.ChildSize = i.ChildSize + spec.MinPrefixLength = i.MinPrefixLength + spec.MaxPrefixLength = i.MaxPrefixLength + spec.EmptyChild = i.EmptyChild + spec.Hash = convertFromIcs23HashOp(i.Hash) + return spec +} + +func (h HashOp) convertToIcs23HashOp() ics23.HashOp { + switch h { + case HashOp_NO_HASH: + return ics23.HashOp_NO_HASH + case HashOp_SHA256: + return ics23.HashOp_SHA256 + case HashOp_SHA512: + return ics23.HashOp_SHA512 + case HashOp_KECCAK: + return ics23.HashOp_KECCAK + case HashOp_RIPEMD160: + return ics23.HashOp_RIPEMD160 + case HashOp_BITCOIN: + return ics23.HashOp_BITCOIN + case HashOp_SHA512_256: + return ics23.HashOp_SHA512_256 + default: + panic("unknown hash op") + } +} + +func convertFromIcs23HashOp(h ics23.HashOp) HashOp { + switch h { + case ics23.HashOp_NO_HASH: + return HashOp_NO_HASH + case ics23.HashOp_SHA256: + return HashOp_SHA256 + case ics23.HashOp_SHA512: + return HashOp_SHA512 + case ics23.HashOp_KECCAK: + return HashOp_KECCAK + case ics23.HashOp_RIPEMD160: + return HashOp_RIPEMD160 + case ics23.HashOp_BITCOIN: + return HashOp_BITCOIN + case ics23.HashOp_SHA512_256: + return HashOp_SHA512_256 + default: + panic("unknown hash op") + } +} + +func (l LengthOp) convertToIcs23LenthOp() ics23.LengthOp { + switch l { + case LengthOp_NO_PREFIX: + return ics23.LengthOp_NO_PREFIX + case LengthOp_VAR_PROTO: + return ics23.LengthOp_VAR_PROTO + case LengthOp_VAR_RLP: + return ics23.LengthOp_VAR_RLP + case LengthOp_FIXED32_BIG: + return ics23.LengthOp_FIXED32_BIG + case LengthOp_FIXED32_LITTLE: + return ics23.LengthOp_FIXED32_LITTLE + case LengthOp_FIXED64_BIG: + return ics23.LengthOp_FIXED64_BIG + case LengthOp_FIXED64_LITTLE: + return ics23.LengthOp_FIXED64_LITTLE + case LengthOp_REQUIRE_32_BYTES: + return ics23.LengthOp_REQUIRE_32_BYTES + case LengthOp_REQUIRE_64_BYTES: + return ics23.LengthOp_REQUIRE_64_BYTES + default: + panic("unknown length op") + } +} + +func convertFromIcs23LengthOp(l ics23.LengthOp) LengthOp { + switch l { + case ics23.LengthOp_NO_PREFIX: + return LengthOp_NO_PREFIX + case ics23.LengthOp_VAR_PROTO: + return LengthOp_VAR_PROTO + case ics23.LengthOp_VAR_RLP: + return LengthOp_VAR_RLP + case ics23.LengthOp_FIXED32_BIG: + return LengthOp_FIXED32_BIG + case ics23.LengthOp_FIXED32_LITTLE: + return LengthOp_FIXED32_LITTLE + case ics23.LengthOp_FIXED64_BIG: + return LengthOp_FIXED64_BIG + case ics23.LengthOp_FIXED64_LITTLE: + return LengthOp_FIXED64_LITTLE + case ics23.LengthOp_REQUIRE_32_BYTES: + return LengthOp_REQUIRE_32_BYTES + case ics23.LengthOp_REQUIRE_64_BYTES: + return LengthOp_REQUIRE_64_BYTES + default: + panic("unknown length op") + } +} diff --git a/ibc/types/proto/proofs.proto b/ibc/types/proto/proofs.proto new file mode 100644 index 000000000..e3659215e --- /dev/null +++ b/ibc/types/proto/proofs.proto @@ -0,0 +1,238 @@ +// ===== !! THIS IS CLONED FROM cosmos/ics23 !! ===== + +syntax = "proto3"; + +option go_package = "github.com/pokt-network/pocket/ibc/types"; + +enum HashOp { + // NO_HASH is the default if no data passed. Note this is an illegal argument some places. + NO_HASH = 0; + SHA256 = 1; + SHA512 = 2; + KECCAK = 3; + RIPEMD160 = 4; + BITCOIN = 5; // ripemd160(sha256(x)) + SHA512_256 = 6; +} + +/** +LengthOp defines how to process the key and value of the LeafOp +to include length information. After encoding the length with the given +algorithm, the length will be prepended to the key and value bytes. +(Each one with it's own encoded length) +*/ +enum LengthOp { + // NO_PREFIX don't include any length info + NO_PREFIX = 0; + // VAR_PROTO uses protobuf (and go-amino) varint encoding of the length + VAR_PROTO = 1; + // VAR_RLP uses rlp int encoding of the length + VAR_RLP = 2; + // FIXED32_BIG uses big-endian encoding of the length as a 32 bit integer + FIXED32_BIG = 3; + // FIXED32_LITTLE uses little-endian encoding of the length as a 32 bit integer + FIXED32_LITTLE = 4; + // FIXED64_BIG uses big-endian encoding of the length as a 64 bit integer + FIXED64_BIG = 5; + // FIXED64_LITTLE uses little-endian encoding of the length as a 64 bit integer + FIXED64_LITTLE = 6; + // REQUIRE_32_BYTES is like NONE, but will fail if the input is not exactly 32 bytes (sha256 output) + REQUIRE_32_BYTES = 7; + // REQUIRE_64_BYTES is like NONE, but will fail if the input is not exactly 64 bytes (sha512 output) + REQUIRE_64_BYTES = 8; +} + +/** +ExistenceProof takes a key and a value and a set of steps to perform on it. +The result of peforming all these steps will provide a "root hash", which can +be compared to the value in a header. + +Since it is computationally infeasible to produce a hash collission for any of the used +cryptographic hash functions, if someone can provide a series of operations to transform +a given key and value into a root hash that matches some trusted root, these key and values +must be in the referenced merkle tree. + +The only possible issue is maliablity in LeafOp, such as providing extra prefix data, +which should be controlled by a spec. Eg. with lengthOp as NONE, + prefix = FOO, key = BAR, value = CHOICE +and + prefix = F, key = OOBAR, value = CHOICE +would produce the same value. + +With LengthOp this is tricker but not impossible. Which is why the "leafPrefixEqual" field +in the ProofSpec is valuable to prevent this mutability. And why all trees should +length-prefix the data before hashing it. +*/ +message ExistenceProof { + bytes key = 1; + bytes value = 2; + LeafOp leaf = 3; + repeated InnerOp path = 4; +} + +/* +NonExistenceProof takes a proof of two neighbors, one left of the desired key, +one right of the desired key. If both proofs are valid AND they are neighbors, +then there is no valid proof for the given key. +*/ +message NonExistenceProof { + bytes key = 1; // TODO: remove this as unnecessary??? we prove a range + ExistenceProof left = 2; + ExistenceProof right = 3; +} + +/* +CommitmentProof is either an ExistenceProof or a NonExistenceProof, or a Batch of such messages +*/ +message CommitmentProof { + oneof proof { + ExistenceProof exist = 1; + NonExistenceProof nonexist = 2; + BatchProof batch = 3; + CompressedBatchProof compressed = 4; + } +} + +/** +LeafOp represents the raw key-value data we wish to prove, and +must be flexible to represent the internal transformation from +the original key-value pairs into the basis hash, for many existing +merkle trees. + +key and value are passed in. So that the signature of this operation is: + leafOp(key, value) -> output + +To process this, first prehash the keys and values if needed (ANY means no hash in this case): + hkey = prehashKey(key) + hvalue = prehashValue(value) + +Then combine the bytes, and hash it + output = hash(prefix || length(hkey) || hkey || length(hvalue) || hvalue) +*/ +message LeafOp { + HashOp hash = 1; + HashOp prehash_key = 2; + HashOp prehash_value = 3; + LengthOp length = 4; + // prefix is a fixed bytes that may optionally be included at the beginning to differentiate + // a leaf node from an inner node. + bytes prefix = 5; +} + +/** +InnerOp represents a merkle-proof step that is not a leaf. +It represents concatenating two children and hashing them to provide the next result. + +The result of the previous step is passed in, so the signature of this op is: + innerOp(child) -> output + +The result of applying InnerOp should be: + output = op.hash(op.prefix || child || op.suffix) + + where the || operator is concatenation of binary data, +and child is the result of hashing all the tree below this step. + +Any special data, like prepending child with the length, or prepending the entire operation with +some value to differentiate from leaf nodes, should be included in prefix and suffix. +If either of prefix or suffix is empty, we just treat it as an empty string +*/ +message InnerOp { + HashOp hash = 1; + bytes prefix = 2; + bytes suffix = 3; +} + +/** +ProofSpec defines what the expected parameters are for a given proof type. +This can be stored in the client and used to validate any incoming proofs. + + verify(ProofSpec, Proof) -> Proof | Error + +As demonstrated in tests, if we don't fix the algorithm used to calculate the +LeafHash for a given tree, there are many possible key-value pairs that can +generate a given hash (by interpretting the preimage differently). +We need this for proper security, requires client knows a priori what +tree format server uses. But not in code, rather a configuration object. +*/ +message ProofSpec { + // any field in the ExistenceProof must be the same as in this spec. + // except Prefix, which is just the first bytes of prefix (spec can be longer) + LeafOp leaf_spec = 1; + InnerSpec inner_spec = 2; + // max_depth (if > 0) is the maximum number of InnerOps allowed (mainly for fixed-depth tries) + int32 max_depth = 3; + // min_depth (if > 0) is the minimum number of InnerOps allowed (mainly for fixed-depth tries) + int32 min_depth = 4; + // prehash_key_before_comparison is a flag that indicates whether to use the + // prehash_key specified by LeafOp to compare lexical ordering of keys for + // non-existence proofs. + bool prehash_key_before_comparison = 5; +} + +/* +InnerSpec contains all store-specific structure info to determine if two proofs from a +given store are neighbors. + +This enables: + + isLeftMost(spec: InnerSpec, op: InnerOp) + isRightMost(spec: InnerSpec, op: InnerOp) + isLeftNeighbor(spec: InnerSpec, left: InnerOp, right: InnerOp) +*/ +message InnerSpec { + // Child order is the ordering of the children node, must count from 0 + // iavl tree is [0, 1] (left then right) + // merk is [0, 2, 1] (left, right, here) + repeated int32 child_order = 1; + int32 child_size = 2; + int32 min_prefix_length = 3; + int32 max_prefix_length = 4; + // empty child is the prehash image that is used when one child is nil (eg. 20 bytes of 0) + bytes empty_child = 5; + // hash is the algorithm that must be used for each InnerOp + HashOp hash = 6; +} + +/* +BatchProof is a group of multiple proof types than can be compressed +*/ +message BatchProof { + repeated BatchEntry entries = 1; +} + +// Use BatchEntry not CommitmentProof, to avoid recursion +message BatchEntry { + oneof proof { + ExistenceProof exist = 1; + NonExistenceProof nonexist = 2; + } +} + +/****** all items here are compressed forms *******/ + +message CompressedBatchProof { + repeated CompressedBatchEntry entries = 1; + repeated InnerOp lookup_inners = 2; +} + +// Use BatchEntry not CommitmentProof, to avoid recursion +message CompressedBatchEntry { + oneof proof { + CompressedExistenceProof exist = 1; + CompressedNonExistenceProof nonexist = 2; + } +} + +message CompressedExistenceProof { + bytes key = 1; + bytes value = 2; + LeafOp leaf = 3; + // these are indexes into the lookup_inners table in CompressedBatchProof + repeated int32 path = 4; +} + +message CompressedNonExistenceProof { + bytes key = 1; // TODO: remove this as unnecessary??? we prove a range + CompressedExistenceProof left = 2; + CompressedExistenceProof right = 3; +}