From 672e2027751b8c35e79a71c472d99eaf004721c9 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Thu, 20 Jul 2023 12:40:22 +0100 Subject: [PATCH 1/7] Add cosmos/ics23 proto into local proto with conversion --- ibc/types/proofs.go | 107 +++++++++++++++++ ibc/types/proto/proofs.proto | 222 +++++++++++++++++++++++++++++++++++ 2 files changed, 329 insertions(+) create mode 100644 ibc/types/proofs.go create mode 100644 ibc/types/proto/proofs.proto diff --git a/ibc/types/proofs.go b/ibc/types/proofs.go new file mode 100644 index 000000000..9b0a5efb2 --- /dev/null +++ b/ibc/types/proofs.go @@ -0,0 +1,107 @@ +package types + +import ics23 "github.com/cosmos/ics23/go" + +var SmtSpec = &ProofSpec{ + LeafSpec: &LeafOp{ + Hash: HashOp_SHA256, + PrehashKey: HashOp_SHA256, + PrehashValue: HashOp_SHA256, + Length: LengthOp_NO_PREFIX, + Prefix: []byte{0}, + }, + InnerSpec: &InnerSpec{ + ChildOrder: []int32{0, 1}, + ChildSize: 32, + MinPrefixLength: 1, + MaxPrefixLength: 1, + EmptyChild: make([]byte, 32), + Hash: HashOp_SHA256, + }, + MaxDepth: 256, + PrehashKeyBeforeComparison: true, +} + +func (p *ProofSpec) ConvertToIcs23ProofSpec() *ics23.ProofSpec { + ics := new(ics23.ProofSpec) + ics.LeafSpec = p.LeafSpec.ConvertToIcs23LeafOp() + ics.InnerSpec = p.InnerSpec.ConvertToIcs23InnerSpec() + ics.MaxDepth = p.MaxDepth + ics.MinDepth = p.MinDepth + ics.PrehashKeyBeforeComparison = p.PrehashKeyBeforeComparison + return ics +} + +func (l *LeafOp) ConvertToIcs23LeafOp() *ics23.LeafOp { + ics := new(ics23.LeafOp) + ics.Hash = l.Hash.ConvertToIcs23HashOp() + ics.PrehashKey = l.PrehashKey.ConvertToIcs23HashOp() + ics.PrehashValue = l.PrehashValue.ConvertToIcs23HashOp() + ics.Length = l.Length.ConvertToIcs23LenthOp() + ics.Prefix = l.Prefix + return ics +} + +func (i *InnerSpec) ConvertToIcs23InnerSpec() *ics23.InnerSpec { + ics := new(ics23.InnerSpec) + ics.ChildOrder = i.ChildOrder + ics.MinPrefixLength = i.MinPrefixLength + ics.MaxPrefixLength = i.MaxPrefixLength + ics.EmptyChild = i.EmptyChild + ics.Hash = i.Hash.ConvertToIcs23HashOp() + return ics +} + +func (h HashOp) ConvertToIcs23HashOp() ics23.HashOp { + switch h { + case HashOp_NO_HASH: + return ics23.HashOp_NO_HASH + case HashOp_SHA256: + return ics23.HashOp_SHA256 + case HashOp_SHA512: + return ics23.HashOp_SHA512 + case HashOp_KECCAK: + return ics23.HashOp_KECCAK + case HashOp_RIPEMD160: + return ics23.HashOp_RIPEMD160 + case HashOp_BITCOIN: + return ics23.HashOp_BITCOIN + case HashOp_SHA512_256: + return ics23.HashOp_SHA512_256 + default: + panic("unknown hash op") + } +} + +func (l LengthOp) ConvertToIcs23LenthOp() ics23.LengthOp { + switch l { + case LengthOp_NO_PREFIX: + return ics23.LengthOp_NO_PREFIX + case LengthOp_VAR_PROTO: + return ics23.LengthOp_VAR_PROTO + case LengthOp_VAR_RLP: + return ics23.LengthOp_VAR_RLP + case LengthOp_FIXED32_BIG: + return ics23.LengthOp_FIXED32_BIG + case LengthOp_FIXED32_LITTLE: + return ics23.LengthOp_FIXED32_LITTLE + case LengthOp_FIXED64_BIG: + return ics23.LengthOp_FIXED64_BIG + case LengthOp_FIXED64_LITTLE: + return ics23.LengthOp_FIXED64_LITTLE + case LengthOp_REQUIRE_32_BYTES: + return ics23.LengthOp_REQUIRE_32_BYTES + case LengthOp_REQUIRE_64_BYTES: + return ics23.LengthOp_REQUIRE_64_BYTES + default: + panic("unknown length op") + } +} + +func (i *InnerOp) ConvertToIcs23InnerOp() *ics23.InnerOp { + ics := new(ics23.InnerOp) + ics.Hash = i.Hash.ConvertToIcs23HashOp() + ics.Prefix = i.Prefix + ics.Suffix = i.Suffix + return ics +} diff --git a/ibc/types/proto/proofs.proto b/ibc/types/proto/proofs.proto new file mode 100644 index 000000000..e79881ed4 --- /dev/null +++ b/ibc/types/proto/proofs.proto @@ -0,0 +1,222 @@ +syntax = "proto3"; + +// This file is a clone from the github.com/cosmos/ics23 repo, it has been +// cloned to be compiled with protoc generic. As such it will produce valid +// protobuf messages that can be serialised and cloned + +option go_package = "github.com/pokt-network/pocket/ibc/types"; + +enum HashOp { + // NO_HASH is the default if no data passed. Note this is an illegal argument some places. + NO_HASH = 0; + SHA256 = 1; + SHA512 = 2; + KECCAK = 3; + RIPEMD160 = 4; + BITCOIN = 5; // ripemd160(sha256(x)) + SHA512_256 = 6; +} + +// LengthOp defines how to process the key and value of the LeafOp +// to include length information. After encoding the length with the given +// algorithm, the length will be prepended to the key and value bytes. +// (Each one with it's own encoded length) +enum LengthOp { + // NO_PREFIX don't include any length info + NO_PREFIX = 0; + // VAR_PROTO uses protobuf (and go-amino) varint encoding of the length + VAR_PROTO = 1; + // VAR_RLP uses rlp int encoding of the length + VAR_RLP = 2; + // FIXED32_BIG uses big-endian encoding of the length as a 32 bit integer + FIXED32_BIG = 3; + // FIXED32_LITTLE uses little-endian encoding of the length as a 32 bit integer + FIXED32_LITTLE = 4; + // FIXED64_BIG uses big-endian encoding of the length as a 64 bit integer + FIXED64_BIG = 5; + // FIXED64_LITTLE uses little-endian encoding of the length as a 64 bit integer + FIXED64_LITTLE = 6; + // REQUIRE_32_BYTES is like NONE, but will fail if the input is not exactly 32 bytes (sha256 output) + REQUIRE_32_BYTES = 7; + // REQUIRE_64_BYTES is like NONE, but will fail if the input is not exactly 64 bytes (sha512 output) + REQUIRE_64_BYTES = 8; +} + +// ExistenceProof takes a key and a value and a set of steps to perform on it. +// The result of peforming all these steps will provide a "root hash", which can +// be compared to the value in a header. +// +// Since it is computationally infeasible to produce a hash collission for any of the used +// cryptographic hash functions, if someone can provide a series of operations to transform +// a given key and value into a root hash that matches some trusted root, these key and values +// must be in the referenced merkle tree. +// +// The only possible issue is maliablity in LeafOp, such as providing extra prefix data, +// which should be controlled by a spec. Eg. with lengthOp as NONE, +// prefix = FOO, key = BAR, value = CHOICE +// and +// prefix = F, key = OOBAR, value = CHOICE +// would produce the same value. +// +// With LengthOp this is tricker but not impossible. Which is why the "leafPrefixEqual" field +// in the ProofSpec is valuable to prevent this mutability. And why all trees should +// length-prefix the data before hashing it. +message ExistenceProof { + bytes key = 1; + bytes value = 2; + LeafOp leaf = 3; + repeated InnerOp path = 4; +} + +// NonExistenceProof takes a proof of two neighbors, one left of the desired key, +// one right of the desired key. If both proofs are valid AND they are neighbors, +// then there is no valid proof for the given key. +message NonExistenceProof { + bytes key = 1; // TODO: remove this as unnecessary??? we prove a range + ExistenceProof left = 2; + ExistenceProof right = 3; +} + +// CommitmentProof is either an ExistenceProof or a NonExistenceProof, or a Batch of such messages +message CommitmentProof { + oneof proof { + ExistenceProof exist = 1; + NonExistenceProof nonexist = 2; + BatchProof batch = 3; + CompressedBatchProof compressed = 4; + } +} + +// LeafOp represents the raw key-value data we wish to prove, and +// must be flexible to represent the internal transformation from +// the original key-value pairs into the basis hash, for many existing +// merkle trees. +// +// key and value are passed in. So that the signature of this operation is: +// leafOp(key, value) -> output +// +// To process this, first prehash the keys and values if needed (ANY means no hash in this case): +// hkey = prehashKey(key) +// hvalue = prehashValue(value) +// +// Then combine the bytes, and hash it +// output = hash(prefix || length(hkey) || hkey || length(hvalue) || hvalue) +message LeafOp { + HashOp hash = 1; + HashOp prehash_key = 2; + HashOp prehash_value = 3; + LengthOp length = 4; + // prefix is a fixed bytes that may optionally be included at the beginning to differentiate + // a leaf node from an inner node. + bytes prefix = 5; +} + +// InnerOp represents a merkle-proof step that is not a leaf. +// It represents concatenating two children and hashing them to provide the next result. +// +// The result of the previous step is passed in, so the signature of this op is: +// innerOp(child) -> output +// +// The result of applying InnerOp should be: +// output = op.hash(op.prefix || child || op.suffix) +// +// where the || operator is concatenation of binary data, +// and child is the result of hashing all the tree below this step. +// +// Any special data, like prepending child with the length, or prepending the entire operation with +// some value to differentiate from leaf nodes, should be included in prefix and suffix. +// If either of prefix or suffix is empty, we just treat it as an empty string +message InnerOp { + HashOp hash = 1; + bytes prefix = 2; + bytes suffix = 3; +} + +// ProofSpec defines what the expected parameters are for a given proof type. +// This can be stored in the client and used to validate any incoming proofs. +// +// verify(ProofSpec, Proof) -> Proof | Error +// +// As demonstrated in tests, if we don't fix the algorithm used to calculate the +// LeafHash for a given tree, there are many possible key-value pairs that can +// generate a given hash (by interpretting the preimage differently). +// We need this for proper security, requires client knows a priori what +// tree format server uses. But not in code, rather a configuration object. +message ProofSpec { + // any field in the ExistenceProof must be the same as in this spec. + // except Prefix, which is just the first bytes of prefix (spec can be longer) + LeafOp leaf_spec = 1; + InnerSpec inner_spec = 2; + // max_depth (if > 0) is the maximum number of InnerOps allowed (mainly for fixed-depth tries) + int32 max_depth = 3; + // min_depth (if > 0) is the minimum number of InnerOps allowed (mainly for fixed-depth tries) + int32 min_depth = 4; + // prehash_key_before_comparison is a flag that indicates whether to use the + // prehash_key specified by LeafOp to compare lexical ordering of keys for + // non-existence proofs. + bool prehash_key_before_comparison = 5; +} + +// InnerSpec contains all store-specific structure info to determine if two proofs from a +// given store are neighbors. +// +// This enables: +// +// isLeftMost(spec: InnerSpec, op: InnerOp) +// isRightMost(spec: InnerSpec, op: InnerOp) +// isLeftNeighbor(spec: InnerSpec, left: InnerOp, right: InnerOp) +message InnerSpec { + // Child order is the ordering of the children node, must count from 0 + // iavl tree is [0, 1] (left then right) + // merk is [0, 2, 1] (left, right, here) + repeated int32 child_order = 1; + int32 child_size = 2; + int32 min_prefix_length = 3; + int32 max_prefix_length = 4; + // empty child is the prehash image that is used when one child is nil (eg. 20 bytes of 0) + bytes empty_child = 5; + // hash is the algorithm that must be used for each InnerOp + HashOp hash = 6; +} + +// BatchProof is a group of multiple proof types than can be compressed +message BatchProof { + repeated BatchEntry entries = 1; +} + +// Use BatchEntry not CommitmentProof, to avoid recursion +message BatchEntry { + oneof proof { + ExistenceProof exist = 1; + NonExistenceProof nonexist = 2; + } +} + +// ====== all items here are compressed forms ======= + +message CompressedBatchProof { + repeated CompressedBatchEntry entries = 1; + repeated InnerOp lookup_inners = 2; +} + +// Use BatchEntry not CommitmentProof, to avoid recursion +message CompressedBatchEntry { + oneof proof { + CompressedExistenceProof exist = 1; + CompressedNonExistenceProof nonexist = 2; + } +} + +message CompressedExistenceProof { + bytes key = 1; + bytes value = 2; + LeafOp leaf = 3; + // these are indexes into the lookup_inners table in CompressedBatchProof + repeated int32 path = 4; +} + +message CompressedNonExistenceProof { + bytes key = 1; // TODO: remove this as unnecessary??? we prove a range + CompressedExistenceProof left = 2; + CompressedExistenceProof right = 3; +} From 513005c0ea23f103416b32edbcf87e2a3ab64eab Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 21 Jul 2023 10:02:33 +0100 Subject: [PATCH 2/7] Add makefile auto download and sed on proto --- Makefile | 20 ++++ ibc/types/proto/proofs.proto | 176 +++++++++++++++++++---------------- 2 files changed, 116 insertions(+), 80 deletions(-) diff --git a/Makefile b/Makefile index 003c72b11..192cd67a6 100644 --- a/Makefile +++ b/Makefile @@ -13,6 +13,15 @@ CWD ?= CURRENT_WORKING_DIRECTIONRY_NOT_SUPPLIED # `VERBOSE_TEST="" make test_persistence` is an easy way to run the same tests without verbose output VERBOSE_TEST ?= -v +# Detect OS using the $(shell uname -s) command +ifeq ($(shell uname -s),Darwin) + # Add macOS-specific commands here + SEDI = sed -i '' +else ifeq ($(shell uname -s),Linux) + # Add Linux-specific commands here + SEDI = sed -i +endif + .SILENT: .PHONY: list ## List all make targets @@ -309,6 +318,7 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the $(PROTOC_SHARED) -I=./p2p/types/proto --go_out=./p2p/types ./p2p/types/proto/*.proto # IBC + make copy_ics23_proto $(PROTOC_SHARED) -I=./ibc/types/proto --go_out=./ibc/types ./ibc/types/proto/*.proto # echo "View generated proto files by running: make protogen_show" @@ -316,6 +326,16 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the # CONSIDERATION: Some proto files contain unused gRPC services so we may need to add the following # if/when we decide to include it: `grpc--go-grpc_opt=paths=source_relative --go-grpc_out=./output/path` +.PHONY: copy_ics23_proto +copy_ics23_proto: + echo "Downloading cosmos/ics23 proto definitions..." + curl -s -o ./ibc/types/proto/proofs.proto https://raw.githubusercontent.com/cosmos/ics23/master/proto/cosmos/ics23/v1/proofs.proto && \ + $(SEDI) \ + -e '/^package/{N;d;}' \ + -e 's@github.com/.*"@github.com/pokt-network/pocket/ibc/types"@g' \ + ./ibc/types/proto/proofs.proto && \ + awk 'BEGIN { print "// ===== !! THIS IS CLONED FROM cosmos/ics23 !! =====\n" } { print }' ./ibc/types/proto/proofs.proto > tmpfile && mv tmpfile ./ibc/types/proto/proofs.proto + .PHONY: protogen_docker_m1 ## TECHDEBT: Test, validate & update. protogen_docker_m1: docker_check diff --git a/ibc/types/proto/proofs.proto b/ibc/types/proto/proofs.proto index e79881ed4..e3659215e 100644 --- a/ibc/types/proto/proofs.proto +++ b/ibc/types/proto/proofs.proto @@ -1,8 +1,6 @@ -syntax = "proto3"; +// ===== !! THIS IS CLONED FROM cosmos/ics23 !! ===== -// This file is a clone from the github.com/cosmos/ics23 repo, it has been -// cloned to be compiled with protoc generic. As such it will produce valid -// protobuf messages that can be serialised and cloned +syntax = "proto3"; option go_package = "github.com/pokt-network/pocket/ibc/types"; @@ -17,10 +15,12 @@ enum HashOp { SHA512_256 = 6; } -// LengthOp defines how to process the key and value of the LeafOp -// to include length information. After encoding the length with the given -// algorithm, the length will be prepended to the key and value bytes. -// (Each one with it's own encoded length) +/** +LengthOp defines how to process the key and value of the LeafOp +to include length information. After encoding the length with the given +algorithm, the length will be prepended to the key and value bytes. +(Each one with it's own encoded length) +*/ enum LengthOp { // NO_PREFIX don't include any length info NO_PREFIX = 0; @@ -42,25 +42,27 @@ enum LengthOp { REQUIRE_64_BYTES = 8; } -// ExistenceProof takes a key and a value and a set of steps to perform on it. -// The result of peforming all these steps will provide a "root hash", which can -// be compared to the value in a header. -// -// Since it is computationally infeasible to produce a hash collission for any of the used -// cryptographic hash functions, if someone can provide a series of operations to transform -// a given key and value into a root hash that matches some trusted root, these key and values -// must be in the referenced merkle tree. -// -// The only possible issue is maliablity in LeafOp, such as providing extra prefix data, -// which should be controlled by a spec. Eg. with lengthOp as NONE, -// prefix = FOO, key = BAR, value = CHOICE -// and -// prefix = F, key = OOBAR, value = CHOICE -// would produce the same value. -// -// With LengthOp this is tricker but not impossible. Which is why the "leafPrefixEqual" field -// in the ProofSpec is valuable to prevent this mutability. And why all trees should -// length-prefix the data before hashing it. +/** +ExistenceProof takes a key and a value and a set of steps to perform on it. +The result of peforming all these steps will provide a "root hash", which can +be compared to the value in a header. + +Since it is computationally infeasible to produce a hash collission for any of the used +cryptographic hash functions, if someone can provide a series of operations to transform +a given key and value into a root hash that matches some trusted root, these key and values +must be in the referenced merkle tree. + +The only possible issue is maliablity in LeafOp, such as providing extra prefix data, +which should be controlled by a spec. Eg. with lengthOp as NONE, + prefix = FOO, key = BAR, value = CHOICE +and + prefix = F, key = OOBAR, value = CHOICE +would produce the same value. + +With LengthOp this is tricker but not impossible. Which is why the "leafPrefixEqual" field +in the ProofSpec is valuable to prevent this mutability. And why all trees should +length-prefix the data before hashing it. +*/ message ExistenceProof { bytes key = 1; bytes value = 2; @@ -68,16 +70,20 @@ message ExistenceProof { repeated InnerOp path = 4; } -// NonExistenceProof takes a proof of two neighbors, one left of the desired key, -// one right of the desired key. If both proofs are valid AND they are neighbors, -// then there is no valid proof for the given key. +/* +NonExistenceProof takes a proof of two neighbors, one left of the desired key, +one right of the desired key. If both proofs are valid AND they are neighbors, +then there is no valid proof for the given key. +*/ message NonExistenceProof { bytes key = 1; // TODO: remove this as unnecessary??? we prove a range ExistenceProof left = 2; ExistenceProof right = 3; } -// CommitmentProof is either an ExistenceProof or a NonExistenceProof, or a Batch of such messages +/* +CommitmentProof is either an ExistenceProof or a NonExistenceProof, or a Batch of such messages +*/ message CommitmentProof { oneof proof { ExistenceProof exist = 1; @@ -87,20 +93,22 @@ message CommitmentProof { } } -// LeafOp represents the raw key-value data we wish to prove, and -// must be flexible to represent the internal transformation from -// the original key-value pairs into the basis hash, for many existing -// merkle trees. -// -// key and value are passed in. So that the signature of this operation is: -// leafOp(key, value) -> output -// -// To process this, first prehash the keys and values if needed (ANY means no hash in this case): -// hkey = prehashKey(key) -// hvalue = prehashValue(value) -// -// Then combine the bytes, and hash it -// output = hash(prefix || length(hkey) || hkey || length(hvalue) || hvalue) +/** +LeafOp represents the raw key-value data we wish to prove, and +must be flexible to represent the internal transformation from +the original key-value pairs into the basis hash, for many existing +merkle trees. + +key and value are passed in. So that the signature of this operation is: + leafOp(key, value) -> output + +To process this, first prehash the keys and values if needed (ANY means no hash in this case): + hkey = prehashKey(key) + hvalue = prehashValue(value) + +Then combine the bytes, and hash it + output = hash(prefix || length(hkey) || hkey || length(hvalue) || hvalue) +*/ message LeafOp { HashOp hash = 1; HashOp prehash_key = 2; @@ -111,37 +119,41 @@ message LeafOp { bytes prefix = 5; } -// InnerOp represents a merkle-proof step that is not a leaf. -// It represents concatenating two children and hashing them to provide the next result. -// -// The result of the previous step is passed in, so the signature of this op is: -// innerOp(child) -> output -// -// The result of applying InnerOp should be: -// output = op.hash(op.prefix || child || op.suffix) -// -// where the || operator is concatenation of binary data, -// and child is the result of hashing all the tree below this step. -// -// Any special data, like prepending child with the length, or prepending the entire operation with -// some value to differentiate from leaf nodes, should be included in prefix and suffix. -// If either of prefix or suffix is empty, we just treat it as an empty string +/** +InnerOp represents a merkle-proof step that is not a leaf. +It represents concatenating two children and hashing them to provide the next result. + +The result of the previous step is passed in, so the signature of this op is: + innerOp(child) -> output + +The result of applying InnerOp should be: + output = op.hash(op.prefix || child || op.suffix) + + where the || operator is concatenation of binary data, +and child is the result of hashing all the tree below this step. + +Any special data, like prepending child with the length, or prepending the entire operation with +some value to differentiate from leaf nodes, should be included in prefix and suffix. +If either of prefix or suffix is empty, we just treat it as an empty string +*/ message InnerOp { HashOp hash = 1; bytes prefix = 2; bytes suffix = 3; } -// ProofSpec defines what the expected parameters are for a given proof type. -// This can be stored in the client and used to validate any incoming proofs. -// -// verify(ProofSpec, Proof) -> Proof | Error -// -// As demonstrated in tests, if we don't fix the algorithm used to calculate the -// LeafHash for a given tree, there are many possible key-value pairs that can -// generate a given hash (by interpretting the preimage differently). -// We need this for proper security, requires client knows a priori what -// tree format server uses. But not in code, rather a configuration object. +/** +ProofSpec defines what the expected parameters are for a given proof type. +This can be stored in the client and used to validate any incoming proofs. + + verify(ProofSpec, Proof) -> Proof | Error + +As demonstrated in tests, if we don't fix the algorithm used to calculate the +LeafHash for a given tree, there are many possible key-value pairs that can +generate a given hash (by interpretting the preimage differently). +We need this for proper security, requires client knows a priori what +tree format server uses. But not in code, rather a configuration object. +*/ message ProofSpec { // any field in the ExistenceProof must be the same as in this spec. // except Prefix, which is just the first bytes of prefix (spec can be longer) @@ -157,14 +169,16 @@ message ProofSpec { bool prehash_key_before_comparison = 5; } -// InnerSpec contains all store-specific structure info to determine if two proofs from a -// given store are neighbors. -// -// This enables: -// -// isLeftMost(spec: InnerSpec, op: InnerOp) -// isRightMost(spec: InnerSpec, op: InnerOp) -// isLeftNeighbor(spec: InnerSpec, left: InnerOp, right: InnerOp) +/* +InnerSpec contains all store-specific structure info to determine if two proofs from a +given store are neighbors. + +This enables: + + isLeftMost(spec: InnerSpec, op: InnerOp) + isRightMost(spec: InnerSpec, op: InnerOp) + isLeftNeighbor(spec: InnerSpec, left: InnerOp, right: InnerOp) +*/ message InnerSpec { // Child order is the ordering of the children node, must count from 0 // iavl tree is [0, 1] (left then right) @@ -179,7 +193,9 @@ message InnerSpec { HashOp hash = 6; } -// BatchProof is a group of multiple proof types than can be compressed +/* +BatchProof is a group of multiple proof types than can be compressed +*/ message BatchProof { repeated BatchEntry entries = 1; } @@ -192,7 +208,7 @@ message BatchEntry { } } -// ====== all items here are compressed forms ======= +/****** all items here are compressed forms *******/ message CompressedBatchProof { repeated CompressedBatchEntry entries = 1; From 0ce694ab5db2c889c707baa5faa1f7163c35089f Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Fri, 21 Jul 2023 21:46:59 +0100 Subject: [PATCH 3/7] add missing conversion field --- ibc/types/proofs.go | 1 + 1 file changed, 1 insertion(+) diff --git a/ibc/types/proofs.go b/ibc/types/proofs.go index 9b0a5efb2..2340a48b2 100644 --- a/ibc/types/proofs.go +++ b/ibc/types/proofs.go @@ -45,6 +45,7 @@ func (l *LeafOp) ConvertToIcs23LeafOp() *ics23.LeafOp { func (i *InnerSpec) ConvertToIcs23InnerSpec() *ics23.InnerSpec { ics := new(ics23.InnerSpec) ics.ChildOrder = i.ChildOrder + ics.ChildSize = i.ChildSize ics.MinPrefixLength = i.MinPrefixLength ics.MaxPrefixLength = i.MaxPrefixLength ics.EmptyChild = i.EmptyChild From ac50e065fe8d23916ae8cca1626cd9ef9bda5314 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Sun, 23 Jul 2023 14:31:34 +0100 Subject: [PATCH 4/7] pick: conversion funcs --- ibc/types/proofs.go | 109 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 109 insertions(+) diff --git a/ibc/types/proofs.go b/ibc/types/proofs.go index 2340a48b2..c9f1b54ef 100644 --- a/ibc/types/proofs.go +++ b/ibc/types/proofs.go @@ -23,6 +23,9 @@ var SmtSpec = &ProofSpec{ } func (p *ProofSpec) ConvertToIcs23ProofSpec() *ics23.ProofSpec { + if p == nil { + return nil + } ics := new(ics23.ProofSpec) ics.LeafSpec = p.LeafSpec.ConvertToIcs23LeafOp() ics.InnerSpec = p.InnerSpec.ConvertToIcs23InnerSpec() @@ -32,7 +35,23 @@ func (p *ProofSpec) ConvertToIcs23ProofSpec() *ics23.ProofSpec { return ics } +func ConvertFromIcs23ProofSpec(p *ics23.ProofSpec) *ProofSpec { + if p == nil { + return nil + } + spc := new(ProofSpec) + spc.LeafSpec = ConvertFromIcs23LeafOp(p.LeafSpec) + spc.InnerSpec = ConvertFromIcs23InnerSpec(p.InnerSpec) + spc.MaxDepth = p.MaxDepth + spc.MinDepth = p.MinDepth + spc.PrehashKeyBeforeComparison = p.PrehashKeyBeforeComparison + return spc +} + func (l *LeafOp) ConvertToIcs23LeafOp() *ics23.LeafOp { + if l == nil { + return nil + } ics := new(ics23.LeafOp) ics.Hash = l.Hash.ConvertToIcs23HashOp() ics.PrehashKey = l.PrehashKey.ConvertToIcs23HashOp() @@ -42,7 +61,23 @@ func (l *LeafOp) ConvertToIcs23LeafOp() *ics23.LeafOp { return ics } +func ConvertFromIcs23LeafOp(l *ics23.LeafOp) *LeafOp { + if l == nil { + return nil + } + op := new(LeafOp) + op.Hash = ConvertFromIcs23HashOp(l.Hash) + op.PrehashKey = ConvertFromIcs23HashOp(l.PrehashKey) + op.PrehashValue = ConvertFromIcs23HashOp(l.PrehashValue) + op.Length = ConvertFromIcs23LengthOp(l.Length) + op.Prefix = l.Prefix + return op +} + func (i *InnerSpec) ConvertToIcs23InnerSpec() *ics23.InnerSpec { + if i == nil { + return nil + } ics := new(ics23.InnerSpec) ics.ChildOrder = i.ChildOrder ics.ChildSize = i.ChildSize @@ -53,6 +88,20 @@ func (i *InnerSpec) ConvertToIcs23InnerSpec() *ics23.InnerSpec { return ics } +func ConvertFromIcs23InnerSpec(i *ics23.InnerSpec) *InnerSpec { + if i == nil { + return nil + } + spec := new(InnerSpec) + spec.ChildOrder = i.ChildOrder + spec.ChildSize = i.ChildSize + spec.MinPrefixLength = i.MinPrefixLength + spec.MaxPrefixLength = i.MaxPrefixLength + spec.EmptyChild = i.EmptyChild + spec.Hash = ConvertFromIcs23HashOp(i.Hash) + return spec +} + func (h HashOp) ConvertToIcs23HashOp() ics23.HashOp { switch h { case HashOp_NO_HASH: @@ -74,6 +123,27 @@ func (h HashOp) ConvertToIcs23HashOp() ics23.HashOp { } } +func ConvertFromIcs23HashOp(h ics23.HashOp) HashOp { + switch h { + case ics23.HashOp_NO_HASH: + return HashOp_NO_HASH + case ics23.HashOp_SHA256: + return HashOp_SHA256 + case ics23.HashOp_SHA512: + return HashOp_SHA512 + case ics23.HashOp_KECCAK: + return HashOp_KECCAK + case ics23.HashOp_RIPEMD160: + return HashOp_RIPEMD160 + case ics23.HashOp_BITCOIN: + return HashOp_BITCOIN + case ics23.HashOp_SHA512_256: + return HashOp_SHA512_256 + default: + panic("unknown hash op") + } +} + func (l LengthOp) ConvertToIcs23LenthOp() ics23.LengthOp { switch l { case LengthOp_NO_PREFIX: @@ -99,10 +169,49 @@ func (l LengthOp) ConvertToIcs23LenthOp() ics23.LengthOp { } } +func ConvertFromIcs23LengthOp(l ics23.LengthOp) LengthOp { + switch l { + case ics23.LengthOp_NO_PREFIX: + return LengthOp_NO_PREFIX + case ics23.LengthOp_VAR_PROTO: + return LengthOp_VAR_PROTO + case ics23.LengthOp_VAR_RLP: + return LengthOp_VAR_RLP + case ics23.LengthOp_FIXED32_BIG: + return LengthOp_FIXED32_BIG + case ics23.LengthOp_FIXED32_LITTLE: + return LengthOp_FIXED32_LITTLE + case ics23.LengthOp_FIXED64_BIG: + return LengthOp_FIXED64_BIG + case ics23.LengthOp_FIXED64_LITTLE: + return LengthOp_FIXED64_LITTLE + case ics23.LengthOp_REQUIRE_32_BYTES: + return LengthOp_REQUIRE_32_BYTES + case ics23.LengthOp_REQUIRE_64_BYTES: + return LengthOp_REQUIRE_64_BYTES + default: + panic("unknown length op") + } +} + func (i *InnerOp) ConvertToIcs23InnerOp() *ics23.InnerOp { + if i == nil { + return nil + } ics := new(ics23.InnerOp) ics.Hash = i.Hash.ConvertToIcs23HashOp() ics.Prefix = i.Prefix ics.Suffix = i.Suffix return ics } + +func ConvertFromIcs23InnerOp(i *ics23.InnerOp) *InnerOp { + if i == nil { + return nil + } + op := new(InnerOp) + op.Hash = ConvertFromIcs23HashOp(i.Hash) + op.Prefix = i.Prefix + op.Suffix = i.Suffix + return op +} From 06cb90e6e3b6e78ed68b060a1bfc0076ef29ed4b Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Sun, 23 Jul 2023 14:57:56 +0100 Subject: [PATCH 5/7] unexport methods --- ibc/types/proofs.go | 52 ++++++++++++++++++++++----------------------- 1 file changed, 26 insertions(+), 26 deletions(-) diff --git a/ibc/types/proofs.go b/ibc/types/proofs.go index c9f1b54ef..cc9c90ed7 100644 --- a/ibc/types/proofs.go +++ b/ibc/types/proofs.go @@ -27,8 +27,8 @@ func (p *ProofSpec) ConvertToIcs23ProofSpec() *ics23.ProofSpec { return nil } ics := new(ics23.ProofSpec) - ics.LeafSpec = p.LeafSpec.ConvertToIcs23LeafOp() - ics.InnerSpec = p.InnerSpec.ConvertToIcs23InnerSpec() + ics.LeafSpec = p.LeafSpec.convertToIcs23LeafOp() + ics.InnerSpec = p.InnerSpec.convertToIcs23InnerSpec() ics.MaxDepth = p.MaxDepth ics.MinDepth = p.MinDepth ics.PrehashKeyBeforeComparison = p.PrehashKeyBeforeComparison @@ -40,41 +40,41 @@ func ConvertFromIcs23ProofSpec(p *ics23.ProofSpec) *ProofSpec { return nil } spc := new(ProofSpec) - spc.LeafSpec = ConvertFromIcs23LeafOp(p.LeafSpec) - spc.InnerSpec = ConvertFromIcs23InnerSpec(p.InnerSpec) + spc.LeafSpec = convertFromIcs23LeafOp(p.LeafSpec) + spc.InnerSpec = convertFromIcs23InnerSpec(p.InnerSpec) spc.MaxDepth = p.MaxDepth spc.MinDepth = p.MinDepth spc.PrehashKeyBeforeComparison = p.PrehashKeyBeforeComparison return spc } -func (l *LeafOp) ConvertToIcs23LeafOp() *ics23.LeafOp { +func (l *LeafOp) convertToIcs23LeafOp() *ics23.LeafOp { if l == nil { return nil } ics := new(ics23.LeafOp) - ics.Hash = l.Hash.ConvertToIcs23HashOp() - ics.PrehashKey = l.PrehashKey.ConvertToIcs23HashOp() - ics.PrehashValue = l.PrehashValue.ConvertToIcs23HashOp() - ics.Length = l.Length.ConvertToIcs23LenthOp() + ics.Hash = l.Hash.convertToIcs23HashOp() + ics.PrehashKey = l.PrehashKey.convertToIcs23HashOp() + ics.PrehashValue = l.PrehashValue.convertToIcs23HashOp() + ics.Length = l.Length.convertToIcs23LenthOp() ics.Prefix = l.Prefix return ics } -func ConvertFromIcs23LeafOp(l *ics23.LeafOp) *LeafOp { +func convertFromIcs23LeafOp(l *ics23.LeafOp) *LeafOp { if l == nil { return nil } op := new(LeafOp) - op.Hash = ConvertFromIcs23HashOp(l.Hash) - op.PrehashKey = ConvertFromIcs23HashOp(l.PrehashKey) - op.PrehashValue = ConvertFromIcs23HashOp(l.PrehashValue) - op.Length = ConvertFromIcs23LengthOp(l.Length) + op.Hash = convertFromIcs23HashOp(l.Hash) + op.PrehashKey = convertFromIcs23HashOp(l.PrehashKey) + op.PrehashValue = convertFromIcs23HashOp(l.PrehashValue) + op.Length = convertFromIcs23LengthOp(l.Length) op.Prefix = l.Prefix return op } -func (i *InnerSpec) ConvertToIcs23InnerSpec() *ics23.InnerSpec { +func (i *InnerSpec) convertToIcs23InnerSpec() *ics23.InnerSpec { if i == nil { return nil } @@ -84,11 +84,11 @@ func (i *InnerSpec) ConvertToIcs23InnerSpec() *ics23.InnerSpec { ics.MinPrefixLength = i.MinPrefixLength ics.MaxPrefixLength = i.MaxPrefixLength ics.EmptyChild = i.EmptyChild - ics.Hash = i.Hash.ConvertToIcs23HashOp() + ics.Hash = i.Hash.convertToIcs23HashOp() return ics } -func ConvertFromIcs23InnerSpec(i *ics23.InnerSpec) *InnerSpec { +func convertFromIcs23InnerSpec(i *ics23.InnerSpec) *InnerSpec { if i == nil { return nil } @@ -98,11 +98,11 @@ func ConvertFromIcs23InnerSpec(i *ics23.InnerSpec) *InnerSpec { spec.MinPrefixLength = i.MinPrefixLength spec.MaxPrefixLength = i.MaxPrefixLength spec.EmptyChild = i.EmptyChild - spec.Hash = ConvertFromIcs23HashOp(i.Hash) + spec.Hash = convertFromIcs23HashOp(i.Hash) return spec } -func (h HashOp) ConvertToIcs23HashOp() ics23.HashOp { +func (h HashOp) convertToIcs23HashOp() ics23.HashOp { switch h { case HashOp_NO_HASH: return ics23.HashOp_NO_HASH @@ -123,7 +123,7 @@ func (h HashOp) ConvertToIcs23HashOp() ics23.HashOp { } } -func ConvertFromIcs23HashOp(h ics23.HashOp) HashOp { +func convertFromIcs23HashOp(h ics23.HashOp) HashOp { switch h { case ics23.HashOp_NO_HASH: return HashOp_NO_HASH @@ -144,7 +144,7 @@ func ConvertFromIcs23HashOp(h ics23.HashOp) HashOp { } } -func (l LengthOp) ConvertToIcs23LenthOp() ics23.LengthOp { +func (l LengthOp) convertToIcs23LenthOp() ics23.LengthOp { switch l { case LengthOp_NO_PREFIX: return ics23.LengthOp_NO_PREFIX @@ -169,7 +169,7 @@ func (l LengthOp) ConvertToIcs23LenthOp() ics23.LengthOp { } } -func ConvertFromIcs23LengthOp(l ics23.LengthOp) LengthOp { +func convertFromIcs23LengthOp(l ics23.LengthOp) LengthOp { switch l { case ics23.LengthOp_NO_PREFIX: return LengthOp_NO_PREFIX @@ -194,23 +194,23 @@ func ConvertFromIcs23LengthOp(l ics23.LengthOp) LengthOp { } } -func (i *InnerOp) ConvertToIcs23InnerOp() *ics23.InnerOp { +func (i *InnerOp) convertToIcs23InnerOp() *ics23.InnerOp { if i == nil { return nil } ics := new(ics23.InnerOp) - ics.Hash = i.Hash.ConvertToIcs23HashOp() + ics.Hash = i.Hash.convertToIcs23HashOp() ics.Prefix = i.Prefix ics.Suffix = i.Suffix return ics } -func ConvertFromIcs23InnerOp(i *ics23.InnerOp) *InnerOp { +func convertFromIcs23InnerOp(i *ics23.InnerOp) *InnerOp { if i == nil { return nil } op := new(InnerOp) - op.Hash = ConvertFromIcs23HashOp(i.Hash) + op.Hash = convertFromIcs23HashOp(i.Hash) op.Prefix = i.Prefix op.Suffix = i.Suffix return op From 064f69cd1a59fb7392b861b5dfac2ee4bc8e3332 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Sun, 23 Jul 2023 15:20:44 +0100 Subject: [PATCH 6/7] remove unused functions --- ibc/types/proofs.go | 22 ---------------------- 1 file changed, 22 deletions(-) diff --git a/ibc/types/proofs.go b/ibc/types/proofs.go index cc9c90ed7..5147ea496 100644 --- a/ibc/types/proofs.go +++ b/ibc/types/proofs.go @@ -193,25 +193,3 @@ func convertFromIcs23LengthOp(l ics23.LengthOp) LengthOp { panic("unknown length op") } } - -func (i *InnerOp) convertToIcs23InnerOp() *ics23.InnerOp { - if i == nil { - return nil - } - ics := new(ics23.InnerOp) - ics.Hash = i.Hash.convertToIcs23HashOp() - ics.Prefix = i.Prefix - ics.Suffix = i.Suffix - return ics -} - -func convertFromIcs23InnerOp(i *ics23.InnerOp) *InnerOp { - if i == nil { - return nil - } - op := new(InnerOp) - op.Hash = convertFromIcs23HashOp(i.Hash) - op.Prefix = i.Prefix - op.Suffix = i.Suffix - return op -} From 2cfcbf1c59d56f8236d51cf8986e59fe83e811d4 Mon Sep 17 00:00:00 2001 From: harry <53987565+h5law@users.noreply.github.com> Date: Mon, 24 Jul 2023 23:27:20 +0100 Subject: [PATCH 7/7] chore: dont download unless file doesnt exist --- Makefile | 18 ++++++++++-------- ibc/types/proofs.go | 2 ++ 2 files changed, 12 insertions(+), 8 deletions(-) diff --git a/Makefile b/Makefile index 192cd67a6..0d59b64d4 100644 --- a/Makefile +++ b/Makefile @@ -318,7 +318,9 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the $(PROTOC_SHARED) -I=./p2p/types/proto --go_out=./p2p/types ./p2p/types/proto/*.proto # IBC - make copy_ics23_proto + @if test ! -e "./ibc/types/proto/proofs.proto"; then \ + make download_ics23_proto; \ + fi $(PROTOC_SHARED) -I=./ibc/types/proto --go_out=./ibc/types ./ibc/types/proto/*.proto # echo "View generated proto files by running: make protogen_show" @@ -326,15 +328,15 @@ protogen_local: go_protoc-go-inject-tag ## Generate go structures for all of the # CONSIDERATION: Some proto files contain unused gRPC services so we may need to add the following # if/when we decide to include it: `grpc--go-grpc_opt=paths=source_relative --go-grpc_out=./output/path` -.PHONY: copy_ics23_proto -copy_ics23_proto: - echo "Downloading cosmos/ics23 proto definitions..." - curl -s -o ./ibc/types/proto/proofs.proto https://raw.githubusercontent.com/cosmos/ics23/master/proto/cosmos/ics23/v1/proofs.proto && \ +.PHONY: download_ics23_proto +download_ics23_proto: + echo "Downloading cosmos/ics23 proto definitions..."; \ + curl -s -o ./ibc/types/proto/proofs.proto https://raw.githubusercontent.com/cosmos/ics23/master/proto/cosmos/ics23/v1/proofs.proto; \ $(SEDI) \ - -e '/^package/{N;d;}' \ - -e 's@github.com/.*"@github.com/pokt-network/pocket/ibc/types"@g' \ + -e '/^package/{N;d;}' \ + -e 's@github.com/.*"@github.com/pokt-network/pocket/ibc/types"@g' \ ./ibc/types/proto/proofs.proto && \ - awk 'BEGIN { print "// ===== !! THIS IS CLONED FROM cosmos/ics23 !! =====\n" } { print }' ./ibc/types/proto/proofs.proto > tmpfile && mv tmpfile ./ibc/types/proto/proofs.proto + awk 'BEGIN { print "// ===== !! THIS IS CLONED FROM cosmos/ics23 !! =====\n" } { print }' ./ibc/types/proto/proofs.proto > tmpfile && mv tmpfile ./ibc/types/proto/proofs.proto; \ .PHONY: protogen_docker_m1 ## TECHDEBT: Test, validate & update. diff --git a/ibc/types/proofs.go b/ibc/types/proofs.go index 5147ea496..cf25da5ab 100644 --- a/ibc/types/proofs.go +++ b/ibc/types/proofs.go @@ -2,6 +2,8 @@ package types import ics23 "github.com/cosmos/ics23/go" +// Copy of ics23.SmtSpec +// Ref: https://github.com/cosmos/ics23/blob/daa1760cb80f8607494ecf9e40482e66717a24e0/go/proof.go#L47 var SmtSpec = &ProofSpec{ LeafSpec: &LeafOp{ Hash: HashOp_SHA256,