diff --git a/CHANGELOG.md b/CHANGELOG.md index 6016b7cce..137a0df7c 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Improvements - [#703](https://github.com/cosmos/iavl/pull/703) New APIs `NewCompressExporter`/`NewCompressImporter` to support more compact snapshot format. +- [#726](https://github.com/cosmos/iavl/pull/726) Make `KVPair` and `ChangeSet` serializable with protobuf. ### Breaking Changes diff --git a/buf.gen.yaml b/buf.gen.yaml new file mode 100644 index 000000000..3be58fde1 --- /dev/null +++ b/buf.gen.yaml @@ -0,0 +1,11 @@ +version: v1 +managed: + enabled: true + go_package_prefix: + default: github.com/cosmos/iavl/gen + except: + - buf.build/cosmos/gogo-proto +plugins: + - plugin: buf.build/protocolbuffers/go + out: gen + opt: paths=source_relative diff --git a/diff.go b/diff.go index bed4feb83..c1159f0a5 100644 --- a/diff.go +++ b/diff.go @@ -2,18 +2,14 @@ package iavl import ( "bytes" -) -// ChangeSet represents the state changes extracted from diffing iavl versions. -type ChangeSet struct { - Pairs []KVPair -} + v1 "github.com/cosmos/iavl/gen/v1" +) -type KVPair struct { - Delete bool - Key []byte - Value []byte -} +type ( + KVPair = v1.KVPair + ChangeSet = v1.ChangeSet +) // KVPairReceiver is callback parameter of method `extractStateChanges` to receive stream of `KVPair`s. type KVPairReceiver func(pair *KVPair) error diff --git a/diff_test.go b/diff_test.go index baa87c982..6b9363c86 100644 --- a/diff_test.go +++ b/diff_test.go @@ -42,14 +42,14 @@ func genChangeSets(r *rand.Rand, n int) []ChangeSet { var changeSets []ChangeSet for i := 0; i < n; i++ { - items := make(map[string]KVPair) + items := make(map[string]*KVPair) start, count, step := r.Int63n(1000), r.Int63n(1000), r.Int63n(10) for i := start; i < start+count*step; i += step { value := make([]byte, 8) binary.LittleEndian.PutUint64(value, uint64(i)) key := fmt.Sprintf("test-%d", i) - items[key] = KVPair{ + items[key] = &KVPair{ Key: []byte(key), Value: value, } @@ -65,7 +65,7 @@ func genChangeSets(r *rand.Rand, n int) []ChangeSet { if pair.Delete { continue } - items[string(pair.Key)] = KVPair{ + items[string(pair.Key)] = &KVPair{ Key: pair.Key, Delete: true, } @@ -77,7 +77,7 @@ func genChangeSets(r *rand.Rand, n int) []ChangeSet { i := r.Int63n(int64(len(lastChangeSet.Pairs))) pair := lastChangeSet.Pairs[i] if !pair.Delete { - items[string(pair.Key)] = KVPair{ + items[string(pair.Key)] = &KVPair{ Key: pair.Key, Value: pair.Value, } diff --git a/gen/v1/changeset.pb.go b/gen/v1/changeset.pb.go new file mode 100644 index 000000000..6f6634ee4 --- /dev/null +++ b/gen/v1/changeset.pb.go @@ -0,0 +1,230 @@ +// Code generated by protoc-gen-go. DO NOT EDIT. +// versions: +// protoc-gen-go v1.30.0 +// protoc (unknown) +// source: v1/changeset.proto + +package v1 + +import ( + protoreflect "google.golang.org/protobuf/reflect/protoreflect" + protoimpl "google.golang.org/protobuf/runtime/protoimpl" + reflect "reflect" + sync "sync" +) + +const ( + // Verify that this generated code is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(20 - protoimpl.MinVersion) + // Verify that runtime/protoimpl is sufficiently up-to-date. + _ = protoimpl.EnforceVersion(protoimpl.MaxVersion - 20) +) + +type KVPair struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Delete bool `protobuf:"varint,1,opt,name=delete,proto3" json:"delete,omitempty"` + Key []byte `protobuf:"bytes,2,opt,name=key,proto3" json:"key,omitempty"` + Value []byte `protobuf:"bytes,3,opt,name=value,proto3" json:"value,omitempty"` +} + +func (x *KVPair) Reset() { + *x = KVPair{} + if protoimpl.UnsafeEnabled { + mi := &file_v1_changeset_proto_msgTypes[0] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *KVPair) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*KVPair) ProtoMessage() {} + +func (x *KVPair) ProtoReflect() protoreflect.Message { + mi := &file_v1_changeset_proto_msgTypes[0] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use KVPair.ProtoReflect.Descriptor instead. +func (*KVPair) Descriptor() ([]byte, []int) { + return file_v1_changeset_proto_rawDescGZIP(), []int{0} +} + +func (x *KVPair) GetDelete() bool { + if x != nil { + return x.Delete + } + return false +} + +func (x *KVPair) GetKey() []byte { + if x != nil { + return x.Key + } + return nil +} + +func (x *KVPair) GetValue() []byte { + if x != nil { + return x.Value + } + return nil +} + +type ChangeSet struct { + state protoimpl.MessageState + sizeCache protoimpl.SizeCache + unknownFields protoimpl.UnknownFields + + Pairs []*KVPair `protobuf:"bytes,1,rep,name=pairs,proto3" json:"pairs,omitempty"` +} + +func (x *ChangeSet) Reset() { + *x = ChangeSet{} + if protoimpl.UnsafeEnabled { + mi := &file_v1_changeset_proto_msgTypes[1] + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + ms.StoreMessageInfo(mi) + } +} + +func (x *ChangeSet) String() string { + return protoimpl.X.MessageStringOf(x) +} + +func (*ChangeSet) ProtoMessage() {} + +func (x *ChangeSet) ProtoReflect() protoreflect.Message { + mi := &file_v1_changeset_proto_msgTypes[1] + if protoimpl.UnsafeEnabled && x != nil { + ms := protoimpl.X.MessageStateOf(protoimpl.Pointer(x)) + if ms.LoadMessageInfo() == nil { + ms.StoreMessageInfo(mi) + } + return ms + } + return mi.MessageOf(x) +} + +// Deprecated: Use ChangeSet.ProtoReflect.Descriptor instead. +func (*ChangeSet) Descriptor() ([]byte, []int) { + return file_v1_changeset_proto_rawDescGZIP(), []int{1} +} + +func (x *ChangeSet) GetPairs() []*KVPair { + if x != nil { + return x.Pairs + } + return nil +} + +var File_v1_changeset_proto protoreflect.FileDescriptor + +var file_v1_changeset_proto_rawDesc = []byte{ + 0x0a, 0x12, 0x76, 0x31, 0x2f, 0x63, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x65, 0x74, 0x2e, 0x70, + 0x72, 0x6f, 0x74, 0x6f, 0x12, 0x04, 0x69, 0x61, 0x76, 0x6c, 0x22, 0x48, 0x0a, 0x06, 0x4b, 0x56, + 0x50, 0x61, 0x69, 0x72, 0x12, 0x16, 0x0a, 0x06, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x18, 0x01, + 0x20, 0x01, 0x28, 0x08, 0x52, 0x06, 0x64, 0x65, 0x6c, 0x65, 0x74, 0x65, 0x12, 0x10, 0x0a, 0x03, + 0x6b, 0x65, 0x79, 0x18, 0x02, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x03, 0x6b, 0x65, 0x79, 0x12, 0x14, + 0x0a, 0x05, 0x76, 0x61, 0x6c, 0x75, 0x65, 0x18, 0x03, 0x20, 0x01, 0x28, 0x0c, 0x52, 0x05, 0x76, + 0x61, 0x6c, 0x75, 0x65, 0x22, 0x2f, 0x0a, 0x09, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x53, 0x65, + 0x74, 0x12, 0x22, 0x0a, 0x05, 0x70, 0x61, 0x69, 0x72, 0x73, 0x18, 0x01, 0x20, 0x03, 0x28, 0x0b, + 0x32, 0x0c, 0x2e, 0x69, 0x61, 0x76, 0x6c, 0x2e, 0x4b, 0x56, 0x50, 0x61, 0x69, 0x72, 0x52, 0x05, + 0x70, 0x61, 0x69, 0x72, 0x73, 0x42, 0x69, 0x0a, 0x08, 0x63, 0x6f, 0x6d, 0x2e, 0x69, 0x61, 0x76, + 0x6c, 0x42, 0x0e, 0x43, 0x68, 0x61, 0x6e, 0x67, 0x65, 0x73, 0x65, 0x74, 0x50, 0x72, 0x6f, 0x74, + 0x6f, 0x50, 0x01, 0x5a, 0x1d, 0x67, 0x69, 0x74, 0x68, 0x75, 0x62, 0x2e, 0x63, 0x6f, 0x6d, 0x2f, + 0x63, 0x6f, 0x73, 0x6d, 0x6f, 0x73, 0x2f, 0x69, 0x61, 0x76, 0x6c, 0x2f, 0x67, 0x65, 0x6e, 0x2f, + 0x76, 0x31, 0xa2, 0x02, 0x03, 0x49, 0x58, 0x58, 0xaa, 0x02, 0x04, 0x49, 0x61, 0x76, 0x6c, 0xca, + 0x02, 0x04, 0x49, 0x61, 0x76, 0x6c, 0xe2, 0x02, 0x10, 0x49, 0x61, 0x76, 0x6c, 0x5c, 0x47, 0x50, + 0x42, 0x4d, 0x65, 0x74, 0x61, 0x64, 0x61, 0x74, 0x61, 0xea, 0x02, 0x04, 0x49, 0x61, 0x76, 0x6c, + 0x62, 0x06, 0x70, 0x72, 0x6f, 0x74, 0x6f, 0x33, +} + +var ( + file_v1_changeset_proto_rawDescOnce sync.Once + file_v1_changeset_proto_rawDescData = file_v1_changeset_proto_rawDesc +) + +func file_v1_changeset_proto_rawDescGZIP() []byte { + file_v1_changeset_proto_rawDescOnce.Do(func() { + file_v1_changeset_proto_rawDescData = protoimpl.X.CompressGZIP(file_v1_changeset_proto_rawDescData) + }) + return file_v1_changeset_proto_rawDescData +} + +var file_v1_changeset_proto_msgTypes = make([]protoimpl.MessageInfo, 2) +var file_v1_changeset_proto_goTypes = []interface{}{ + (*KVPair)(nil), // 0: iavl.KVPair + (*ChangeSet)(nil), // 1: iavl.ChangeSet +} +var file_v1_changeset_proto_depIdxs = []int32{ + 0, // 0: iavl.ChangeSet.pairs:type_name -> iavl.KVPair + 1, // [1:1] is the sub-list for method output_type + 1, // [1:1] is the sub-list for method input_type + 1, // [1:1] is the sub-list for extension type_name + 1, // [1:1] is the sub-list for extension extendee + 0, // [0:1] is the sub-list for field type_name +} + +func init() { file_v1_changeset_proto_init() } +func file_v1_changeset_proto_init() { + if File_v1_changeset_proto != nil { + return + } + if !protoimpl.UnsafeEnabled { + file_v1_changeset_proto_msgTypes[0].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*KVPair); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + file_v1_changeset_proto_msgTypes[1].Exporter = func(v interface{}, i int) interface{} { + switch v := v.(*ChangeSet); i { + case 0: + return &v.state + case 1: + return &v.sizeCache + case 2: + return &v.unknownFields + default: + return nil + } + } + } + type x struct{} + out := protoimpl.TypeBuilder{ + File: protoimpl.DescBuilder{ + GoPackagePath: reflect.TypeOf(x{}).PkgPath(), + RawDescriptor: file_v1_changeset_proto_rawDesc, + NumEnums: 0, + NumMessages: 2, + NumExtensions: 0, + NumServices: 0, + }, + GoTypes: file_v1_changeset_proto_goTypes, + DependencyIndexes: file_v1_changeset_proto_depIdxs, + MessageInfos: file_v1_changeset_proto_msgTypes, + }.Build() + File_v1_changeset_proto = out.File + file_v1_changeset_proto_rawDesc = nil + file_v1_changeset_proto_goTypes = nil + file_v1_changeset_proto_depIdxs = nil +} diff --git a/nodedb.go b/nodedb.go index 54933c2e6..148cb96c9 100644 --- a/nodedb.go +++ b/nodedb.go @@ -840,7 +840,7 @@ func (ndb *nodeDB) traverseStateChanges(startVersion, endVersion int64, fn func( var changeSet ChangeSet receiveKVPair := func(pair *KVPair) error { - changeSet.Pairs = append(changeSet.Pairs, *pair) + changeSet.Pairs = append(changeSet.Pairs, pair) return nil } diff --git a/proto/buf.lock b/proto/buf.lock new file mode 100644 index 000000000..c91b5810c --- /dev/null +++ b/proto/buf.lock @@ -0,0 +1,2 @@ +# Generated by buf. DO NOT EDIT. +version: v1 diff --git a/proto/buf.md b/proto/buf.md new file mode 100644 index 000000000..74cd1259a --- /dev/null +++ b/proto/buf.md @@ -0,0 +1,3 @@ +## IAVL + +Defines messages used by IAVL library, currently only `ChangeSet` and `KVPair`. diff --git a/proto/buf.yaml b/proto/buf.yaml new file mode 100644 index 000000000..f2008dbda --- /dev/null +++ b/proto/buf.yaml @@ -0,0 +1,10 @@ +version: v1 +name: buf.build/cosmos/iavl +breaking: + use: + - FILE +lint: + use: + - DEFAULT + - COMMENTS + - FILE_LOWER_SNAKE_CASE diff --git a/proto/v1/changeset.proto b/proto/v1/changeset.proto new file mode 100644 index 000000000..3bc38dfa4 --- /dev/null +++ b/proto/v1/changeset.proto @@ -0,0 +1,12 @@ +syntax = "proto3"; +package iavl; + +message KVPair { + bool delete = 1; + bytes key = 2; + bytes value = 3; +} + +message ChangeSet { + repeated KVPair pairs = 1; +}