From d0b5367e8e5abcb358bbcf0bf670154a92353e65 Mon Sep 17 00:00:00 2001 From: Rohit Nayak <57520317+rohit-nayak-ps@users.noreply.github.com> Date: Thu, 16 Mar 2023 17:41:19 +0100 Subject: [PATCH] Backport: VStreamer: improve representation of integers in json data types (2b43fd7dfb804b421f71705eedab7c043675b648) Signed-off-by: Brendan Dougherty --- go.mod | 4 +- go.sum | 4 +- go/mysql/binlog_event_json.go | 196 +++++++++++++----- go/mysql/binlog_event_json_test.go | 87 +++++++- .../vreplication/unsharded_init_data.sql | 2 +- .../vreplication/vreplication_test.go | 14 +- .../vreplication/vplayer_flaky_test.go | 21 +- 7 files changed, 249 insertions(+), 79 deletions(-) diff --git a/go.mod b/go.mod index 1baf6f63a59..248a4aac8a0 100644 --- a/go.mod +++ b/go.mod @@ -71,7 +71,6 @@ require ( github.com/spf13/cobra v1.4.0 github.com/spf13/pflag v1.0.5 github.com/spf13/viper v1.8.1 - github.com/spyzhov/ajson v0.4.2 github.com/stretchr/testify v1.7.1 github.com/tchap/go-patricia v2.2.6+incompatible github.com/tebeka/selenium v0.9.9 @@ -118,6 +117,7 @@ require ( require ( github.com/bndr/gotabulate v1.1.2 github.com/hashicorp/go-version v1.6.0 + github.com/spyzhov/ajson v0.8.0 golang.org/x/exp v0.0.0-20230811145659-89c5cff77bcb modernc.org/sqlite v1.20.3 ) @@ -202,3 +202,5 @@ require ( modernc.org/token v1.0.1 // indirect sigs.k8s.io/structured-merge-diff/v4 v4.0.3 // indirect ) + +replace github.com/spyzhov/ajson v0.8.0 => github.com/rohit-nayak-ps/ajson v0.7.2-0.20230316112806-97deb03d883c diff --git a/go.sum b/go.sum index 03fdd00c745..7c6fbf8bba2 100644 --- a/go.sum +++ b/go.sum @@ -650,6 +650,8 @@ github.com/rogpeppe/fastuuid v1.2.0/go.mod h1:jVj6XXZzXRy/MSR5jhDC/2q6DgLz+nrA6L github.com/rogpeppe/go-internal v1.3.0/go.mod h1:M8bDsm7K2OlrFYOpmOWEs/qY81heoFRclV5y23lUDJ4= github.com/rogpeppe/go-internal v1.6.1 h1:/FiVV8dS/e+YqF2JvO3yXRFbBLTIuSDkuC7aBOAvL+k= github.com/rogpeppe/go-internal v1.6.1/go.mod h1:xXDCJY+GAPziupqXw64V24skbSoqbTEfhy4qGm1nDQc= +github.com/rohit-nayak-ps/ajson v0.7.2-0.20230316112806-97deb03d883c h1:Y/4qcogoZA2WUtLWMk/yXfJSpaIG3mK3r9Lw4kaARL4= +github.com/rohit-nayak-ps/ajson v0.7.2-0.20230316112806-97deb03d883c/go.mod h1:63V+CGM6f1Bu/p4nLIN8885ojBdt88TbLoSFzyqMuVA= github.com/russross/blackfriday v1.5.2/go.mod h1:JO/DiYxRf+HjHt06OyowR9PTA263kcR/rfWxYHBV53g= github.com/russross/blackfriday/v2 v2.1.0 h1:JIOH55/0cWyOuilr9/qlrm0BSXldqnqwMsf35Ld67mk= github.com/russross/blackfriday/v2 v2.1.0/go.mod h1:+Rmxgy9KzJVeS9/2gXHxylqXiyQDYRxCVz55jmeOWTM= @@ -695,8 +697,6 @@ github.com/spf13/pflag v1.0.5/go.mod h1:McXfInJRrz4CZXVZOBLb0bTZqETkiAhM9Iw0y3An github.com/spf13/viper v1.3.2/go.mod h1:ZiWeW+zYFKm7srdB9IoDzzZXaJaI5eL9QjNiN/DMA2s= github.com/spf13/viper v1.8.1 h1:Kq1fyeebqsBfbjZj4EL7gj2IO0mMaiyjYUWcUsl2O44= github.com/spf13/viper v1.8.1/go.mod h1:o0Pch8wJ9BVSWGQMbra6iw0oQ5oktSIBaujf1rJH9Ns= -github.com/spyzhov/ajson v0.4.2 h1:JMByd/jZApPKDvNsmO90X2WWGbmT2ahDFp73QhZbg3s= -github.com/spyzhov/ajson v0.4.2/go.mod h1:63V+CGM6f1Bu/p4nLIN8885ojBdt88TbLoSFzyqMuVA= github.com/stretchr/objx v0.1.0/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.1.1/go.mod h1:HFkY916IF+rwdDfMAkV7OtwuqBVzrE8GR6GFx+wExME= github.com/stretchr/objx v0.2.0 h1:Hbg2NidpLE8veEBkEZTL3CvlkUIVzuU9jDplZO54c48= diff --git a/go/mysql/binlog_event_json.go b/go/mysql/binlog_event_json.go index 82b53311c0f..e055b8866ca 100644 --- a/go/mysql/binlog_event_json.go +++ b/go/mysql/binlog_event_json.go @@ -153,7 +153,7 @@ func (jh *BinlogJSON) register(typ jsonDataType, Plugin jsonPlugin) { func (jh *BinlogJSON) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { Plugin := jh.plugins[typ] if Plugin == nil { - return nil, fmt.Errorf("Plugin not found for type %d", typ) + return nil, fmt.Errorf("plugin not found for type %d", typ) } return Plugin.getNode(typ, data, pos) } @@ -316,59 +316,157 @@ type intPlugin struct { var _ jsonPlugin = (*intPlugin)(nil) -func (ih intPlugin) getVal(typ jsonDataType, data []byte, pos int) (value float64) { +func (ipl intPlugin) getVal(typ jsonDataType, data []byte, pos int) (value int64) { var val uint64 - var val2 float64 - size := ih.sizes[typ] + var val2 int64 + size := ipl.sizes[typ] for i := 0; i < size; i++ { val = val + uint64(data[pos+i])<<(8*i) } switch typ { case jsonInt16: - val2 = float64(int16(val)) - case jsonUint16: - val2 = float64(uint16(val)) + val2 = int64(int16(val)) case jsonInt32: - val2 = float64(int32(val)) - case jsonUint32: - val2 = float64(uint32(val)) + val2 = int64(int32(val)) case jsonInt64: - val2 = float64(int64(val)) - case jsonUint64: - val2 = float64(val) - case jsonDouble: - val2 = math.Float64frombits(val) + val2 = int64(val) } return val2 } -func (ih intPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { - val := ih.getVal(typ, data, pos) - node = ajson.NumericNode("", val) +func (ipl intPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { + val := ipl.getVal(typ, data, pos) + node = ajson.IntegerNode("", val) return node, nil } func newIntPlugin() *intPlugin { - ih := &intPlugin{ + ipl := &intPlugin{ info: &jsonPluginInfo{ name: "Int", - types: []jsonDataType{jsonInt64, jsonInt32, jsonInt16, jsonUint16, jsonUint32, jsonUint64, jsonDouble}, + types: []jsonDataType{jsonInt64, jsonInt32, jsonInt16}, + }, + sizes: make(map[jsonDataType]int), + } + ipl.sizes = map[jsonDataType]int{ + jsonInt64: 8, + jsonInt32: 4, + jsonInt16: 2, + } + for _, typ := range ipl.info.types { + binlogJSON.register(typ, ipl) + } + return ipl +} + +//endregion + +//region uint plugin + +func init() { + newUintPlugin() +} + +type uintPlugin struct { + info *jsonPluginInfo + sizes map[jsonDataType]int +} + +var _ jsonPlugin = (*uintPlugin)(nil) + +func (upl uintPlugin) getVal(typ jsonDataType, data []byte, pos int) (value uint64) { + var val uint64 + var val2 uint64 + size := upl.sizes[typ] + for i := 0; i < size; i++ { + val = val + uint64(data[pos+i])<<(8*i) + } + switch typ { + case jsonUint16: + val2 = uint64(uint16(val)) + case jsonUint32: + val2 = uint64(uint32(val)) + case jsonUint64: + val2 = val + } + return val2 +} + +func (upl uintPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { + val := upl.getVal(typ, data, pos) + node = ajson.UnsignedIntegerNode("", val) + return node, nil +} + +func newUintPlugin() *uintPlugin { + upl := &uintPlugin{ + info: &jsonPluginInfo{ + name: "Uint", + types: []jsonDataType{jsonUint16, jsonUint32, jsonUint64}, }, sizes: make(map[jsonDataType]int), } - ih.sizes = map[jsonDataType]int{ + upl.sizes = map[jsonDataType]int{ jsonUint64: 8, - jsonInt64: 8, jsonUint32: 4, - jsonInt32: 4, jsonUint16: 2, - jsonInt16: 2, + } + for _, typ := range upl.info.types { + binlogJSON.register(typ, upl) + } + return upl +} + +//endregion + +//region float plugin + +func init() { + newFloatPlugin() +} + +type floatPlugin struct { + info *jsonPluginInfo + sizes map[jsonDataType]int +} + +var _ jsonPlugin = (*floatPlugin)(nil) + +func (flp floatPlugin) getVal(typ jsonDataType, data []byte, pos int) (value float64) { + var val uint64 + var val2 float64 + size := flp.sizes[typ] + for i := 0; i < size; i++ { + val = val + uint64(data[pos+i])<<(8*i) + } + switch typ { + case jsonDouble: + val2 = math.Float64frombits(val) + } + return val2 +} + +func (flp floatPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { + val := flp.getVal(typ, data, pos) + node = ajson.NumericNode("", val) + return node, nil +} + +func newFloatPlugin() *floatPlugin { + fp := &floatPlugin{ + info: &jsonPluginInfo{ + name: "Float", + types: []jsonDataType{jsonDouble}, + }, + sizes: make(map[jsonDataType]int), + } + fp.sizes = map[jsonDataType]int{ jsonDouble: 8, } - for _, typ := range ih.info.types { - binlogJSON.register(typ, ih) + for _, typ := range fp.info.types { + binlogJSON.register(typ, fp) } - return ih + return fp } //endregion @@ -385,7 +483,7 @@ type literalPlugin struct { var _ jsonPlugin = (*literalPlugin)(nil) -func (lh literalPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { +func (lpl literalPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { val := jsonDataLiteral(data[pos]) switch val { case jsonNullLiteral: @@ -401,14 +499,14 @@ func (lh literalPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *a } func newLiteralPlugin() *literalPlugin { - lh := &literalPlugin{ + lpl := &literalPlugin{ info: &jsonPluginInfo{ name: "Literal", types: []jsonDataType{jsonLiteral}, }, } - binlogJSON.register(jsonLiteral, lh) - return lh + binlogJSON.register(jsonLiteral, lpl) + return lpl } //endregion @@ -427,7 +525,7 @@ var _ jsonPlugin = (*opaquePlugin)(nil) // other types are stored as catch-all opaque types: documentation on these is scarce. // we currently know about (and support) date/time/datetime/decimal. -func (oh opaquePlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { +func (opl opaquePlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { dataType := data[pos] start := 3 // account for length of stored value end := start + 8 // all currently supported opaque data types are 8 bytes in size @@ -484,14 +582,14 @@ func (oh opaquePlugin) getNode(typ jsonDataType, data []byte, pos int) (node *aj } func newOpaquePlugin() *opaquePlugin { - oh := &opaquePlugin{ + opl := &opaquePlugin{ info: &jsonPluginInfo{ name: "Opaque", types: []jsonDataType{jsonOpaque}, }, } - binlogJSON.register(jsonOpaque, oh) - return oh + binlogJSON.register(jsonOpaque, opl) + return opl } //endregion @@ -508,7 +606,7 @@ type stringPlugin struct { var _ jsonPlugin = (*stringPlugin)(nil) -func (sh stringPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { +func (spl stringPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { size, pos := readVariableLength(data, pos) node = ajson.StringNode("", string(data[pos:pos+size])) @@ -516,14 +614,14 @@ func (sh stringPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *aj } func newStringPlugin() *stringPlugin { - sh := &stringPlugin{ + spl := &stringPlugin{ info: &jsonPluginInfo{ name: "String", types: []jsonDataType{jsonString}, }, } - binlogJSON.register(jsonString, sh) - return sh + binlogJSON.register(jsonString, spl) + return spl } //endregion @@ -542,7 +640,7 @@ var _ jsonPlugin = (*arrayPlugin)(nil) // arrays are stored thus: // | type_identifier(one of [2,3]) | elem count | obj size | list of offsets+lengths of values | actual values | -func (ah arrayPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { +func (apl arrayPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { jlog("JSON Array %s, len %d", jsonDataTypeToString(uint(typ)), len(data)) var nodes []*ajson.Node var elem *ajson.Node @@ -565,15 +663,15 @@ func (ah arrayPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajs } func newArrayPlugin() *arrayPlugin { - ah := &arrayPlugin{ + apl := &arrayPlugin{ info: &jsonPluginInfo{ name: "Array", types: []jsonDataType{jsonSmallArray, jsonLargeArray}, }, } - binlogJSON.register(jsonSmallArray, ah) - binlogJSON.register(jsonLargeArray, ah) - return ah + binlogJSON.register(jsonSmallArray, apl) + binlogJSON.register(jsonLargeArray, apl) + return apl } //endregion @@ -592,7 +690,7 @@ var _ jsonPlugin = (*objectPlugin)(nil) // objects are stored thus: // | type_identifier(0/1) | elem count | obj size | list of offsets+lengths of keys | list of offsets+lengths of values | actual keys | actual values | -func (oh objectPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { +func (opl objectPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *ajson.Node, err error) { jlog("JSON Type is %s, len %d", jsonDataTypeToString(uint(typ)), len(data)) // "large" decides number of bytes used to specify element count and total object size: 4 bytes for large, 2 for small @@ -640,15 +738,15 @@ func (oh objectPlugin) getNode(typ jsonDataType, data []byte, pos int) (node *aj } func newObjectPlugin() *objectPlugin { - oh := &objectPlugin{ + opl := &objectPlugin{ info: &jsonPluginInfo{ name: "Object", types: []jsonDataType{jsonSmallObject, jsonLargeObject}, }, } - binlogJSON.register(jsonSmallObject, oh) - binlogJSON.register(jsonLargeObject, oh) - return oh + binlogJSON.register(jsonSmallObject, opl) + binlogJSON.register(jsonLargeObject, opl) + return opl } //endregion diff --git a/go/mysql/binlog_event_json_test.go b/go/mysql/binlog_event_json_test.go index 711965386ed..5cc61440084 100644 --- a/go/mysql/binlog_event_json_test.go +++ b/go/mysql/binlog_event_json_test.go @@ -18,50 +18,65 @@ package mysql import ( "encoding/json" + "fmt" "testing" "github.com/stretchr/testify/require" ) func TestJSONTypes(t *testing.T) { + // most of these test cases have been taken from open source java/python adapters + // like https://github.com/shyiko/mysql-binlog-connector-java/pull/119/files testcases := []struct { + name string data []byte expected string isMap bool }{{ + name: "null", data: []byte{}, expected: `null`, }, { + name: "map, string value", data: []byte{0, 1, 0, 14, 0, 11, 0, 1, 0, 12, 12, 0, 97, 1, 98}, expected: `{"a":"b"}`, }, { + name: "map, int value", data: []byte{0, 1, 0, 12, 0, 11, 0, 1, 0, 5, 2, 0, 97}, expected: `{"a":2}`, }, { + name: "map, object value", data: []byte{0, 1, 0, 29, 0, 11, 0, 4, 0, 0, 15, 0, 97, 115, 100, 102, 1, 0, 14, 0, 11, 0, 3, 0, 5, 123, 0, 102, 111, 111}, expected: `{"asdf":{"foo":123}}`, }, { + name: "list of ints", data: []byte{2, 2, 0, 10, 0, 5, 1, 0, 5, 2, 0}, expected: `[1,2]`, }, { + name: "list of maps", data: []byte{0, 4, 0, 60, 0, 32, 0, 1, 0, 33, 0, 1, 0, 34, 0, 2, 0, 36, 0, 2, 0, 12, 38, 0, 12, 40, 0, 12, 42, 0, 2, 46, 0, 97, 99, 97, 98, 98, 99, 1, 98, 1, 100, 3, 97, 98, 99, 2, 0, 14, 0, 12, 10, 0, 12, 12, 0, 1, 120, 1, 121}, expected: `{"a":"b","c":"d","ab":"abc","bc":["x","y"]}`, isMap: true, }, { + name: "list with one string", data: []byte{2, 1, 0, 37, 0, 12, 8, 0, 0, 4, 104, 101, 114, 101}, expected: `["here"]`, }, { + name: "list varied", data: []byte{2, 3, 0, 37, 0, 12, 13, 0, 2, 18, 0, 12, 33, 0, 4, 104, 101, 114, 101, 2, 0, 15, 0, 12, 10, 0, 12, 12, 0, 1, 73, 2, 97, 109, 3, 33, 33, 33}, expected: `["here",["I","am"],"!!!"]`, }, { + name: "string", data: []byte{12, 13, 115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103}, expected: `"scalar string"`, }, { + name: "map, long string value", data: []byte{0, 1, 0, 149, 0, 11, 0, 6, 0, 12, 17, 0, 115, 99, 111, 112, 101, 115, 130, 1, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 66, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 65, 69, 65, 65, 65, 65, 65, 65, 69, 65, 65, 65, 65, 65, 65, 56, 65, 65, 65, 66, 103, 65, 65, 65, 65, 65, 65, 66, 65, 65, 65, 65, 67, 65, 65, 65, 65, 65, 65, 65, 65, 65, 84, 216, 142, 184}, expected: `{"scopes":"AAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABAAAAAAAAAAAAAAAAAEAAAAAAEAAAAAA8AAABgAAAAAABAAAACAAAAAAAAA"}`, }, { // repeat the same string 10 times, to test the case where length of string // requires 2 bytes to store + name: "long string", data: []byte{12, 130, 1, 115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103, 115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103, @@ -75,89 +90,139 @@ func TestJSONTypes(t *testing.T) { 115, 99, 97, 108, 97, 114, 32, 115, 116, 114, 105, 110, 103}, expected: `"scalar stringscalar stringscalar stringscalar stringscalar stringscalar stringscalar stringscalar stringscalar stringscalar string"`, }, { + name: "bool true", data: []byte{4, 1}, expected: `true`, }, { + name: "bool false", data: []byte{4, 2}, expected: `false`, }, { + name: "bool null", data: []byte{4, 0}, expected: `null`, }, { + name: "uint16 max", + data: []byte{6, 255, 255}, + expected: `65535`, + }, { + name: "uint32 max", + data: []byte{8, 255, 255, 255, 255}, + expected: `4294967295`, + }, { + name: "uint64 max", + data: []byte{10, 255, 255, 255, 255, 255, 255, 255, 255}, + expected: `18446744073709551615`, + }, { + name: "int16 -1", data: []byte{5, 255, 255}, expected: `-1`, }, { - data: []byte{6, 1, 0}, - expected: `1`, + name: "int32 -1", + data: []byte{7, 255, 255, 255, 255}, + expected: `-1`, + }, { + name: "int64 -1", + data: []byte{9, 255, 255, 255, 255, 255, 255, 255, 255}, + expected: `-1`, }, { + name: "int16 max", data: []byte{5, 255, 127}, expected: `32767`, }, { + name: "int32 max", + data: []byte{7, 255, 255, 255, 127}, + expected: `2147483647`, + }, { + name: "int64 max", + data: []byte{9, 255, 255, 255, 255, 255, 255, 255, 127}, + expected: `9223372036854775807`, + }, { + name: "uint16/1", + data: []byte{6, 1, 0}, + expected: `1`, + }, { + name: "int32/32768", data: []byte{7, 0, 128, 0, 0}, expected: `32768`, }, { + name: "int16/neg", data: []byte{5, 0, 128}, expected: `-32768`, }, { + name: "int32/neg", data: []byte{7, 255, 127, 255, 255}, expected: `-32769`, }, { - data: []byte{7, 255, 255, 255, 127}, - expected: `2.147483647e+09`, - }, { + name: "uint32", data: []byte{8, 0, 128, 0, 0}, expected: `32768`, }, { + name: "int64", data: []byte{9, 0, 0, 0, 128, 0, 0, 0, 0}, - expected: `2.147483648e+09`, + expected: `2147483648`, }, { + name: "int32/neg", data: []byte{7, 0, 0, 0, 128}, - expected: `-2.147483648e+09`, + expected: `-2147483648`, }, { + name: "int64/neg", data: []byte{9, 255, 255, 255, 127, 255, 255, 255, 255}, - expected: `-2.147483649e+09`, + expected: `-2147483649`, }, { + name: "uint64", data: []byte{10, 255, 255, 255, 255, 255, 255, 255, 255}, - expected: `1.8446744073709552e+19`, + expected: `18446744073709551615`, }, { + name: "int64/neg", data: []byte{9, 0, 0, 0, 0, 0, 0, 0, 128}, - expected: `-9.223372036854776e+18`, + expected: `-9223372036854775808`, }, { + name: "double", data: []byte{11, 110, 134, 27, 240, 249, 33, 9, 64}, expected: `3.14159`, }, { + name: "empty map", data: []byte{0, 0, 0, 4, 0}, expected: `{}`, }, { + name: "empty list", data: []byte{2, 0, 0, 4, 0}, expected: `[]`, }, { // opaque, datetime + name: "datetime", data: []byte{15, 12, 8, 0, 0, 0, 25, 118, 31, 149, 25}, expected: `"2015-01-15 23:24:25.000000"`, }, { // opaque, time + name: "time", data: []byte{15, 11, 8, 0, 0, 0, 25, 118, 1, 0, 0}, expected: `"23:24:25.000000"`, }, { // opaque, time + name: "time2", data: []byte{15, 11, 8, 192, 212, 1, 25, 118, 1, 0, 0}, expected: `"23:24:25.120000"`, }, { // opaque, date + name: "date", data: []byte{15, 10, 8, 0, 0, 0, 0, 0, 30, 149, 25}, expected: `"2015-01-15"`, }, { // opaque, decimal + name: "decimal", data: []byte{15, 246, 8, 13, 4, 135, 91, 205, 21, 4, 210}, expected: `1.234567891234e+08`, }, { // opaque, bit field. Not yet implemented. + name: "bitfield: unimplemented", data: []byte{15, 16, 2, 202, 254}, expected: `opaque type 16 is not supported yet, data [2 202 254]`, }} for _, tc := range testcases { - t.Run(tc.expected, func(t *testing.T) { + name := fmt.Sprintf("%s (%s)", tc.name, tc.expected) + t.Run(name, func(t *testing.T) { val, err := getJSONValue(tc.data) if err != nil { require.Equal(t, tc.expected, err.Error()) diff --git a/go/test/endtoend/vreplication/unsharded_init_data.sql b/go/test/endtoend/vreplication/unsharded_init_data.sql index b12aaa9bf79..019945609db 100644 --- a/go/test/endtoend/vreplication/unsharded_init_data.sql +++ b/go/test/endtoend/vreplication/unsharded_init_data.sql @@ -1,6 +1,6 @@ insert into customer(cid, name, typ, sport, meta) values(1, 'Jøhn "❤️" Rizzolo',1,'football,baseball','{}'); insert into customer(cid, name, typ, sport, meta) values(2, 'Paül','soho','cricket',convert(x'7b7d' using utf8mb4)); -insert into customer(cid, name, typ, sport) values(3, 'ringo','enterprise',''); +insert into customer(cid, name, typ, sport, meta) values(3, 'ringo','enterprise','',null); insert into merchant(mname, category) values('Monoprice', 'eléctronics'); insert into merchant(mname, category) values('newegg', 'elec†ronics'); insert into product(pid, description) values(1, 'keyböard ⌨️'); diff --git a/go/test/endtoend/vreplication/vreplication_test.go b/go/test/endtoend/vreplication/vreplication_test.go index 6e8d347ade3..d0e95789f82 100644 --- a/go/test/endtoend/vreplication/vreplication_test.go +++ b/go/test/endtoend/vreplication/vreplication_test.go @@ -606,9 +606,11 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl query := "select cid from customer" require.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", query, query)) - insertQuery1 := "insert into customer(cid, name) values(1001, 'tempCustomer1')" - matchInsertQuery1 := "insert into customer(cid, `name`) values (:vtg1, :vtg2)" - require.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", insertQuery1, matchInsertQuery1)) + insertQuery1 := "insert into customer(cid, name, meta) values(1001, 'tempCustomer1', '{\"a\": 1629849600, \"b\": 930701976723823}')" + + matchInsertQuery0 := "insert into customer(cid, `name`) values (:vtg1, :vtg2)" + matchInsertQuery1 := "insert into customer(cid, `name`, meta) values (:vtg1, :vtg2, :vtg3)" + validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", insertQuery1, matchInsertQuery1) // confirm that the backticking of table names in the routing rules works tbls := []string{"Lead", "Lead-1"} @@ -671,12 +673,12 @@ func shardCustomer(t *testing.T, testReverse bool, cells []*Cell, sourceCellOrAl require.Contains(t, output, "'customer.bmd5'") insertQuery1 = "insert into customer(cid, name) values(1002, 'tempCustomer5')" - require.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", insertQuery1, matchInsertQuery1)) + require.True(t, validateThatQueryExecutesOnTablet(t, vtgateConn, productTab, "product", insertQuery1, matchInsertQuery0)) // both inserts go into 80-, this tests the edge-case where a stream (-80) has no relevant new events after the previous switch insertQuery1 = "insert into customer(cid, name) values(1003, 'tempCustomer6')" - require.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, customerTab1, "customer", insertQuery1, matchInsertQuery1)) + require.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, customerTab1, "customer", insertQuery1, matchInsertQuery0)) insertQuery1 = "insert into customer(cid, name) values(1004, 'tempCustomer7')" - require.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, customerTab2, "customer", insertQuery1, matchInsertQuery1)) + require.False(t, validateThatQueryExecutesOnTablet(t, vtgateConn, customerTab2, "customer", insertQuery1, matchInsertQuery0)) //Go forward again switchReads(t, allCellNames, ksWorkflow) diff --git a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go index 5dbe260d68b..ce9b2ce06bd 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vplayer_flaky_test.go @@ -27,13 +27,16 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql" + "github.com/spyzhov/ajson" "github.com/stretchr/testify/require" - "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/vt/log" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/binlog/binlogplayer" - "vitess.io/vitess/go/vt/log" + binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" ) @@ -1523,21 +1526,21 @@ func TestPlayerTypes(t *testing.T) { }} if enableJSONColumnTesting { testcases = append(testcases, testcase{ - input: "insert into vitess_json(val1,val2,val3,val4,val5) values (null,'{}','123','{\"a\":[42,100]}', '{\"foo\":\"bar\"}')", + input: "insert into vitess_json(val1,val2,val3,val4,val5) values (null,'{}','1629849600','{\"a\":[42,-1,3.1415,-128,127,-9223372036854775808,9223372036854775807,18446744073709551615]}', '{\"foo\":\"bar\"}')", output: "insert into vitess_json(id,val1,val2,val3,val4,val5) values (1," + - "convert(null using utf8mb4)," + "convert('{}' using utf8mb4)," + "convert('123' using utf8mb4)," + - "convert('{\\\"a\\\":[42,100]}' using utf8mb4)," + "convert('{\\\"foo\\\":\\\"bar\\\"}' using utf8mb4))", + "convert(null using utf8mb4)," + "convert('{}' using utf8mb4)," + "convert('1629849600' using utf8mb4)," + + "convert('{\\\"a\\\":[42,-1,3.1415,-128,127,-9223372036854775808,9223372036854775807,18446744073709551615]}' using utf8mb4)," + "convert('{\\\"foo\\\":\\\"bar\\\"}' using utf8mb4))", table: "vitess_json", data: [][]string{ - {"1", "", "{}", "123", `{"a": [42, 100]}`, `{"foo": "bar"}`}, + {"1", "", "{}", "1629849600", `{"a": [42, -1, 3.1415, -128, 127, -9223372036854775808, 9223372036854775807, 18446744073709551615]}`, `{"foo": "bar"}`}, }, }) testcases = append(testcases, testcase{ - input: "update vitess_json set val4 = '{\"a\": [98, 123]}', val5 = convert(x'7b7d' using utf8mb4)", - output: "update vitess_json set val1=convert(null using utf8mb4), val2=convert('{}' using utf8mb4), val3=convert('123' using utf8mb4), val4=convert('{\\\"a\\\":[98,123]}' using utf8mb4), val5=convert('{}' using utf8mb4) where id=1", + input: "update vitess_json set val4 = '{\"a\": [-9223372036854775808, -2147483648]}', val5 = convert(x'7b7d' using utf8mb4)", + output: "update vitess_json set val1=convert(null using utf8mb4), val2=convert('{}' using utf8mb4), val3=convert('1629849600' using utf8mb4), val4=convert('{\\\"a\\\":[-9223372036854775808,-2147483648]}' using utf8mb4), val5=convert('{}' using utf8mb4) where id=1", table: "vitess_json", data: [][]string{ - {"1", "", "{}", "123", `{"a": [98, 123]}`, `{}`}, + {"1", "", "{}", "1629849600", `{"a": [-9223372036854775808, -2147483648]}`, `{}`}, }, }) }