From e40a738c726e74c50fad3b6926a6ad67dde52d27 Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Sat, 15 Jul 2023 16:11:39 +0200 Subject: [PATCH 01/13] collations: Fix appending to initial slice The logic here was wrong and we were always starting from the start of slice if an existing buffer was passed in to append to. Instead we need to actually append from the end of the slice. Signed-off-by: Dirkjan Bussink --- go/mysql/collations/charset/convert.go | 3 +++ 1 file changed, 3 insertions(+) diff --git a/go/mysql/collations/charset/convert.go b/go/mysql/collations/charset/convert.go index 6054ae33559..bc51e9b8377 100644 --- a/go/mysql/collations/charset/convert.go +++ b/go/mysql/collations/charset/convert.go @@ -33,6 +33,7 @@ func convertFastFromUTF8(dst []byte, dstCharset Charset, src []byte) ([]byte, er if dst == nil { dst = make([]byte, len(src)*3) } else { + nDst = len(dst) dst = dst[:cap(dst)] } @@ -65,6 +66,7 @@ func convertSlow(dst []byte, dstCharset Charset, src []byte, srcCharset Charset) if dst == nil { dst = make([]byte, len(src)*3) } else { + nDst = len(dst) dst = dst[:cap(dst)] } @@ -180,6 +182,7 @@ func Collapse(dst []byte, src []rune, dstCharset Charset) []byte { if dst == nil { dst = make([]byte, len(src)*dstCharset.MaxWidth()) } else { + nDst = len(dst) dst = dst[:cap(dst)] } for _, c := range src { From 66fac65a7072744bf586e75cb7794de4e2022ae0 Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Sat, 15 Jul 2023 16:15:50 +0200 Subject: [PATCH 02/13] Fix encoding issues for evaluated expressions As described in #13493 we have various bugs where we transcode inputs incorrectly and where we lack collation information in the results returned. The changes here fix a number of things: - Ensure that we always return collation information for querypb.Field instances. This also updates a lot of the static mocked inputs to ensure that they match what real queries would return. Many of the changes associated with this part of the fixes is ensuring the mocks are expectations are updated properly. - Introduce an IntroduceExpr in the evalengine. One of the issues identified is that we didn't correctly handle hex escaped values and we would never see them being unescaped if a charset introducer was used. The cause of this is that we directly would change the collation information on the expression. This is wrong in the case of a HexNum or HexVal, since we loose information we need to transcode those. We could work around this for the AST evaluator, but the compiler couldn't be fixed as the real underlying value was not available with the actual hex encoded value. By introducing a new expression, we can also now properly compile this. - No flags information was sent along with evaluated results. This is similar to the collation information, although usually not as important for clients. We opted here to fix this nonetheless and match MySQL as closely as possible. - Introduce a `collations.SystemCollation` to use consistently for all internal table / column names information. It was using mostly the constant for utf8mb3 or no information at all which would be problematic as it means we woulnd't know what encoding it was. - Convert outputs to the connection collation / charset. This was entirely missing from Vitess but is behavior that MySQL has. If you read from say a `latin1` encoded column with a connection charset of `utf8mb4`, the value is converted before it's sent over the wire to `utf8mb4`. This fix also applies for the `evalengine` tests where we can now test also non UTF-8 compatible encodings, fixing a bunch of todos. This uncovered some other small bugs in the `evalengine` where we were converting things wrong. - The rowstreamer would return collation information from the result set, but that is now wrong because of the above conversion fixes. This means we need to use the actual fields information which we already have available to see the proper collation. We can copy all the attributes for the PK column. - To be explicit, it includes a rename from `collations.CollationUtf8ID` to `collations.CollationUtf8mb3ID` to be explicit about this collation name and match how MySQL is also changing this to be explicit. Signed-off-by: Dirkjan Bussink --- go/cmd/vtgateclienttest/services/echo.go | 7 +- go/mysql/collations/env.go | 12 +- go/mysql/collations/local.go | 12 + go/mysql/query.go | 19 +- go/mysql/query_test.go | 87 +++--- go/mysql/schema.go | 6 +- go/mysql/server_flaky_test.go | 48 +-- go/vt/binlog/binlog_streamer_rbr_test.go | 23 +- go/vt/vtctl/vtctl.go | 8 +- go/vt/vtexplain/vtexplain_vttablet.go | 75 +++-- go/vt/vtexplain/vtexplain_vttablet_test.go | 6 +- go/vt/vtgate/endtoend/vstream_test.go | 9 +- go/vt/vtgate/engine/fake_vcursor_test.go | 2 +- go/vt/vtgate/engine/filter.go | 2 +- go/vt/vtgate/engine/insert.go | 2 +- go/vt/vtgate/engine/limit.go | 8 +- go/vt/vtgate/engine/lock.go | 2 +- go/vt/vtgate/engine/memory_sort.go | 7 +- go/vt/vtgate/engine/online_ddl.go | 6 +- go/vt/vtgate/engine/projection.go | 33 +- go/vt/vtgate/engine/projection_test.go | 49 +++ go/vt/vtgate/engine/route_test.go | 16 +- go/vt/vtgate/engine/routing.go | 10 +- go/vt/vtgate/engine/set.go | 38 +-- go/vt/vtgate/engine/update.go | 2 +- go/vt/vtgate/engine/vindex_func.go | 5 +- go/vt/vtgate/engine/vindex_lookup.go | 2 +- go/vt/vtgate/evalengine/api_literal.go | 6 +- go/vt/vtgate/evalengine/compiler_asm.go | 14 + go/vt/vtgate/evalengine/compiler_test.go | 97 ++++++ go/vt/vtgate/evalengine/eval.go | 4 + go/vt/vtgate/evalengine/eval_bytes.go | 2 +- go/vt/vtgate/evalengine/eval_result.go | 24 +- go/vt/vtgate/evalengine/expr_collate.go | 44 ++- go/vt/vtgate/evalengine/expr_env.go | 6 +- go/vt/vtgate/evalengine/fn_string.go | 34 ++- go/vt/vtgate/evalengine/format.go | 7 + .../evalengine/integration/comparison_test.go | 2 +- .../evalengine/integration/fuzz_test.go | 4 +- go/vt/vtgate/evalengine/mysql_test.go | 6 +- go/vt/vtgate/evalengine/testcases/inputs.go | 4 +- go/vt/vtgate/evalengine/translate.go | 22 +- go/vt/vtgate/evalengine/translate_card.go | 2 + go/vt/vtgate/evalengine/translate_convert.go | 2 +- go/vt/vtgate/evalengine/translate_test.go | 2 +- go/vt/vtgate/executor.go | 8 +- go/vt/vtgate/executor_select_test.go | 287 +++++++++--------- go/vt/vtgate/executor_test.go | 34 ++- go/vt/vtgate/legacy_scatter_conn_test.go | 4 +- go/vt/vtgate/plan_execute.go | 2 +- .../planbuilder/expression_converter.go | 3 +- go/vt/vtgate/planbuilder/from.go | 2 +- go/vt/vtgate/planbuilder/operators/route.go | 3 +- .../planbuilder/operators/sharded_routing.go | 2 +- go/vt/vtgate/planbuilder/plan_test.go | 2 +- .../planbuilder/predicate_rewrite_test.go | 4 +- go/vt/vtgate/planbuilder/project.go | 7 +- go/vt/vtgate/planbuilder/show.go | 10 +- go/vt/vtgate/planbuilder/vindex_func.go | 13 +- go/vt/vtgate/semantics/early_rewriter.go | 3 +- go/vt/vtgate/simplifier/simplifier_test.go | 3 +- go/vt/vtgate/tabletgateway_flaky_test.go | 17 +- go/vt/vttablet/sandboxconn/sandboxconn.go | 15 +- go/vt/vttablet/tabletserver/query_executor.go | 4 +- .../tabletserver/schema/historian_test.go | 21 +- .../tabletserver/vstreamer/planbuilder.go | 20 +- .../vstreamer/planbuilder_test.go | 180 +++++++---- .../tabletserver/vstreamer/rowstreamer.go | 15 +- .../vstreamer/rowstreamer_test.go | 34 +-- .../vstreamer/uvstreamer_flaky_test.go | 11 +- .../tabletserver/vstreamer/vstreamer.go | 6 +- .../vstreamer/vstreamer_flaky_test.go | 10 +- 72 files changed, 975 insertions(+), 523 deletions(-) diff --git a/go/cmd/vtgateclienttest/services/echo.go b/go/cmd/vtgateclienttest/services/echo.go index eec08305674..5a4d78aeb3e 100644 --- a/go/cmd/vtgateclienttest/services/echo.go +++ b/go/cmd/vtgateclienttest/services/echo.go @@ -24,6 +24,7 @@ import ( "sort" "strings" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/callerid" "vitess.io/vitess/go/vt/vtgate/vtgateservice" @@ -77,13 +78,13 @@ func echoQueryResult(vals map[string]any) *sqltypes.Result { // The first two returned fields are always a field with a MySQL NULL value, // and another field with a zero-length string. // Client tests can use this to check that they correctly distinguish the two. - qr.Fields = append(qr.Fields, &querypb.Field{Name: "null", Type: sqltypes.VarBinary}) + qr.Fields = append(qr.Fields, &querypb.Field{Name: "null", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}) row = append(row, sqltypes.NULL) - qr.Fields = append(qr.Fields, &querypb.Field{Name: "emptyString", Type: sqltypes.VarBinary}) + qr.Fields = append(qr.Fields, &querypb.Field{Name: "emptyString", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}) row = append(row, sqltypes.NewVarBinary("")) for k, v := range vals { - qr.Fields = append(qr.Fields, &querypb.Field{Name: k, Type: sqltypes.VarBinary}) + qr.Fields = append(qr.Fields, &querypb.Field{Name: k, Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}) val := reflect.ValueOf(v) if val.Kind() == reflect.Map { diff --git a/go/mysql/collations/env.go b/go/mysql/collations/env.go index 0c063e140d5..9144d196665 100644 --- a/go/mysql/collations/env.go +++ b/go/mysql/collations/env.go @@ -194,7 +194,7 @@ func makeEnv(version collver) *Environment { // A few interesting character set values. // See http://dev.mysql.com/doc/internals/en/character-set.html#packet-Protocol::CharacterSet const ( - CollationUtf8ID = 33 + CollationUtf8mb3ID = 33 CollationUtf8mb4ID = 255 CollationBinaryID = 63 CollationUtf8mb4BinID = 46 @@ -204,6 +204,16 @@ const ( // Binary is the default Binary collation var Binary = ID(CollationBinaryID).Get() +// SystemCollation is the default collation for the system tables +// such as the information schema. This is still utf8mb3 to match +// MySQLs behavior. This means that you can't use utf8mb4 in table +// names, column names, without running into significant issues. +var SystemCollation = TypedCollation{ + Collation: CollationUtf8mb3ID, + Coercibility: CoerceCoercible, + Repertoire: RepertoireUnicode, +} + // CharsetAlias returns the internal charset name for the given charset. // For now, this only maps `utf8` to `utf8mb3`; in future versions of MySQL, // this mapping will change, so it's important to use this helper so that diff --git a/go/mysql/collations/local.go b/go/mysql/collations/local.go index c0d3c10da09..15adc1a9e05 100644 --- a/go/mysql/collations/local.go +++ b/go/mysql/collations/local.go @@ -22,6 +22,7 @@ import ( "sync" "vitess.io/vitess/go/internal/flag" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/servenv" ) @@ -46,3 +47,14 @@ func Local() *Environment { func Default() ID { return ID(Local().DefaultConnectionCharset()) } + +func DefaultCollationForType(t sqltypes.Type) ID { + switch t { + case sqltypes.VarChar, sqltypes.Char, sqltypes.Text: + return Default() + case sqltypes.TypeJSON: + return CollationUtf8mb4ID + default: + return CollationBinaryID + } +} diff --git a/go/mysql/query.go b/go/mysql/query.go index f8adb91f60f..8a4291bc85f 100644 --- a/go/mysql/query.go +++ b/go/mysql/query.go @@ -22,6 +22,7 @@ import ( "strconv" "strings" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/vterrors" @@ -1073,7 +1074,9 @@ func (c *Conn) writePrepare(fld []*querypb.Field, prepare *PrepareData) error { if err := c.writeColumnDefinition(&querypb.Field{ Name: "?", Type: sqltypes.VarBinary, - Charset: 63}); err != nil { + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), + }); err != nil { return err } } @@ -1517,3 +1520,17 @@ func val2MySQLLen(v sqltypes.Value) (int, error) { } return length, nil } + +func FlagsForColumn(t sqltypes.Type, col collations.ID) uint32 { + var fl uint32 + if sqltypes.IsNumber(t) { + fl |= uint32(querypb.MySqlFlag_NUM_FLAG) + } + if sqltypes.IsUnsigned(t) { + fl |= uint32(querypb.MySqlFlag_UNSIGNED_FLAG) + } + if sqltypes.IsQuoted(t) && col == collations.CollationBinaryID { + fl |= uint32(querypb.MySqlFlag_BINARY_FLAG) + } + return fl +} diff --git a/go/mysql/query_test.go b/go/mysql/query_test.go index 5eceadbbdcb..bf902b5165f 100644 --- a/go/mysql/query_test.go +++ b/go/mysql/query_test.go @@ -24,6 +24,8 @@ import ( "google.golang.org/protobuf/proto" + "vitess.io/vitess/go/mysql/collations" + "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" @@ -48,8 +50,10 @@ func MockPrepareData(t *testing.T) (*PrepareData, *sqltypes.Result) { result := &sqltypes.Result{ Fields: []*querypb.Field{ { - Name: "id", - Type: querypb.Type_INT32, + Name: "id", + Type: querypb.Type_INT32, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }, Rows: [][]sqltypes.Value{ @@ -399,12 +403,15 @@ func TestQueries(t *testing.T) { checkQuery(t, "type and name", sConn, cConn, &sqltypes.Result{ Fields: []*querypb.Field{ { - Name: "id", - Type: querypb.Type_INT32, + Name: "id", + Type: querypb.Type_INT32, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, { - Name: "name", - Type: querypb.Type_VARCHAR, + Name: "name", + Type: querypb.Type_VARCHAR, + Charset: uint32(collations.Default()), }, }, Rows: [][]sqltypes.Value{ @@ -424,36 +431,36 @@ func TestQueries(t *testing.T) { // One row has all NULL values. checkQuery(t, "all types", sConn, cConn, &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "Type_INT8 ", Type: querypb.Type_INT8}, - {Name: "Type_UINT8 ", Type: querypb.Type_UINT8}, - {Name: "Type_INT16 ", Type: querypb.Type_INT16}, - {Name: "Type_UINT16 ", Type: querypb.Type_UINT16}, - {Name: "Type_INT24 ", Type: querypb.Type_INT24}, - {Name: "Type_UINT24 ", Type: querypb.Type_UINT24}, - {Name: "Type_INT32 ", Type: querypb.Type_INT32}, - {Name: "Type_UINT32 ", Type: querypb.Type_UINT32}, - {Name: "Type_INT64 ", Type: querypb.Type_INT64}, - {Name: "Type_UINT64 ", Type: querypb.Type_UINT64}, - {Name: "Type_FLOAT32 ", Type: querypb.Type_FLOAT32}, - {Name: "Type_FLOAT64 ", Type: querypb.Type_FLOAT64}, - {Name: "Type_TIMESTAMP", Type: querypb.Type_TIMESTAMP}, - {Name: "Type_DATE ", Type: querypb.Type_DATE}, - {Name: "Type_TIME ", Type: querypb.Type_TIME}, - {Name: "Type_DATETIME ", Type: querypb.Type_DATETIME}, - {Name: "Type_YEAR ", Type: querypb.Type_YEAR}, - {Name: "Type_DECIMAL ", Type: querypb.Type_DECIMAL}, - {Name: "Type_TEXT ", Type: querypb.Type_TEXT}, - {Name: "Type_BLOB ", Type: querypb.Type_BLOB}, - {Name: "Type_VARCHAR ", Type: querypb.Type_VARCHAR}, - {Name: "Type_VARBINARY", Type: querypb.Type_VARBINARY}, - {Name: "Type_CHAR ", Type: querypb.Type_CHAR}, - {Name: "Type_BINARY ", Type: querypb.Type_BINARY}, - {Name: "Type_BIT ", Type: querypb.Type_BIT}, - {Name: "Type_ENUM ", Type: querypb.Type_ENUM}, - {Name: "Type_SET ", Type: querypb.Type_SET}, + {Name: "Type_INT8 ", Type: querypb.Type_INT8, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "Type_UINT8 ", Type: querypb.Type_UINT8, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG)}, + {Name: "Type_INT16 ", Type: querypb.Type_INT16, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "Type_UINT16 ", Type: querypb.Type_UINT16, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG)}, + {Name: "Type_INT24 ", Type: querypb.Type_INT24, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "Type_UINT24 ", Type: querypb.Type_UINT24, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG)}, + {Name: "Type_INT32 ", Type: querypb.Type_INT32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "Type_UINT32 ", Type: querypb.Type_UINT32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG)}, + {Name: "Type_INT64 ", Type: querypb.Type_INT64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "Type_UINT64 ", Type: querypb.Type_UINT64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG)}, + {Name: "Type_FLOAT32 ", Type: querypb.Type_FLOAT32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "Type_FLOAT64 ", Type: querypb.Type_FLOAT64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "Type_TIMESTAMP", Type: querypb.Type_TIMESTAMP, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_TIMESTAMP_FLAG)}, + {Name: "Type_DATE ", Type: querypb.Type_DATE, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, + {Name: "Type_TIME ", Type: querypb.Type_TIME, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, + {Name: "Type_DATETIME ", Type: querypb.Type_DATETIME, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, + {Name: "Type_YEAR ", Type: querypb.Type_YEAR, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_UNSIGNED_FLAG | querypb.MySqlFlag_NUM_FLAG)}, + {Name: "Type_DECIMAL ", Type: querypb.Type_DECIMAL, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "Type_TEXT ", Type: querypb.Type_TEXT, Charset: uint32(collations.Default())}, + {Name: "Type_BLOB ", Type: querypb.Type_BLOB, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, + {Name: "Type_VARCHAR ", Type: querypb.Type_VARCHAR, Charset: uint32(collations.Default())}, + {Name: "Type_VARBINARY", Type: querypb.Type_VARBINARY, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, + {Name: "Type_CHAR ", Type: querypb.Type_CHAR, Charset: uint32(collations.Default())}, + {Name: "Type_BINARY ", Type: querypb.Type_BINARY, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, + {Name: "Type_BIT ", Type: querypb.Type_BIT, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, + {Name: "Type_ENUM ", Type: querypb.Type_ENUM, Charset: uint32(collations.Default()), Flags: uint32(querypb.MySqlFlag_ENUM_FLAG)}, + {Name: "Type_SET ", Type: querypb.Type_SET, Charset: uint32(collations.Default()), Flags: uint32(querypb.MySqlFlag_SET_FLAG)}, // Skip TUPLE, not possible in Result. - {Name: "Type_GEOMETRY ", Type: querypb.Type_GEOMETRY}, - {Name: "Type_JSON ", Type: querypb.Type_JSON}, + {Name: "Type_GEOMETRY ", Type: querypb.Type_GEOMETRY, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_BLOB_FLAG)}, + {Name: "Type_JSON ", Type: querypb.Type_JSON, Charset: collations.CollationUtf8mb4ID}, }, Rows: [][]sqltypes.Value{ { @@ -526,8 +533,9 @@ func TestQueries(t *testing.T) { checkQuery(t, "first empty string", sConn, cConn, &sqltypes.Result{ Fields: []*querypb.Field{ { - Name: "name", - Type: querypb.Type_VARCHAR, + Name: "name", + Type: querypb.Type_VARCHAR, + Charset: uint32(collations.Default()), }, }, Rows: [][]sqltypes.Value{ @@ -544,7 +552,9 @@ func TestQueries(t *testing.T) { checkQuery(t, "type only", sConn, cConn, &sqltypes.Result{ Fields: []*querypb.Field{ { - Type: querypb.Type_INT64, + Type: querypb.Type_INT64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }, Rows: [][]sqltypes.Value{ @@ -667,6 +677,7 @@ func checkQueryInternal(t *testing.T, query string, sConn, cConn *Conn, result * if !got.Equal(&expected) { for i, f := range got.Fields { if i < len(expected.Fields) && !proto.Equal(f, expected.Fields[i]) { + t.Logf("Query = %v", query) t.Logf("Got field(%v) = %v", i, f) t.Logf("Expected field(%v) = %v", i, expected.Fields[i]) } diff --git a/go/mysql/schema.go b/go/mysql/schema.go index c67517c08ce..c2f6c12e1f7 100644 --- a/go/mysql/schema.go +++ b/go/mysql/schema.go @@ -111,7 +111,7 @@ var BaseShowTablesFields = []*querypb.Field{{ Database: "information_schema", OrgName: "TABLE_NAME", ColumnLength: 192, - Charset: collations.CollationUtf8ID, + Charset: uint32(collations.SystemCollation.Collation), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), }, { Name: "t.table_type", @@ -121,7 +121,7 @@ var BaseShowTablesFields = []*querypb.Field{{ Database: "information_schema", OrgName: "TABLE_TYPE", ColumnLength: 192, - Charset: collations.CollationUtf8ID, + Charset: uint32(collations.SystemCollation.Collation), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), }, { Name: "unix_timestamp(t.create_time)", @@ -137,7 +137,7 @@ var BaseShowTablesFields = []*querypb.Field{{ Database: "information_schema", OrgName: "TABLE_COMMENT", ColumnLength: 6144, - Charset: collations.CollationUtf8ID, + Charset: uint32(collations.SystemCollation.Collation), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), }, { Name: "i.file_size", diff --git a/go/mysql/server_flaky_test.go b/go/mysql/server_flaky_test.go index 4b0bb936fc9..3e10ef0d0ed 100644 --- a/go/mysql/server_flaky_test.go +++ b/go/mysql/server_flaky_test.go @@ -32,6 +32,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/utils" vtenv "vitess.io/vitess/go/vt/env" @@ -46,12 +48,15 @@ import ( var selectRowsResult = &sqltypes.Result{ Fields: []*querypb.Field{ { - Name: "id", - Type: querypb.Type_INT32, + Name: "id", + Type: querypb.Type_INT32, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, { - Name: "name", - Type: querypb.Type_VARCHAR, + Name: "name", + Type: querypb.Type_VARCHAR, + Charset: uint32(collations.CollationUtf8mb4ID), }, }, Rows: [][]sqltypes.Value{ @@ -136,8 +141,9 @@ func (th *testHandler) ComQuery(c *Conn, query string, callback func(*sqltypes.R callback(&sqltypes.Result{ Fields: []*querypb.Field{ { - Name: "schema_name", - Type: querypb.Type_VARCHAR, + Name: "schema_name", + Type: querypb.Type_VARCHAR, + Charset: uint32(collations.Default()), }, }, Rows: [][]sqltypes.Value{ @@ -154,8 +160,9 @@ func (th *testHandler) ComQuery(c *Conn, query string, callback func(*sqltypes.R callback(&sqltypes.Result{ Fields: []*querypb.Field{ { - Name: "ssl_flag", - Type: querypb.Type_VARCHAR, + Name: "ssl_flag", + Type: querypb.Type_VARCHAR, + Charset: uint32(collations.Default()), }, }, Rows: [][]sqltypes.Value{ @@ -168,12 +175,14 @@ func (th *testHandler) ComQuery(c *Conn, query string, callback func(*sqltypes.R callback(&sqltypes.Result{ Fields: []*querypb.Field{ { - Name: "user", - Type: querypb.Type_VARCHAR, + Name: "user", + Type: querypb.Type_VARCHAR, + Charset: uint32(collations.Default()), }, { - Name: "user_data", - Type: querypb.Type_VARCHAR, + Name: "user_data", + Type: querypb.Type_VARCHAR, + Charset: uint32(collations.Default()), }, }, Rows: [][]sqltypes.Value{ @@ -186,8 +195,9 @@ func (th *testHandler) ComQuery(c *Conn, query string, callback func(*sqltypes.R case "50ms delay": callback(&sqltypes.Result{ Fields: []*querypb.Field{{ - Name: "result", - Type: querypb.Type_VARCHAR, + Name: "result", + Type: querypb.Type_VARCHAR, + Charset: uint32(collations.Default()), }}, }) time.Sleep(50 * time.Millisecond) @@ -201,8 +211,9 @@ func (th *testHandler) ComQuery(c *Conn, query string, callback func(*sqltypes.R callback(&sqltypes.Result{ Fields: []*querypb.Field{ { - Name: "result", - Type: querypb.Type_VARCHAR, + Name: "result", + Type: querypb.Type_VARCHAR, + Charset: uint32(collations.Default()), }, }, Rows: [][]sqltypes.Value{ @@ -1486,8 +1497,9 @@ func TestServerFlush(t *testing.T) { assert.Fail(t, "duration out of expected range", "duration: %v, want between %v and %v", duration.String(), (mysqlServerFlushDelay).String(), want.String()) } want1 := []*querypb.Field{{ - Name: "result", - Type: querypb.Type_VARCHAR, + Name: "result", + Type: querypb.Type_VARCHAR, + Charset: uint32(collations.Default()), }} assert.Equal(t, want1, flds) diff --git a/go/vt/binlog/binlog_streamer_rbr_test.go b/go/vt/binlog/binlog_streamer_rbr_test.go index 121a73a9d99..93f70026eba 100644 --- a/go/vt/binlog/binlog_streamer_rbr_test.go +++ b/go/vt/binlog/binlog_streamer_rbr_test.go @@ -23,6 +23,7 @@ import ( "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/mysql/binlog" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vttablet/tabletserver/schema" @@ -44,11 +45,14 @@ func TestStreamerParseRBREvents(t *testing.T) { se.SetTableForTests(&schema.Table{ Name: sqlparser.NewIdentifierCS("vt_a"), Fields: []*querypb.Field{{ - Name: "id", - Type: querypb.Type_INT64, + Name: "id", + Type: querypb.Type_INT64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, { - Name: "message", - Type: querypb.Type_VARCHAR, + Name: "message", + Type: querypb.Type_VARCHAR, + Charset: uint32(collations.Default()), }}, }) @@ -290,11 +294,14 @@ func TestStreamerParseRBRNameEscapes(t *testing.T) { se.SetTableForTests(&schema.Table{ Name: sqlparser.NewIdentifierCS("insert"), Fields: []*querypb.Field{{ - Name: "update", - Type: querypb.Type_INT64, + Name: "update", + Type: querypb.Type_INT64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, { - Name: "delete", - Type: querypb.Type_VARCHAR, + Name: "delete", + Type: querypb.Type_VARCHAR, + Charset: uint32(collations.Default()), }}, }) diff --git a/go/vt/vtctl/vtctl.go b/go/vt/vtctl/vtctl.go index 44c04f0dcb4..023669a4619 100644 --- a/go/vt/vtctl/vtctl.go +++ b/go/vt/vtctl/vtctl.go @@ -97,6 +97,8 @@ import ( "google.golang.org/protobuf/encoding/protojson" "google.golang.org/protobuf/proto" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/cmd/vtctldclient/cli" "vitess.io/vitess/go/flagutil" "vitess.io/vitess/go/json2" @@ -4027,8 +4029,10 @@ func PrintAllCommands(logger logutil.Logger) { func queryResultForTabletResults(results map[string]*sqltypes.Result) *sqltypes.Result { var qr = &sqltypes.Result{} defaultFields := []*querypb.Field{{ - Name: "Tablet", - Type: sqltypes.VarBinary, + Name: "Tablet", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }} var row2 []sqltypes.Value for tabletAlias, result := range results { diff --git a/go/vt/vtexplain/vtexplain_vttablet.go b/go/vt/vtexplain/vtexplain_vttablet.go index 0f876811156..14349dd05f1 100644 --- a/go/vt/vtexplain/vtexplain_vttablet.go +++ b/go/vt/vtexplain/vtexplain_vttablet.go @@ -285,7 +285,9 @@ func newTabletEnvironment(ddls []sqlparser.DDLStatement, opts *Options) (*tablet schemaQueries := map[string]*sqltypes.Result{ "select unix_timestamp()": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG), }}, Rows: [][]sqltypes.Value{ {sqltypes.NewInt32(1427325875)}, @@ -293,95 +295,111 @@ func newTabletEnvironment(ddls []sqlparser.DDLStatement, opts *Options) (*tablet }, "select @@global.sql_mode": { Fields: []*querypb.Field{{ - Type: sqltypes.VarChar, + Type: sqltypes.VarChar, + Charset: uint32(collations.SystemCollation.Collation), }}, Rows: [][]sqltypes.Value{ - {sqltypes.NewVarBinary("STRICT_TRANS_TABLES")}, + {sqltypes.NewVarChar("STRICT_TRANS_TABLES")}, }, }, "select @@session.sql_mode as sql_mode": { Fields: []*querypb.Field{{ - Name: "sql_mode", - Type: sqltypes.VarChar, + Name: "sql_mode", + Type: sqltypes.VarChar, + Charset: uint32(collations.SystemCollation.Collation), }}, Rows: [][]sqltypes.Value{ - {sqltypes.NewVarBinary("STRICT_TRANS_TABLES")}, + {sqltypes.NewVarChar("STRICT_TRANS_TABLES")}, }, }, "select @@autocommit": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), }}, Rows: [][]sqltypes.Value{ - {sqltypes.NewVarBinary("1")}, + {sqltypes.NewInt64(1)}, }, }, "select @@sql_auto_is_null": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), }}, Rows: [][]sqltypes.Value{ - {sqltypes.NewVarBinary("0")}, + {sqltypes.NewInt64(0)}, }, }, "set @@session.sql_log_bin = 0": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Uint64, + Charset: collations.CollationBinaryID, }}, Rows: [][]sqltypes.Value{}, }, "create database if not exists `_vt`": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Uint64, + Charset: collations.CollationBinaryID, }}, Rows: [][]sqltypes.Value{}, }, "drop table if exists `_vt`.redo_log_transaction": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Uint64, + Charset: collations.CollationBinaryID, }}, Rows: [][]sqltypes.Value{}, }, "drop table if exists `_vt`.redo_log_statement": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Uint64, + Charset: collations.CollationBinaryID, }}, Rows: [][]sqltypes.Value{}, }, "drop table if exists `_vt`.transaction": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Uint64, + Charset: collations.CollationBinaryID, }}, Rows: [][]sqltypes.Value{}, }, "drop table if exists `_vt`.participant": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Uint64, + Charset: collations.CollationBinaryID, }}, Rows: [][]sqltypes.Value{}, }, "create table if not exists `_vt`.redo_state(\n dtid varbinary(512),\n state bigint,\n time_created bigint,\n primary key(dtid)\n\t) engine=InnoDB": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Uint64, + Charset: collations.CollationBinaryID, }}, Rows: [][]sqltypes.Value{}, }, "create table if not exists `_vt`.redo_statement(\n dtid varbinary(512),\n id bigint,\n statement mediumblob,\n primary key(dtid, id)\n\t) engine=InnoDB": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Uint64, + Charset: collations.CollationBinaryID, }}, Rows: [][]sqltypes.Value{}, }, "create table if not exists `_vt`.dt_state(\n dtid varbinary(512),\n state bigint,\n time_created bigint,\n primary key(dtid)\n\t) engine=InnoDB": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Uint64, + Charset: collations.CollationBinaryID, }}, Rows: [][]sqltypes.Value{}, }, "create table if not exists `_vt`.dt_participant(\n dtid varbinary(512),\n\tid bigint,\n\tkeyspace varchar(256),\n\tshard varchar(256),\n primary key(dtid, id)\n\t) engine=InnoDB": { Fields: []*querypb.Field{{ - Type: sqltypes.Uint64, + Type: sqltypes.Uint64, + Charset: collations.CollationBinaryID, }}, Rows: [][]sqltypes.Value{}, }, @@ -459,15 +477,17 @@ func newTabletEnvironment(ddls []sqlparser.DDLStatement, opts *Options) (*tablet var colTypes []*querypb.Field var colValues [][]sqltypes.Value colType := &querypb.Field{ - Name: "column_type", - Type: sqltypes.VarChar, + Name: "column_type", + Type: sqltypes.VarChar, + Charset: uint32(collations.Default()), } colTypes = append(colTypes, colType) for _, col := range ddl.GetTableSpec().Columns { colName := strings.ToLower(col.Name.String()) rowType := &querypb.Field{ - Name: colName, - Type: col.Type.SQLType(), + Name: colName, + Type: col.Type.SQLType(), + Charset: uint32(collations.SystemCollation.Collation), } rowTypes = append(rowTypes, rowType) tEnv.tableColumns[table][colName] = col.Type.SQLType() @@ -626,9 +646,12 @@ func (t *explainTablet) handleSelect(query string) (*sqltypes.Result, error) { rows := make([][]sqltypes.Value, 0, rowCount) for i, col := range colNames { colType := colTypes[i] + cs := collations.DefaultCollationForType(colType) fields[i] = &querypb.Field{ - Name: col, - Type: colType, + Name: col, + Type: colType, + Charset: uint32(cs), + Flags: mysql.FlagsForColumn(colType, cs), } } diff --git a/go/vt/vtexplain/vtexplain_vttablet_test.go b/go/vt/vtexplain/vtexplain_vttablet_test.go index 8fd28d07adf..9f6159ad2f9 100644 --- a/go/vt/vtexplain/vtexplain_vttablet_test.go +++ b/go/vt/vtexplain/vtexplain_vttablet_test.go @@ -135,7 +135,7 @@ create table test_partitioned ( t1 := tables["t1"] require.NotNil(t, t1, "table t1 wasn't parsed properly") - wantCols := `[{"name":"id","type":778},{"name":"val","type":6165}]` + wantCols := `[{"name":"id","type":778,"charset":33,"flags":32800},{"name":"val","type":6165,"charset":33}]` got, _ := json.Marshal(t1.Fields) assert.Equal(t, wantCols, string(got)) @@ -143,14 +143,14 @@ create table test_partitioned ( t.Errorf("expected HasPrimary && t1.PKColumns == [0] got %v", t1.PKColumns) } pkCol := t1.GetPKColumn(0) - if pkCol == nil || pkCol.String() != `name:"id" type:UINT64` { + if pkCol == nil || pkCol.String() != `name:"id" type:UINT64 charset:33 flags:32800` { t.Errorf("expected pkCol[0] == id, got %v", pkCol) } t2 := tables["t2"] require.NotNil(t, t2, "table t2 wasn't parsed properly") - wantCols = `[{"name":"val","type":6163}]` + wantCols = `[{"name":"val","type":6163,"charset":33}]` got, _ = json.Marshal(t2.Fields) assert.Equal(t, wantCols, string(got)) diff --git a/go/vt/vtgate/endtoend/vstream_test.go b/go/vt/vtgate/endtoend/vstream_test.go index 03c2e86ff14..ac213f77cec 100644 --- a/go/vt/vtgate/endtoend/vstream_test.go +++ b/go/vt/vtgate/endtoend/vstream_test.go @@ -25,6 +25,7 @@ import ( "sync" "testing" + "vitess.io/vitess/go/mysql/collations" vtgatepb "vitess.io/vitess/go/vt/proto/vtgate" "github.com/stretchr/testify/require" @@ -404,7 +405,7 @@ func TestVStreamCopyResume(t *testing.T) { // lastPK is id1=4, meaning we should only copy rows for id1 IN(5,6,7,8,9) lastPK := sqltypes.Result{ - Fields: []*query.Field{{Name: "id1", Type: query.Type_INT64}}, + Fields: []*query.Field{{Name: "id1", Type: query.Type_INT64, Charset: collations.CollationBinaryID, Flags: uint32(query.MySqlFlag_NUM_FLAG | query.MySqlFlag_BINARY_FLAG)}}, Rows: [][]sqltypes.Value{{sqltypes.NewInt64(4)}}, } tableLastPK := []*binlogdatapb.TableLastPK{{ @@ -466,9 +467,9 @@ func TestVStreamCopyResume(t *testing.T) { `type:ROW row_event:{table_name:"ks.t1_copy_resume" row_changes:{after:{lengths:1 lengths:2 values:"990"}} keyspace:"ks" shard:"-80"} keyspace:"ks" shard:"-80"`, `type:ROW timestamp:[0-9]+ row_event:{table_name:"ks.t1_copy_resume" row_changes:{before:{lengths:1 lengths:1 values:"99"} after:{lengths:1 lengths:2 values:"990"}} keyspace:"ks" shard:"-80"} current_time:[0-9]+ keyspace:"ks" shard:"-80"`, } - redash80 := regexp.MustCompile(`(?i)type:VGTID vgtid:{shard_gtids:{keyspace:"ks" shard:"-80" gtid:".+" table_p_ks:{table_name:"t1_copy_resume" lastpk:{fields:{name:"id1" type:INT64} rows:{lengths:1 values:"[0-9]"}}}} shard_gtids:{keyspace:"ks" shard:"80-" gtid:".+"}} keyspace:"ks" shard:"(-80|80-)"`) - re80dash := regexp.MustCompile(`(?i)type:VGTID vgtid:{shard_gtids:{keyspace:"ks" shard:"-80" gtid:".+"} shard_gtids:{keyspace:"ks" shard:"80-" gtid:".+" table_p_ks:{table_name:"t1_copy_resume" lastpk:{fields:{name:"id1" type:INT64} rows:{lengths:1 values:"[0-9]"}}}}} keyspace:"ks" shard:"(-80|80-)"`) - both := regexp.MustCompile(`(?i)type:VGTID vgtid:{shard_gtids:{keyspace:"ks" shard:"-80" gtid:".+" table_p_ks:{table_name:"t1_copy_resume" lastpk:{fields:{name:"id1" type:INT64} rows:{lengths:1 values:"[0-9]"}}}} shard_gtids:{keyspace:"ks" shard:"80-" gtid:".+" table_p_ks:{table_name:"t1_copy_resume" lastpk:{fields:{name:"id1" type:INT64} rows:{lengths:1 values:"[0-9]"}}}}} keyspace:"ks" shard:"(-80|80-)"`) + redash80 := regexp.MustCompile(`(?i)type:VGTID vgtid:{shard_gtids:{keyspace:"ks" shard:"-80" gtid:".+" table_p_ks:{table_name:"t1_copy_resume" lastpk:{fields:{name:"id1" type:INT64 charset:63 flags:[0-9]+} rows:{lengths:1 values:"[0-9]"}}}} shard_gtids:{keyspace:"ks" shard:"80-" gtid:".+"}} keyspace:"ks" shard:"(-80|80-)"`) + re80dash := regexp.MustCompile(`(?i)type:VGTID vgtid:{shard_gtids:{keyspace:"ks" shard:"-80" gtid:".+"} shard_gtids:{keyspace:"ks" shard:"80-" gtid:".+" table_p_ks:{table_name:"t1_copy_resume" lastpk:{fields:{name:"id1" type:INT64 charset:63 flags:[0-9]+} rows:{lengths:1 values:"[0-9]"}}}}} keyspace:"ks" shard:"(-80|80-)"`) + both := regexp.MustCompile(`(?i)type:VGTID vgtid:{shard_gtids:{keyspace:"ks" shard:"-80" gtid:".+" table_p_ks:{table_name:"t1_copy_resume" lastpk:{fields:{name:"id1" type:INT64 charset:63 flags:[0-9]+} rows:{lengths:1 values:"[0-9]"}}}} shard_gtids:{keyspace:"ks" shard:"80-" gtid:".+" table_p_ks:{table_name:"t1_copy_resume" lastpk:{fields:{name:"id1" type:INT64 charset:63 flags:[0-9]+} rows:{lengths:1 values:"[0-9]"}}}}} keyspace:"ks" shard:"(-80|80-)"`) var evs []*binlogdatapb.VEvent for { diff --git a/go/vt/vtgate/engine/fake_vcursor_test.go b/go/vt/vtgate/engine/fake_vcursor_test.go index 4149d4418e5..a2602da4651 100644 --- a/go/vt/vtgate/engine/fake_vcursor_test.go +++ b/go/vt/vtgate/engine/fake_vcursor_test.go @@ -104,7 +104,7 @@ func (t *noopVCursor) SetContextWithValue(key, value interface{}) func() { // ConnCollation implements VCursor func (t *noopVCursor) ConnCollation() collations.ID { - return collations.CollationUtf8mb4ID + return collations.Default() } func (t *noopVCursor) TimeZone() *time.Location { diff --git a/go/vt/vtgate/engine/filter.go b/go/vt/vtgate/engine/filter.go index 64e5f8ce02c..bab94335e67 100644 --- a/go/vt/vtgate/engine/filter.go +++ b/go/vt/vtgate/engine/filter.go @@ -87,7 +87,7 @@ func (f *Filter) TryStreamExecute(ctx context.Context, vcursor VCursor, bindVars if err != nil { return err } - intEvalResult, err := evalResult.Value().ToInt64() + intEvalResult, err := evalResult.Value(vcursor.ConnCollation()).ToInt64() if err != nil { return err } diff --git a/go/vt/vtgate/engine/insert.go b/go/vt/vtgate/engine/insert.go index 7da6edc3454..2cf1964b3f2 100644 --- a/go/vt/vtgate/engine/insert.go +++ b/go/vt/vtgate/engine/insert.go @@ -679,7 +679,7 @@ func (ins *Insert) getInsertShardedRoute( if err != nil { return nil, nil, err } - rowsResolvedValues = append(rowsResolvedValues, result.Value()) + rowsResolvedValues = append(rowsResolvedValues, result.Value(vcursor.ConnCollation())) } // This is the first iteration: allocate for transpose. if colIdx == 0 { diff --git a/go/vt/vtgate/engine/limit.go b/go/vt/vtgate/engine/limit.go index b70c760b5c0..70cc7d2fb58 100644 --- a/go/vt/vtgate/engine/limit.go +++ b/go/vt/vtgate/engine/limit.go @@ -165,18 +165,18 @@ func (l *Limit) NeedsTransaction() bool { func (l *Limit) getCountAndOffset(ctx context.Context, vcursor VCursor, bindVars map[string]*querypb.BindVariable) (count int, offset int, err error) { env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor) - count, err = getIntFrom(env, l.Count) + count, err = getIntFrom(env, vcursor, l.Count) if err != nil { return } - offset, err = getIntFrom(env, l.Offset) + offset, err = getIntFrom(env, vcursor, l.Offset) if err != nil { return } return } -func getIntFrom(env *evalengine.ExpressionEnv, expr evalengine.Expr) (int, error) { +func getIntFrom(env *evalengine.ExpressionEnv, vcursor VCursor, expr evalengine.Expr) (int, error) { if expr == nil { return 0, nil } @@ -184,7 +184,7 @@ func getIntFrom(env *evalengine.ExpressionEnv, expr evalengine.Expr) (int, error if err != nil { return 0, err } - value := evalResult.Value() + value := evalResult.Value(vcursor.ConnCollation()) if value.IsNull() { return 0, nil } diff --git a/go/vt/vtgate/engine/lock.go b/go/vt/vtgate/engine/lock.go index 8e696507636..c1701f6c166 100644 --- a/go/vt/vtgate/engine/lock.go +++ b/go/vt/vtgate/engine/lock.go @@ -97,7 +97,7 @@ func (l *Lock) execLock(ctx context.Context, vcursor VCursor, bindVars map[strin if err != nil { return nil, err } - lName = er.Value().ToString() + lName = er.Value(vcursor.ConnCollation()).ToString() } qr, err := lf.execLock(ctx, vcursor, bindVars, rss[0]) if err != nil { diff --git a/go/vt/vtgate/engine/memory_sort.go b/go/vt/vtgate/engine/memory_sort.go index 4b6df446995..91c431f0a61 100644 --- a/go/vt/vtgate/engine/memory_sort.go +++ b/go/vt/vtgate/engine/memory_sort.go @@ -168,13 +168,14 @@ func (ms *MemorySort) fetchCount(ctx context.Context, vcursor VCursor, bindVars if err != nil { return 0, err } - if !resolved.Value().IsIntegral() { + value := resolved.Value(vcursor.ConnCollation()) + if !value.IsIntegral() { return 0, sqltypes.ErrIncompatibleTypeCast } - count, err := strconv.Atoi(resolved.Value().RawStr()) + count, err := strconv.Atoi(value.RawStr()) if err != nil || count < 0 { - return 0, fmt.Errorf("requested limit is out of range: %v", resolved.Value().RawStr()) + return 0, fmt.Errorf("requested limit is out of range: %v", value.RawStr()) } return count, nil } diff --git a/go/vt/vtgate/engine/online_ddl.go b/go/vt/vtgate/engine/online_ddl.go index 67290103285..215b8d9be5f 100644 --- a/go/vt/vtgate/engine/online_ddl.go +++ b/go/vt/vtgate/engine/online_ddl.go @@ -20,6 +20,7 @@ import ( "context" "fmt" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/key" querypb "vitess.io/vitess/go/vt/proto/query" @@ -76,8 +77,9 @@ func (v *OnlineDDL) TryExecute(ctx context.Context, vcursor VCursor, bindVars ma result = &sqltypes.Result{ Fields: []*querypb.Field{ { - Name: "uuid", - Type: sqltypes.VarChar, + Name: "uuid", + Type: sqltypes.VarChar, + Charset: uint32(collations.Default()), }, }, Rows: [][]sqltypes.Value{}, diff --git a/go/vt/vtgate/engine/projection.go b/go/vt/vtgate/engine/projection.go index dcd79f0f703..604030cb909 100644 --- a/go/vt/vtgate/engine/projection.go +++ b/go/vt/vtgate/engine/projection.go @@ -20,6 +20,8 @@ import ( "context" "sync" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/vtgate/evalengine" @@ -63,16 +65,16 @@ func (p *Projection) TryExecute(ctx context.Context, vcursor VCursor, bindVars m resultRow := make(sqltypes.Row, 0, len(p.Exprs)) env.Row = row for _, exp := range p.Exprs { - result, err := env.Evaluate(exp) + c, err := env.Evaluate(exp) if err != nil { return nil, err } - resultRow = append(resultRow, result.Value()) + resultRow = append(resultRow, c.Value(vcursor.ConnCollation())) } resultRows = append(resultRows, resultRow) } if wantfields { - result.Fields, err = p.evalFields(env, result.Fields) + result.Fields, err = p.evalFields(env, result.Fields, vcursor) if err != nil { return nil, err } @@ -90,7 +92,7 @@ func (p *Projection) TryStreamExecute(ctx context.Context, vcursor VCursor, bind var err error if wantfields { once.Do(func() { - fields, err = p.evalFields(env, qr.Fields) + fields, err = p.evalFields(env, qr.Fields, vcursor) if err != nil { return } @@ -113,7 +115,7 @@ func (p *Projection) TryStreamExecute(ctx context.Context, vcursor VCursor, bind if err != nil { return err } - resultRow = append(resultRow, c.Value()) + resultRow = append(resultRow, c.Value(vcursor.ConnCollation())) } resultRows = append(resultRows, resultRow) } @@ -129,23 +131,34 @@ func (p *Projection) GetFields(ctx context.Context, vcursor VCursor, bindVars ma return nil, err } env := evalengine.NewExpressionEnv(ctx, bindVars, vcursor) - qr.Fields, err = p.evalFields(env, qr.Fields) + qr.Fields, err = p.evalFields(env, qr.Fields, vcursor) if err != nil { return nil, err } return qr, nil } -func (p *Projection) evalFields(env *evalengine.ExpressionEnv, infields []*querypb.Field) ([]*querypb.Field, error) { +func (p *Projection) evalFields(env *evalengine.ExpressionEnv, infields []*querypb.Field, vcursor VCursor) ([]*querypb.Field, error) { var fields []*querypb.Field for i, col := range p.Cols { - q, err := env.TypeOf(p.Exprs[i], infields) + q, f, err := env.TypeOf(p.Exprs[i], infields) if err != nil { return nil, err } + var cs collations.ID = collations.CollationBinaryID + if sqltypes.IsQuoted(q) && !sqltypes.IsBinary(q) { + cs = vcursor.ConnCollation() + } + + fl := mysql.FlagsForColumn(q, cs) + if !sqltypes.IsNull(q) && !f.Nullable() { + fl |= uint32(querypb.MySqlFlag_NOT_NULL_FLAG) + } fields = append(fields, &querypb.Field{ - Name: col, - Type: q, + Name: col, + Type: q, + Charset: uint32(cs), + Flags: fl, }) } return fields, nil diff --git a/go/vt/vtgate/engine/projection_test.go b/go/vt/vtgate/engine/projection_test.go index 8013c923806..37d1730e2e1 100644 --- a/go/vt/vtgate/engine/projection_test.go +++ b/go/vt/vtgate/engine/projection_test.go @@ -24,6 +24,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/sqlparser" @@ -120,3 +122,50 @@ func TestHexAndBinaryArgument(t *testing.T) { require.NoError(t, err) assert.Equal(t, `[[VARBINARY("\t")]]`, fmt.Sprintf("%v", qr.Rows)) } + +func TestFields(t *testing.T) { + var testCases = []struct { + name string + bindVar *querypb.BindVariable + typ querypb.Type + collation collations.ID + }{ + { + name: `integer`, + bindVar: sqltypes.Int64BindVariable(10), + typ: querypb.Type_INT64, + collation: collations.CollationBinaryID, + }, + { + name: `string`, + bindVar: sqltypes.StringBindVariable("test"), + typ: querypb.Type_VARCHAR, + collation: collations.Default(), + }, + { + name: `binary`, + bindVar: sqltypes.BytesBindVariable([]byte("test")), + typ: querypb.Type_VARBINARY, + collation: collations.CollationBinaryID, + }, + } + + for _, testCase := range testCases { + t.Run(testCase.name, func(t *testing.T) { + bindExpr, err := evalengine.Translate(sqlparser.NewArgument("vtg1"), nil) + require.NoError(t, err) + proj := &Projection{ + Cols: []string{"col"}, + Exprs: []evalengine.Expr{bindExpr}, + Input: &SingleRow{}, + noTxNeeded: noTxNeeded{}, + } + qr, err := proj.TryExecute(context.Background(), &noopVCursor{}, map[string]*querypb.BindVariable{ + "vtg1": testCase.bindVar, + }, true) + require.NoError(t, err) + assert.Equal(t, testCase.typ, qr.Fields[0].Type) + assert.Equal(t, testCase.collation, collations.ID(qr.Fields[0].Charset)) + }) + } +} diff --git a/go/vt/vtgate/engine/route_test.go b/go/vt/vtgate/engine/route_test.go index bb0def01169..e7f3ade2e57 100644 --- a/go/vt/vtgate/engine/route_test.go +++ b/go/vt/vtgate/engine/route_test.go @@ -94,7 +94,7 @@ func TestInformationSchemaWithTableAndSchemaWithRoutedTables(t *testing.T) { stringListToExprList := func(in []string) []evalengine.Expr { var schema []evalengine.Expr for _, s := range in { - schema = append(schema, evalengine.NewLiteralString([]byte(s), collations.TypedCollation{})) + schema = append(schema, evalengine.NewLiteralString([]byte(s), collations.SystemCollation)) } return schema } @@ -109,7 +109,7 @@ func TestInformationSchemaWithTableAndSchemaWithRoutedTables(t *testing.T) { tests := []testCase{{ testName: "both schema and table predicates - routed table", tableSchema: []string{"schema"}, - tableName: map[string]evalengine.Expr{"table_name": evalengine.NewLiteralString([]byte("table"), collations.TypedCollation{})}, + tableName: map[string]evalengine.Expr{"table_name": evalengine.NewLiteralString([]byte("table"), collations.SystemCollation)}, routed: true, expectedLog: []string{ "FindTable(`schema`.`table`)", @@ -118,7 +118,7 @@ func TestInformationSchemaWithTableAndSchemaWithRoutedTables(t *testing.T) { }, { testName: "both schema and table predicates - not routed", tableSchema: []string{"schema"}, - tableName: map[string]evalengine.Expr{"table_name": evalengine.NewLiteralString([]byte("table"), collations.TypedCollation{})}, + tableName: map[string]evalengine.Expr{"table_name": evalengine.NewLiteralString([]byte("table"), collations.SystemCollation)}, routed: false, expectedLog: []string{ "FindTable(`schema`.`table`)", @@ -127,7 +127,7 @@ func TestInformationSchemaWithTableAndSchemaWithRoutedTables(t *testing.T) { }, { testName: "multiple schema and table predicates", tableSchema: []string{"schema", "schema", "schema"}, - tableName: map[string]evalengine.Expr{"t1": evalengine.NewLiteralString([]byte("table"), collations.TypedCollation{}), "t2": evalengine.NewLiteralString([]byte("table"), collations.TypedCollation{}), "t3": evalengine.NewLiteralString([]byte("table"), collations.TypedCollation{})}, + tableName: map[string]evalengine.Expr{"t1": evalengine.NewLiteralString([]byte("table"), collations.SystemCollation), "t2": evalengine.NewLiteralString([]byte("table"), collations.SystemCollation), "t3": evalengine.NewLiteralString([]byte("table"), collations.SystemCollation)}, routed: false, expectedLog: []string{ "FindTable(`schema`.`table`)", @@ -137,7 +137,7 @@ func TestInformationSchemaWithTableAndSchemaWithRoutedTables(t *testing.T) { "ExecuteMultiShard schema.1: dummy_select {__replacevtschemaname: type:INT64 value:\"1\" t1: type:VARCHAR value:\"table\" t2: type:VARCHAR value:\"table\" t3: type:VARCHAR value:\"table\"} false false"}, }, { testName: "table name predicate - routed table", - tableName: map[string]evalengine.Expr{"table_name": evalengine.NewLiteralString([]byte("tableName"), collations.TypedCollation{})}, + tableName: map[string]evalengine.Expr{"table_name": evalengine.NewLiteralString([]byte("tableName"), collations.SystemCollation)}, routed: true, expectedLog: []string{ "FindTable(tableName)", @@ -145,7 +145,7 @@ func TestInformationSchemaWithTableAndSchemaWithRoutedTables(t *testing.T) { "ExecuteMultiShard routedKeyspace.1: dummy_select {table_name: type:VARCHAR value:\"routedTable\"} false false"}, }, { testName: "table name predicate - not routed", - tableName: map[string]evalengine.Expr{"table_name": evalengine.NewLiteralString([]byte("tableName"), collations.TypedCollation{})}, + tableName: map[string]evalengine.Expr{"table_name": evalengine.NewLiteralString([]byte("tableName"), collations.SystemCollation)}, routed: false, expectedLog: []string{ "FindTable(tableName)", @@ -660,7 +660,7 @@ func TestSelectLike(t *testing.T) { sel.Vindex = vindex sel.Values = []evalengine.Expr{ - evalengine.NewLiteralString([]byte("a%"), collations.TypedCollation{}), + evalengine.NewLiteralString([]byte("a%"), collations.SystemCollation), } // md5("a") = 0cc175b9c0f1b6a831c399e269772661 // keyspace id prefix for "a" is 0x0c @@ -690,7 +690,7 @@ func TestSelectLike(t *testing.T) { vc.Rewind() sel.Values = []evalengine.Expr{ - evalengine.NewLiteralString([]byte("ab%"), collations.TypedCollation{}), + evalengine.NewLiteralString([]byte("ab%"), collations.SystemCollation), } // md5("b") = 92eb5ffee6ae2fec3ad71c777531578f // keyspace id prefix for "ab" is 0x0c92 diff --git a/go/vt/vtgate/engine/routing.go b/go/vt/vtgate/engine/routing.go index 47fd382172f..a4f6dabde20 100644 --- a/go/vt/vtgate/engine/routing.go +++ b/go/vt/vtgate/engine/routing.go @@ -197,7 +197,7 @@ func (rp *RoutingParameters) routeInfoSchemaQuery(ctx context.Context, vcursor V if err != nil { return nil, err } - ks := result.Value().ToString() + ks := result.Value(vcursor.ConnCollation()).ToString() if specifiedKS == "" { specifiedKS = ks } @@ -215,7 +215,7 @@ func (rp *RoutingParameters) routeInfoSchemaQuery(ctx context.Context, vcursor V if err != nil { return nil, err } - tabName := val.Value().ToString() + tabName := val.Value(vcursor.ConnCollation()).ToString() tableNames[tblBvName] = tabName bindVars[tblBvName] = sqltypes.StringBindVariable(tabName) } @@ -338,7 +338,7 @@ func (rp *RoutingParameters) equal(ctx context.Context, vcursor VCursor, bindVar if err != nil { return nil, nil, err } - rss, _, err := resolveShards(ctx, vcursor, rp.Vindex.(vindexes.SingleColumn), rp.Keyspace, []sqltypes.Value{value.Value()}) + rss, _, err := resolveShards(ctx, vcursor, rp.Vindex.(vindexes.SingleColumn), rp.Keyspace, []sqltypes.Value{value.Value(vcursor.ConnCollation())}) if err != nil { return nil, nil, err } @@ -357,7 +357,7 @@ func (rp *RoutingParameters) equalMultiCol(ctx context.Context, vcursor VCursor, if err != nil { return nil, nil, err } - rowValue = append(rowValue, v.Value()) + rowValue = append(rowValue, v.Value(vcursor.ConnCollation())) } rss, _, err := resolveShardsMultiCol(ctx, vcursor, rp.Vindex.(vindexes.MultiColumn), rp.Keyspace, [][]sqltypes.Value{rowValue}, false /* shardIdsNeeded */) @@ -577,7 +577,7 @@ func generateRowColValues(ctx context.Context, vcursor VCursor, bindVars map[str return nil, nil, err } isSingleVal[colIdx] = nil - lv = []sqltypes.Value{v.Value()} + lv = []sqltypes.Value{v.Value(vcursor.ConnCollation())} } multiColValues = append(multiColValues, lv) } diff --git a/go/vt/vtgate/engine/set.go b/go/vt/vtgate/engine/set.go index 4b6f9fbd898..09663bc5e31 100644 --- a/go/vt/vtgate/engine/set.go +++ b/go/vt/vtgate/engine/set.go @@ -194,7 +194,7 @@ func (u *UserDefinedVariable) Execute(ctx context.Context, vcursor VCursor, env if err != nil { return err } - return vcursor.Session().SetUDV(u.Name, value.Value()) + return vcursor.Session().SetUDV(u.Name, value.Value(vcursor.ConnCollation())) } var _ SetOp = (*SysVarIgnore)(nil) @@ -459,13 +459,13 @@ func (svss *SysVarSetAware) Execute(ctx context.Context, vcursor VCursor, env *e noop := func(context.Context, bool) error { return nil } err = svss.setBoolSysVar(ctx, env, noop) case sysvars.SQLSelectLimit.Name: - intValue, err := svss.evalAsInt64(env) + intValue, err := svss.evalAsInt64(env, vcursor) if err != nil { return err } vcursor.Session().SetSQLSelectLimit(intValue) // nolint:errcheck case sysvars.TransactionMode.Name: - str, err := svss.evalAsString(env) + str, err := svss.evalAsString(env, vcursor) if err != nil { return err } @@ -475,7 +475,7 @@ func (svss *SysVarSetAware) Execute(ctx context.Context, vcursor VCursor, env *e } vcursor.Session().SetTransactionMode(vtgatepb.TransactionMode(out)) case sysvars.Workload.Name: - str, err := svss.evalAsString(env) + str, err := svss.evalAsString(env, vcursor) if err != nil { return err } @@ -485,7 +485,7 @@ func (svss *SysVarSetAware) Execute(ctx context.Context, vcursor VCursor, env *e } vcursor.Session().SetWorkload(querypb.ExecuteOptions_Workload(out)) case sysvars.DDLStrategy.Name: - str, err := svss.evalAsString(env) + str, err := svss.evalAsString(env, vcursor) if err != nil { return err } @@ -494,7 +494,7 @@ func (svss *SysVarSetAware) Execute(ctx context.Context, vcursor VCursor, env *e } vcursor.Session().SetDDLStrategy(str) case sysvars.QueryTimeout.Name: - queryTimeout, err := svss.evalAsInt64(env) + queryTimeout, err := svss.evalAsInt64(env, vcursor) if err != nil { return err } @@ -502,7 +502,7 @@ func (svss *SysVarSetAware) Execute(ctx context.Context, vcursor VCursor, env *e case sysvars.SessionEnableSystemSettings.Name: err = svss.setBoolSysVar(ctx, env, vcursor.Session().SetSessionEnableSystemSettings) case sysvars.Charset.Name, sysvars.Names.Name: - str, err := svss.evalAsString(env) + str, err := svss.evalAsString(env, vcursor) if err != nil { return err } @@ -514,19 +514,19 @@ func (svss *SysVarSetAware) Execute(ctx context.Context, vcursor VCursor, env *e return vterrors.Errorf(vtrpcpb.Code_UNIMPLEMENTED, "charset/name %v is not supported", str) } case sysvars.ReadAfterWriteGTID.Name: - str, err := svss.evalAsString(env) + str, err := svss.evalAsString(env, vcursor) if err != nil { return err } vcursor.Session().SetReadAfterWriteGTID(str) case sysvars.ReadAfterWriteTimeOut.Name: - val, err := svss.evalAsFloat(env) + val, err := svss.evalAsFloat(env, vcursor) if err != nil { return err } vcursor.Session().SetReadAfterWriteTimeout(val) case sysvars.SessionTrackGTIDs.Name: - str, err := svss.evalAsString(env) + str, err := svss.evalAsString(env, vcursor) if err != nil { return err } @@ -545,15 +545,15 @@ func (svss *SysVarSetAware) Execute(ctx context.Context, vcursor VCursor, env *e return err } -func (svss *SysVarSetAware) evalAsInt64(env *evalengine.ExpressionEnv) (int64, error) { +func (svss *SysVarSetAware) evalAsInt64(env *evalengine.ExpressionEnv, vcursor VCursor) (int64, error) { value, err := env.Evaluate(svss.Expr) if err != nil { return 0, err } - v := value.Value() + v := value.Value(vcursor.ConnCollation()) if !v.IsIntegral() { - return 0, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongTypeForVar, "incorrect argument type to variable '%s': %s", svss.Name, value.Value().Type().String()) + return 0, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongTypeForVar, "incorrect argument type to variable '%s': %s", svss.Name, v.Type().String()) } intValue, err := v.ToInt64() if err != nil { @@ -562,28 +562,28 @@ func (svss *SysVarSetAware) evalAsInt64(env *evalengine.ExpressionEnv) (int64, e return intValue, nil } -func (svss *SysVarSetAware) evalAsFloat(env *evalengine.ExpressionEnv) (float64, error) { +func (svss *SysVarSetAware) evalAsFloat(env *evalengine.ExpressionEnv, vcursor VCursor) (float64, error) { value, err := env.Evaluate(svss.Expr) if err != nil { return 0, err } - v := value.Value() + v := value.Value(vcursor.ConnCollation()) floatValue, err := v.ToFloat64() if err != nil { - return 0, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongTypeForVar, "incorrect argument type to variable '%s': %s", svss.Name, value.Value().Type().String()) + return 0, vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongTypeForVar, "incorrect argument type to variable '%s': %s", svss.Name, v.Type().String()) } return floatValue, nil } -func (svss *SysVarSetAware) evalAsString(env *evalengine.ExpressionEnv) (string, error) { +func (svss *SysVarSetAware) evalAsString(env *evalengine.ExpressionEnv, vcursor VCursor) (string, error) { value, err := env.Evaluate(svss.Expr) if err != nil { return "", err } - v := value.Value() + v := value.Value(vcursor.ConnCollation()) if !v.IsText() && !v.IsBinary() { - return "", vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongTypeForVar, "incorrect argument type to variable '%s': %s", svss.Name, value.Value().Type().String()) + return "", vterrors.NewErrorf(vtrpcpb.Code_INVALID_ARGUMENT, vterrors.WrongTypeForVar, "incorrect argument type to variable '%s': %s", svss.Name, v.Type().String()) } return v.ToString(), nil diff --git a/go/vt/vtgate/engine/update.go b/go/vt/vtgate/engine/update.go index 7b6721af9ac..72aef9005c6 100644 --- a/go/vt/vtgate/engine/update.go +++ b/go/vt/vtgate/engine/update.go @@ -161,7 +161,7 @@ func (upd *Update) updateVindexEntries(ctx context.Context, vcursor VCursor, bin if err != nil { return err } - vindexColumnKeys = append(vindexColumnKeys, resolvedVal.Value()) + vindexColumnKeys = append(vindexColumnKeys, resolvedVal.Value(vcursor.ConnCollation())) } else { // Set the column value to original as this column in vindex is not updated. vindexColumnKeys = append(vindexColumnKeys, origColValue) diff --git a/go/vt/vtgate/engine/vindex_func.go b/go/vt/vtgate/engine/vindex_func.go index 2c52ddbb193..eb55ff32d8c 100644 --- a/go/vt/vtgate/engine/vindex_func.go +++ b/go/vt/vtgate/engine/vindex_func.go @@ -117,10 +117,11 @@ func (vf *VindexFunc) mapVindex(ctx context.Context, vcursor VCursor, bindVars m return nil, err } var values []sqltypes.Value - if k.Value().Type() == querypb.Type_TUPLE { + value := k.Value(vcursor.ConnCollation()) + if value.Type() == querypb.Type_TUPLE { values = k.TupleValues() } else { - values = append(values, k.Value()) + values = append(values, value) } result := &sqltypes.Result{ Fields: vf.Fields, diff --git a/go/vt/vtgate/engine/vindex_lookup.go b/go/vt/vtgate/engine/vindex_lookup.go index e51a4575a5c..2cb7f852be5 100644 --- a/go/vt/vtgate/engine/vindex_lookup.go +++ b/go/vt/vtgate/engine/vindex_lookup.go @@ -254,7 +254,7 @@ func (vr *VindexLookup) generateIds(ctx context.Context, vcursor VCursor, bindVa } switch vr.Opcode { case Equal, EqualUnique: - return []sqltypes.Value{value.Value()}, nil + return []sqltypes.Value{value.Value(vcursor.ConnCollation())}, nil case IN: return value.TupleValues(), nil } diff --git a/go/vt/vtgate/evalengine/api_literal.go b/go/vt/vtgate/evalengine/api_literal.go index 03a8a4f1238..996e4b2f505 100644 --- a/go/vt/vtgate/evalengine/api_literal.go +++ b/go/vt/vtgate/evalengine/api_literal.go @@ -214,11 +214,7 @@ func NewBindVarTuple(key string) *BindVariable { // NewColumn returns a column expression func NewColumn(offset int) *Column { - return NewColumnWithCollation(offset, collations.TypedCollation{ - Collation: collations.Unknown, - Coercibility: collations.CoerceCoercible, - Repertoire: collations.RepertoireUnicode, - }) + return NewColumnWithCollation(offset, collations.SystemCollation) } func NewColumnWithCollation(offset int, coll collations.TypedCollation) *Column { diff --git a/go/vt/vtgate/evalengine/compiler_asm.go b/go/vt/vtgate/evalengine/compiler_asm.go index 1267eaf1d1d..b56801a0dde 100644 --- a/go/vt/vtgate/evalengine/compiler_asm.go +++ b/go/vt/vtgate/evalengine/compiler_asm.go @@ -3330,6 +3330,7 @@ func (asm *assembler) Fn_Now(t querypb.Type, format *datetime.Strftime, prec uin val := env.vm.arena.newEvalBytesEmpty() val.tt = int16(t) val.bytes = format.Format(env.time(utc), prec) + val.col = collationBinary env.vm.stack[env.vm.sp] = val env.vm.sp++ return 1 @@ -3346,6 +3347,7 @@ func (asm *assembler) Fn_Sysdate(prec uint8) { now = now.In(tz) } val.bytes = datetime.NewDateTimeFromStd(now).Format(prec) + val.col = collationBinary env.vm.stack[env.vm.sp] = val env.vm.sp++ return 1 @@ -3358,6 +3360,7 @@ func (asm *assembler) Fn_Curdate() { val := env.vm.arena.newEvalBytesEmpty() val.tt = int16(sqltypes.Date) val.bytes = datetime.Date_YYYY_MM_DD.Format(env.time(false), 0) + val.col = collationBinary env.vm.stack[env.vm.sp] = val env.vm.sp++ return 1 @@ -3370,6 +3373,7 @@ func (asm *assembler) Fn_UtcDate() { val := env.vm.arena.newEvalBytesEmpty() val.tt = int16(sqltypes.Date) val.bytes = datetime.Date_YYYY_MM_DD.Format(env.time(true), 0) + val.col = collationBinary env.vm.stack[env.vm.sp] = val env.vm.sp++ return 1 @@ -4722,3 +4726,13 @@ func (asm *assembler) Fn_REGEXP_REPLACE_slow(merged collations.TypedCollation, f return 1 }, "FN REGEXP_REPLACE_SLOW VARCHAR(SP-2), VARCHAR(SP-1)") } + +func (asm *assembler) Introduce(offset int, t sqltypes.Type, col collations.TypedCollation) { + asm.emit(func(env *ExpressionEnv) int { + arg := evalToBinary(env.vm.stack[env.vm.sp-offset]) + arg.tt = int16(t) + arg.col = col + env.vm.stack[env.vm.sp-offset] = arg + return 1 + }, "INTRODUCE (SP-1)") +} diff --git a/go/vt/vtgate/evalengine/compiler_test.go b/go/vt/vtgate/evalengine/compiler_test.go index 1b5ace371c9..8fa5a3b15c5 100644 --- a/go/vt/vtgate/evalengine/compiler_test.go +++ b/go/vt/vtgate/evalengine/compiler_test.go @@ -448,6 +448,22 @@ func TestCompilerSingle(t *testing.T) { expression: `REGEXP_REPLACE(1234, 12, 6, 1)`, result: `TEXT("634")`, }, + { + expression: `_latin1 0xFF`, + result: `VARCHAR("ÿ")`, + }, + { + expression: `TRIM(_latin1 0xA078A0 FROM _utf8mb4 0xC2A078C2A0)`, + result: `VARCHAR("")`, + }, + { + expression: `CONCAT_WS("😊😂🤢", date '2000-01-01', _latin1 0xFF)`, + result: `VARCHAR("2000-01-01😊😂🤢ÿ")`, + }, + { + expression: `concat('test', _latin1 0xff)`, + result: `VARCHAR("testÿ")`, + }, } for _, tc := range testCases { @@ -499,3 +515,84 @@ func TestCompilerSingle(t *testing.T) { }) } } + +func TestBindVarLiteral(t *testing.T) { + var testCases = []struct { + expression string + bindType func(expr sqlparser.Expr) + bindVar *querypb.BindVariable + result string + }{ + { + expression: `_latin1 :vtg1 /* HEXNUM */`, + bindType: func(expr sqlparser.Expr) { + expr.(*sqlparser.IntroducerExpr).Expr.(*sqlparser.Argument).Type = sqltypes.HexNum + }, + bindVar: sqltypes.HexNumBindVariable([]byte("0xFF")), + result: `VARCHAR("ÿ")`, + }, + { + expression: `cast(:vtg1 /* HEXVAL */ as char character set latin1)`, + bindType: func(expr sqlparser.Expr) { + expr.(*sqlparser.CastExpr).Expr.(*sqlparser.Argument).Type = sqltypes.HexVal + }, + bindVar: sqltypes.HexValBindVariable([]byte("0'FF'")), + result: `VARCHAR("ÿ")`, + }, + } + + for _, tc := range testCases { + t.Run(tc.expression, func(t *testing.T) { + expr, err := sqlparser.ParseExpr(tc.expression) + if err != nil { + t.Fatal(err) + } + + tc.bindType(expr) + + fields := evalengine.FieldResolver(makeFields(nil)) + cfg := &evalengine.Config{ + ResolveColumn: fields.Column, + ResolveType: fields.Type, + Collation: collations.CollationUtf8mb4ID, + Optimization: evalengine.OptimizationLevelCompilerDebug, + } + + converted, err := evalengine.Translate(expr, cfg) + if err != nil { + t.Fatal(err) + } + + result := `VARCHAR("ÿ")` + + env := evalengine.EmptyExpressionEnv() + env.BindVars = map[string]*querypb.BindVariable{ + "vtg1": tc.bindVar, + } + + expected, err := env.Evaluate(evalengine.Deoptimize(converted)) + if err != nil { + t.Fatal(err) + } + if expected.String() != result { + t.Fatalf("bad evaluation from eval engine: got %s, want %s", expected.String(), result) + } + + if cfg.CompilerErr != nil { + t.Fatalf("bad compilation: %v", cfg.CompilerErr) + } + + // re-run the same evaluation multiple times to ensure results are always consistent + for i := 0; i < 8; i++ { + res, err := env.EvaluateVM(converted.(*evalengine.CompiledExpr)) + if err != nil { + t.Fatal(err) + } + + if res.String() != result { + t.Errorf("bad evaluation from compiler: got %s, want %s (iteration %d)", res, result, i) + } + } + }) + } +} diff --git a/go/vt/vtgate/evalengine/eval.go b/go/vt/vtgate/evalengine/eval.go index d264ea52c1b..d11bba24dde 100644 --- a/go/vt/vtgate/evalengine/eval.go +++ b/go/vt/vtgate/evalengine/eval.go @@ -66,6 +66,10 @@ const ( flagIntegerRange = flagIntegerOvf | flagIntegerCap | flagIntegerUdf ) +func (f typeFlag) Nullable() bool { + return f&flagNullable != 0 || f&flagNull != 0 +} + type eval interface { ToRawBytes() []byte SQLType() sqltypes.Type diff --git a/go/vt/vtgate/evalengine/eval_bytes.go b/go/vt/vtgate/evalengine/eval_bytes.go index 67c094a5f67..adc9cb32f2d 100644 --- a/go/vt/vtgate/evalengine/eval_bytes.go +++ b/go/vt/vtgate/evalengine/eval_bytes.go @@ -115,7 +115,7 @@ func (e *evalBytes) Hash(h *vthash.Hasher) { } func (e *evalBytes) isBinary() bool { - return e.SQLType() == sqltypes.VarBinary + return e.SQLType() == sqltypes.VarBinary || e.SQLType() == sqltypes.Binary || e.SQLType() == sqltypes.Blob } func (e *evalBytes) isHexOrBitLiteral() bool { diff --git a/go/vt/vtgate/evalengine/eval_result.go b/go/vt/vtgate/evalengine/eval_result.go index a549ef209bf..8603cc6eac4 100644 --- a/go/vt/vtgate/evalengine/eval_result.go +++ b/go/vt/vtgate/evalengine/eval_result.go @@ -20,6 +20,7 @@ import ( "strings" "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/collations/charset" "vitess.io/vitess/go/sqltypes" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/vterrors" @@ -29,8 +30,25 @@ type EvalResult struct { v eval } -// Value allows for retrieval of the value we expose for public consumption -func (er EvalResult) Value() sqltypes.Value { +// Value allows for retrieval of the value we expose for public consumption. +// It will be converted to the passed in collation which is the connection +// collation and what the client expects the result to be in. +func (er EvalResult) Value(id collations.ID) sqltypes.Value { + str, ok := er.v.(*evalBytes) + if ok && !str.isBinary() { + if str.col.Collation == id { + return sqltypes.MakeTrusted(str.SQLType(), str.bytes) + } + dst, err := charset.Convert(nil, id.Get().Charset(), str.bytes, str.col.Collation.Get().Charset()) + if err != nil { + // If we can't convert, we just return what we have, but it's going + // to be invalidly encoded. Should normally never happen as only utf8mb4 + // is really supported for the connection character set anyway and all + // other charsets can be converted to utf8mb4. + return sqltypes.MakeTrusted(str.SQLType(), str.bytes) + } + return sqltypes.MakeTrusted(str.SQLType(), dst) + } return evalToSQLValue(er.v) } @@ -39,7 +57,7 @@ func (er EvalResult) Collation() collations.ID { } func (er EvalResult) String() string { - return er.Value().String() + return er.Value(collations.Default()).String() } // TupleValues allows for retrieval of the value we expose for public consumption diff --git a/go/vt/vtgate/evalengine/expr_collate.go b/go/vt/vtgate/evalengine/expr_collate.go index 2ba2e3dba61..6e705a7081e 100644 --- a/go/vt/vtgate/evalengine/expr_collate.go +++ b/go/vt/vtgate/evalengine/expr_collate.go @@ -49,7 +49,7 @@ var collationJSON = collations.TypedCollation{ } var collationUtf8mb3 = collations.TypedCollation{ - Collation: collations.CollationUtf8ID, + Collation: collations.CollationUtf8mb3ID, Coercibility: collations.CoerceCoercible, Repertoire: collations.RepertoireUnicode, } @@ -65,6 +65,11 @@ type ( UnaryExpr TypedCollation collations.TypedCollation } + + IntroducerExpr struct { + UnaryExpr + TypedCollation collations.TypedCollation + } ) var _ Expr = (*CollateExpr)(nil) @@ -208,3 +213,40 @@ func (ca *collationAggregation) add(env *collations.Environment, tc collations.T func (ca *collationAggregation) result() collations.TypedCollation { return ca.cur } + +var _ Expr = (*IntroducerExpr)(nil) + +func (expr *IntroducerExpr) eval(env *ExpressionEnv) (eval, error) { + e, err := expr.Inner.eval(env) + if err != nil { + return nil, err + } + if expr.TypedCollation.Collation == collations.CollationBinaryID { + return evalToBinary(e), nil + } + return evalToVarchar(e, expr.TypedCollation.Collation, false) +} + +func (expr *IntroducerExpr) typeof(env *ExpressionEnv, fields []*querypb.Field) (sqltypes.Type, typeFlag) { + if expr.TypedCollation.Collation == collations.CollationBinaryID { + return sqltypes.VarBinary, flagExplicitCollation + } + return sqltypes.VarChar, flagExplicitCollation +} + +func (expr *IntroducerExpr) compile(c *compiler) (ctype, error) { + _, err := expr.Inner.compile(c) + if err != nil { + return ctype{}, err + } + + var ct ctype + ct.Type = sqltypes.VarChar + if expr.TypedCollation.Collation == collations.CollationBinaryID { + ct.Type = sqltypes.VarBinary + } + c.asm.Introduce(1, ct.Type, expr.TypedCollation) + ct.Col = expr.TypedCollation + ct.Flag = flagExplicitCollation + return ct, nil +} diff --git a/go/vt/vtgate/evalengine/expr_env.go b/go/vt/vtgate/evalengine/expr_env.go index 9707b5b9b93..e67e25e70a6 100644 --- a/go/vt/vtgate/evalengine/expr_env.go +++ b/go/vt/vtgate/evalengine/expr_env.go @@ -91,12 +91,12 @@ func (env *ExpressionEnv) Evaluate(expr Expr) (EvalResult, error) { var ErrAmbiguousType = errors.New("the type of this expression cannot be statically computed") -func (env *ExpressionEnv) TypeOf(expr Expr, fields []*querypb.Field) (sqltypes.Type, error) { +func (env *ExpressionEnv) TypeOf(expr Expr, fields []*querypb.Field) (sqltypes.Type, typeFlag, error) { ty, f := expr.typeof(env, fields) if f&flagAmbiguousType != 0 { - return ty, ErrAmbiguousType + return ty, f, ErrAmbiguousType } - return ty, nil + return ty, f, nil } // EmptyExpressionEnv returns a new ExpressionEnv with no bind vars or row diff --git a/go/vt/vtgate/evalengine/fn_string.go b/go/vt/vtgate/evalengine/fn_string.go index 78d8c4b4c5e..7146ac03b68 100644 --- a/go/vt/vtgate/evalengine/fn_string.go +++ b/go/vt/vtgate/evalengine/fn_string.go @@ -620,25 +620,25 @@ func (call builtinPad) eval(env *ExpressionEnv) (eval, error) { } } - length := evalToInt64(l).i - if length < 0 { - return nil, nil - } - + cs := text.col.Collation.Get().Charset() pad, ok := p.(*evalBytes) - if !ok { - pad, err = evalToVarchar(p, call.collate, true) + if !ok || pad.col.Collation.Get().Charset() != cs { + pad, err = evalToVarchar(p, text.col.Collation, true) if err != nil { return nil, err } } + length := evalToInt64(l).i + if length < 0 { + return nil, nil + } + if !validMaxLength(int64(len(pad.bytes)), length) { return nil, nil } // LPAD / RPAD operates on characters, not bytes - cs := text.col.Collation.Get().Charset() strLen := charset.Length(cs, text.bytes) if strLen >= int(length) { @@ -700,14 +700,19 @@ func (call builtinPad) compile(c *compiler) (ctype, error) { case str.isTextual(): col = str.Col default: - c.asm.Convert_xc(3, sqltypes.VarChar, col.Collation, 0, false) + c.asm.Convert_xce(3, sqltypes.VarChar, col.Collation) } _ = c.compileToInt64(l, 2) switch { case pad.isTextual(): + fromCharset := pad.Col.Collation.Get().Charset() + toCharset := col.Collation.Get().Charset() + if fromCharset != toCharset && !toCharset.IsSuperset(fromCharset) { + c.asm.Convert_xce(1, sqltypes.VarChar, col.Collation) + } default: - c.asm.Convert_xc(1, sqltypes.VarChar, col.Collation, 0, false) + c.asm.Convert_xce(1, sqltypes.VarChar, col.Collation) } if call.left { @@ -859,7 +864,7 @@ func (call builtinTrim) eval(env *ExpressionEnv) (eval, error) { } pat, ok := p.(*evalBytes) - if !ok { + if !ok || pat.col.Collation.Get().Charset() != text.col.Collation.Get().Charset() { pat, err = evalToVarchar(p, text.col.Collation, true) if err != nil { return nil, err @@ -919,8 +924,13 @@ func (call builtinTrim) compile(c *compiler) (ctype, error) { switch { case pat.isTextual(): + fromCharset := pat.Col.Collation.Get().Charset() + toCharset := col.Collation.Get().Charset() + if fromCharset != toCharset && !toCharset.IsSuperset(fromCharset) { + c.asm.Convert_xce(1, sqltypes.VarChar, col.Collation) + } default: - c.asm.Convert_xc(1, sqltypes.VarChar, col.Collation, 0, false) + c.asm.Convert_xce(1, sqltypes.VarChar, col.Collation) } switch call.trim { diff --git a/go/vt/vtgate/evalengine/format.go b/go/vt/vtgate/evalengine/format.go index 6d726c63260..4c043d399d4 100644 --- a/go/vt/vtgate/evalengine/format.go +++ b/go/vt/vtgate/evalengine/format.go @@ -122,6 +122,13 @@ func (c *CollateExpr) format(w *formatter, depth int) { w.WriteString(coll.Name()) } +func (i *IntroducerExpr) format(w *formatter, depth int) { + w.WriteString("_") + coll := i.TypedCollation.Collation.Get() + w.WriteString(coll.Name()) + i.Inner.format(w, depth) +} + func (n *NotExpr) format(w *formatter, depth int) { w.WriteString("NOT ") n.Inner.format(w, depth) diff --git a/go/vt/vtgate/evalengine/integration/comparison_test.go b/go/vt/vtgate/evalengine/integration/comparison_test.go index 1ee8fa0a515..9677a39bad0 100644 --- a/go/vt/vtgate/evalengine/integration/comparison_test.go +++ b/go/vt/vtgate/evalengine/integration/comparison_test.go @@ -146,7 +146,7 @@ func compareRemoteExprEnv(t *testing.T, env *evalengine.ExpressionEnv, conn *mys var localVal, remoteVal sqltypes.Value var localCollation, remoteCollation collations.ID if localErr == nil { - v := local.Value() + v := local.Value(collations.Default()) if debugCheckCollations { if v.IsNull() { localCollation = collations.CollationBinaryID diff --git a/go/vt/vtgate/evalengine/integration/fuzz_test.go b/go/vt/vtgate/evalengine/integration/fuzz_test.go index 563bb323244..dea6f295bcf 100644 --- a/go/vt/vtgate/evalengine/integration/fuzz_test.go +++ b/go/vt/vtgate/evalengine/integration/fuzz_test.go @@ -169,7 +169,7 @@ func evaluateLocalEvalengine(env *evalengine.ExpressionEnv, query string, fields }() eval, err = env.Evaluate(local) if err == nil && debugCheckTypes { - tt, err = env.TypeOf(local, fields) + tt, _, err = env.TypeOf(local, fields) if errors.Is(err, evalengine.ErrAmbiguousType) { tt = -1 err = nil @@ -229,7 +229,7 @@ func TestGenerateFuzzCases(t *testing.T) { remoteErr: remoteErr, } if localErr == nil { - res.localVal = eval.Value() + res.localVal = eval.Value(collations.Default()) } if remoteErr == nil { res.remoteVal = remote.Rows[0][0] diff --git a/go/vt/vtgate/evalengine/mysql_test.go b/go/vt/vtgate/evalengine/mysql_test.go index 987ad906b88..82450895871 100644 --- a/go/vt/vtgate/evalengine/mysql_test.go +++ b/go/vt/vtgate/evalengine/mysql_test.go @@ -130,11 +130,11 @@ func TestMySQLGolden(t *testing.T) { continue } if tc.Error != "" { - t.Errorf("query %d: %s\nmysql err: %s\nvitess val: %s", testcount, tc.Query, tc.Error, eval.Value()) + t.Errorf("query %d: %s\nmysql err: %s\nvitess val: %s", testcount, tc.Query, tc.Error, eval.Value(collations.Default())) continue } - if eval.Value().String() != tc.Value { - t.Errorf("query %d: %s\nmysql val: %s\nvitess val: %s", testcount, tc.Query, tc.Value, eval.Value()) + if eval.String() != tc.Value { + t.Errorf("query %d: %s\nmysql val: %s\nvitess val: %s", testcount, tc.Query, tc.Value, eval.Value(collations.Default())) continue } ok++ diff --git a/go/vt/vtgate/evalengine/testcases/inputs.go b/go/vt/vtgate/evalengine/testcases/inputs.go index 5785375955f..3af5b5b6723 100644 --- a/go/vt/vtgate/evalengine/testcases/inputs.go +++ b/go/vt/vtgate/evalengine/testcases/inputs.go @@ -192,6 +192,7 @@ var inputStrings = []string{ "-999999999999999999999999", "_binary 'Müller' ", "_utf8mb4 'abcABCÅå'", + "_latin1 0xFF", // TODO: support other multibyte encodings // "_dec8 'ÒòÅå'", // "_utf8mb3 'abcABCÅå'", @@ -273,9 +274,8 @@ var inputTrimStrings = []string{ "_utf8mb4 '\nabcABCÅå '", // utf8mb4 version of the non-breaking space "_utf8mb4 0xC2A078C2A0", - // TODO: support other multibyte encodings // latin1 version of the non-breaking space - ///"_latin1 0xA078A0", + "_latin1 0xA078A0", } var ipInputs = []string{ diff --git a/go/vt/vtgate/evalengine/translate.go b/go/vt/vtgate/evalengine/translate.go index 8cc6df7bd02..a49faded2c5 100644 --- a/go/vt/vtgate/evalengine/translate.go +++ b/go/vt/vtgate/evalengine/translate.go @@ -371,25 +371,23 @@ func (ast *astCompiler) translateIntroducerExpr(introduced *sqlparser.Introducer return nil, err } } + return expr, nil case *BindVariable: if lit.Type == sqltypes.Tuple { panic("parser allowed introducer before tuple") } - switch collation { - case collations.CollationBinaryID: - lit.Type = sqltypes.VarBinary - lit.Collation = collationBinary - lit.typed = true - default: - lit.Type = sqltypes.VarChar - lit.Collation.Collation = collation - lit.typed = true - } + return &IntroducerExpr{ + UnaryExpr: UnaryExpr{expr}, + TypedCollation: collations.TypedCollation{ + Collation: collation, + Coercibility: collations.CoerceExplicit, + Repertoire: collations.RepertoireUnicode, + }, + }, nil default: panic("character set introducers are only supported for literals and arguments") } - return expr, nil } func (ast *astCompiler) translateIntegral(lit *sqlparser.Literal) (int, bool, error) { @@ -417,7 +415,7 @@ func (ast *astCompiler) translateUnaryExpr(unary *sqlparser.UnaryExpr) (Expr, er case sqlparser.TildaOp: return &BitwiseNotExpr{UnaryExpr: UnaryExpr{expr}}, nil case sqlparser.NStringOp: - return &ConvertExpr{UnaryExpr: UnaryExpr{expr}, Type: "NCHAR", Collation: collations.CollationUtf8ID}, nil + return &ConvertExpr{UnaryExpr: UnaryExpr{expr}, Type: "NCHAR", Collation: collations.CollationUtf8mb3ID}, nil default: return nil, translateExprNotSupported(unary) } diff --git a/go/vt/vtgate/evalengine/translate_card.go b/go/vt/vtgate/evalengine/translate_card.go index 6238aa2b8ef..c8bd04d1dcc 100644 --- a/go/vt/vtgate/evalengine/translate_card.go +++ b/go/vt/vtgate/evalengine/translate_card.go @@ -131,6 +131,8 @@ func (ast *astCompiler) cardExpr(expr Expr) error { return ast.cardUnary(expr.Inner) case *CollateExpr: return ast.cardUnary(expr.Inner) + case *IntroducerExpr: + return ast.cardUnary(expr.Inner) case *IsExpr: return ast.cardUnary(expr.Inner) case *BitwiseNotExpr: diff --git a/go/vt/vtgate/evalengine/translate_convert.go b/go/vt/vtgate/evalengine/translate_convert.go index da631053530..12b1fdfc9ab 100644 --- a/go/vt/vtgate/evalengine/translate_convert.go +++ b/go/vt/vtgate/evalengine/translate_convert.go @@ -105,7 +105,7 @@ func (ast *astCompiler) translateConvertExpr(expr sqlparser.Expr, convertType *s convert.Scale, sqlparser.String(expr), decimal.MyMaxScale) } case "NCHAR": - convert.Collation = collations.CollationUtf8ID + convert.Collation = collations.CollationUtf8mb3ID case "CHAR": convert.Collation, err = ast.translateConvertCharset(convertType.Charset.Name, convertType.Charset.Binary) if err != nil { diff --git a/go/vt/vtgate/evalengine/translate_test.go b/go/vt/vtgate/evalengine/translate_test.go index 955e28981ff..11e59a5de3c 100644 --- a/go/vt/vtgate/evalengine/translate_test.go +++ b/go/vt/vtgate/evalengine/translate_test.go @@ -317,7 +317,7 @@ func TestEvaluate(t *testing.T) { // Then require.NoError(t, err) - assert.Equal(t, test.expected, r.Value(), "expected %s", test.expected.String()) + assert.Equal(t, test.expected, r.Value(collations.Default()), "expected %s", test.expected.String()) }) } } diff --git a/go/vt/vtgate/executor.go b/go/vt/vtgate/executor.go index c474188eb4b..9698825a9d2 100644 --- a/go/vt/vtgate/executor.go +++ b/go/vt/vtgate/executor.go @@ -400,7 +400,7 @@ func (e *Executor) execute(ctx context.Context, mysqlCtx vtgateservice.MySQLConn } // addNeededBindVars adds bind vars that are needed by the plan -func (e *Executor) addNeededBindVars(bindVarNeeds *sqlparser.BindVarNeeds, bindVars map[string]*querypb.BindVariable, session *SafeSession) error { +func (e *Executor) addNeededBindVars(vcursor *vcursorImpl, bindVarNeeds *sqlparser.BindVarNeeds, bindVars map[string]*querypb.BindVariable, session *SafeSession) error { for _, funcName := range bindVarNeeds.NeedFunctionResult { switch funcName { case sqlparser.DBVarName: @@ -494,7 +494,7 @@ func (e *Executor) addNeededBindVars(bindVarNeeds *sqlparser.BindVarNeeds, bindV if err != nil { return err } - bindVars[key] = sqltypes.ValueBindVariable(evaluated.Value()) + bindVars[key] = sqltypes.ValueBindVariable(evaluated.Value(vcursor.collation)) } } } @@ -1224,7 +1224,7 @@ func buildVarCharFields(names ...string) []*querypb.Field { fields[i] = &querypb.Field{ Name: v, Type: sqltypes.VarChar, - Charset: collations.CollationUtf8ID, + Charset: uint32(collations.SystemCollation.Collation), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), } } @@ -1325,7 +1325,7 @@ func (e *Executor) handlePrepare(ctx context.Context, safeSession *SafeSession, return nil, err } - err = e.addNeededBindVars(plan.BindVarNeeds, bindVars, safeSession) + err = e.addNeededBindVars(vcursor, plan.BindVarNeeds, bindVars, safeSession) if err != nil { logStats.Error = err return nil, err diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index d5136711510..60c3002fa0a 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -27,6 +27,7 @@ import ( "time" _flag "vitess.io/vitess/go/internal/flag" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/sqlparser" @@ -168,8 +169,8 @@ func TestSystemVariablesMySQLBelow80(t *testing.T) { sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "orig", Type: sqltypes.VarChar}, - {Name: "new", Type: sqltypes.VarChar}, + {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar(""), @@ -206,8 +207,8 @@ func TestSystemVariablesWithSetVarDisabled(t *testing.T) { sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "orig", Type: sqltypes.VarChar}, - {Name: "new", Type: sqltypes.VarChar}, + {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar(""), @@ -248,8 +249,8 @@ func TestSetSystemVariablesTx(t *testing.T) { sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "orig", Type: sqltypes.VarChar}, - {Name: "new", Type: sqltypes.VarChar}, + {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar(""), @@ -291,8 +292,8 @@ func TestSetSystemVariables(t *testing.T) { lookup.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "orig", Type: sqltypes.VarChar}, - {Name: "new", Type: sqltypes.VarChar}, + {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar(""), @@ -325,7 +326,7 @@ func TestSetSystemVariables(t *testing.T) { lookup.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "sql_safe_updates", Type: sqltypes.VarChar}, + {Name: "sql_safe_updates", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar("0"), @@ -348,7 +349,7 @@ func TestSetSystemVariables(t *testing.T) { lookup.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "max_tmp_tables", Type: sqltypes.VarChar}, + {Name: "max_tmp_tables", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar("4"), @@ -371,7 +372,7 @@ func TestSetSystemVariables(t *testing.T) { lookup.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "max_tmp_tables", Type: sqltypes.VarChar}, + {Name: "max_tmp_tables", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar("1"), @@ -400,8 +401,8 @@ func TestSetSystemVariablesWithReservedConnection(t *testing.T) { sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "orig", Type: sqltypes.VarChar}, - {Name: "new", Type: sqltypes.VarChar}, + {Name: "orig", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "new", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar("only_full_group_by"), @@ -605,8 +606,8 @@ func TestStreamBuffering(t *testing.T) { // such that the splitting of the Result into multiple Result responses gets tested. sbclookup.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.VarChar}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), @@ -633,8 +634,8 @@ func TestStreamBuffering(t *testing.T) { require.NoError(t, err) wantResults := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.VarChar}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, }, { Rows: [][]sqltypes.Value{{ @@ -657,9 +658,9 @@ func TestStreamLimitOffset(t *testing.T) { // such that the splitting of the Result into multiple Result responses gets tested. sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "textcol", Type: sqltypes.VarChar}, - {Name: "weight_string(id)", Type: sqltypes.VarBinary}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "weight_string(id)", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), @@ -674,9 +675,9 @@ func TestStreamLimitOffset(t *testing.T) { sbc2.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "textcol", Type: sqltypes.VarChar}, - {Name: "weight_string(id)", Type: sqltypes.VarBinary}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, + {Name: "weight_string(id)", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(2), @@ -702,8 +703,8 @@ func TestStreamLimitOffset(t *testing.T) { require.NoError(t, err) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "textcol", Type: sqltypes.VarChar}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, Rows: [][]sqltypes.Value{{ @@ -736,7 +737,7 @@ func TestSelectLastInsertId(t *testing.T) { result, err := executorExec(executor, sql, map[string]*querypb.BindVariable{}) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "last_insert_id()", Type: sqltypes.Uint64}, + {Name: "last_insert_id()", Type: sqltypes.Uint64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG)}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewUint64(52), @@ -764,19 +765,19 @@ func TestSelectSystemVariables(t *testing.T) { result, err := executorExec(executor, sql, map[string]*querypb.BindVariable{}) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "@@autocommit", Type: sqltypes.Int64}, - {Name: "@@client_found_rows", Type: sqltypes.Int64}, - {Name: "@@skip_query_plan_cache", Type: sqltypes.Int64}, - {Name: "@@enable_system_settings", Type: sqltypes.Int64}, - {Name: "@@sql_select_limit", Type: sqltypes.Int64}, - {Name: "@@transaction_mode", Type: sqltypes.VarChar}, - {Name: "@@workload", Type: sqltypes.VarChar}, - {Name: "@@read_after_write_gtid", Type: sqltypes.VarChar}, - {Name: "@@read_after_write_timeout", Type: sqltypes.Float64}, - {Name: "@@session_track_gtids", Type: sqltypes.VarChar}, - {Name: "@@ddl_strategy", Type: sqltypes.VarChar}, - {Name: "@@socket", Type: sqltypes.VarChar}, - {Name: "@@query_timeout", Type: sqltypes.Int64}, + {Name: "@@autocommit", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, + {Name: "@@client_found_rows", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, + {Name: "@@skip_query_plan_cache", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, + {Name: "@@enable_system_settings", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, + {Name: "@@sql_select_limit", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, + {Name: "@@transaction_mode", Type: sqltypes.VarChar, Charset: uint32(collations.Default()), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG)}, + {Name: "@@workload", Type: sqltypes.VarChar, Charset: uint32(collations.Default()), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG)}, + {Name: "@@read_after_write_gtid", Type: sqltypes.VarChar, Charset: uint32(collations.Default()), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG)}, + {Name: "@@read_after_write_timeout", Type: sqltypes.Float64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, + {Name: "@@session_track_gtids", Type: sqltypes.VarChar, Charset: uint32(collations.Default()), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG)}, + {Name: "@@ddl_strategy", Type: sqltypes.VarChar, Charset: uint32(collations.Default()), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG)}, + {Name: "@@socket", Type: sqltypes.VarChar, Charset: uint32(collations.Default()), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG)}, + {Name: "@@query_timeout", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, }, Rows: [][]sqltypes.Value{{ // the following are the uninitialised session values @@ -821,9 +822,9 @@ func TestSelectInitializedVitessAwareVariable(t *testing.T) { result, err := executorExec(executor, sql, nil) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "@@autocommit", Type: sqltypes.Int64}, - {Name: "@@enable_system_settings", Type: sqltypes.Int64}, - {Name: "@@query_timeout", Type: sqltypes.Int64}, + {Name: "@@autocommit", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, + {Name: "@@enable_system_settings", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, + {Name: "@@query_timeout", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt64(1), @@ -846,7 +847,7 @@ func TestSelectUserDefinedVariable(t *testing.T) { require.NoError(t, err) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "@foo", Type: sqltypes.Null}, + {Name: "@foo", Type: sqltypes.Null, Charset: collations.CollationBinaryID}, }, Rows: [][]sqltypes.Value{{ sqltypes.NULL, @@ -859,7 +860,7 @@ func TestSelectUserDefinedVariable(t *testing.T) { require.NoError(t, err) wantResult = &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "@foo", Type: sqltypes.VarChar}, + {Name: "@foo", Type: sqltypes.VarChar, Charset: uint32(collations.Default()), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG)}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar("bar"), @@ -882,7 +883,7 @@ func TestFoundRows(t *testing.T) { result, err := executorExec(executor, sql, map[string]*querypb.BindVariable{}) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "found_rows()", Type: sqltypes.Int64}, + {Name: "found_rows()", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt64(1), @@ -912,7 +913,7 @@ func testRowCount(t *testing.T, executor *Executor, wantRowCount int64) { result, err := executorExec(executor, "select row_count()", map[string]*querypb.BindVariable{}) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "row_count()", Type: sqltypes.Int64}, + {Name: "row_count()", Type: sqltypes.Int64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt64(wantRowCount), @@ -929,7 +930,7 @@ func TestSelectLastInsertIdInUnion(t *testing.T) { result1 := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -943,7 +944,7 @@ func TestSelectLastInsertIdInUnion(t *testing.T) { require.NoError(t, err) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG)}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(52), @@ -974,8 +975,8 @@ func TestLastInsertIDInVirtualTable(t *testing.T) { executor.normalize = true result1 := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -1006,7 +1007,7 @@ func TestLastInsertIDInSubQueryExpression(t *testing.T) { require.NoError(t, err) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "x", Type: sqltypes.Uint64}, + {Name: "x", Type: sqltypes.Uint64, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG)}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewUint64(12345), @@ -1035,7 +1036,7 @@ func TestSelectDatabase(t *testing.T) { map[string]*querypb.BindVariable{}) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "database()", Type: sqltypes.VarChar}, + {Name: "database()", Type: sqltypes.VarChar, Charset: uint32(collations.Default()), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG)}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewVarChar("TestExecutor@primary"), @@ -1118,7 +1119,7 @@ func TestSelectBindvars(t *testing.T) { lookup.Queries = nil lookup.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "user_id", Type: sqltypes.Int32}, + {Name: "user_id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, RowsAffected: 0, InsertID: 0, @@ -1753,9 +1754,9 @@ func TestSelectScatterOrderBy(t *testing.T) { sbc := hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_PRIMARY, true, 1, nil) sbc.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "col1", Type: sqltypes.Int32}, - {Name: "col2", Type: sqltypes.Int32}, - {Name: "weight_string(col2)", Type: sqltypes.VarBinary}, + {Name: "col1", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col2", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "weight_string(col2)", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -1785,8 +1786,8 @@ func TestSelectScatterOrderBy(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "col1", Type: sqltypes.Int32}, - {Name: "col2", Type: sqltypes.Int32}, + {Name: "col1", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col2", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, } @@ -1819,8 +1820,8 @@ func TestSelectScatterOrderByVarChar(t *testing.T) { sbc := hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_PRIMARY, true, 1, nil) sbc.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "col1", Type: sqltypes.Int32}, - {Name: "textcol", Type: sqltypes.VarChar}, + {Name: "col1", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -1850,8 +1851,8 @@ func TestSelectScatterOrderByVarChar(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "col1", Type: sqltypes.Int32}, - {Name: "textcol", Type: sqltypes.VarChar}, + {Name: "col1", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, InsertID: 0, } @@ -1883,9 +1884,9 @@ func TestStreamSelectScatterOrderBy(t *testing.T) { sbc := hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_PRIMARY, true, 1, nil) sbc.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, - {Name: "weight_string(col)", Type: sqltypes.VarBinary}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "weight_string(col)", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -1912,8 +1913,8 @@ func TestStreamSelectScatterOrderBy(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, } for i := 0; i < 4; i++ { @@ -1941,8 +1942,8 @@ func TestStreamSelectScatterOrderByVarChar(t *testing.T) { sbc := hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_PRIMARY, true, 1, nil) sbc.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "textcol", Type: sqltypes.VarChar}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -1969,8 +1970,8 @@ func TestStreamSelectScatterOrderByVarChar(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "textcol", Type: sqltypes.VarChar}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "textcol", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, } for i := 0; i < 4; i++ { @@ -1999,9 +2000,9 @@ func TestSelectScatterAggregate(t *testing.T) { sbc := hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_PRIMARY, true, 1, nil) sbc.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "col", Type: sqltypes.Int32}, - {Name: "sum(foo)", Type: sqltypes.Int32}, - {Name: "weight_string(col)", Type: sqltypes.VarBinary}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "sum(foo)", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "weight_string(col)", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2028,8 +2029,8 @@ func TestSelectScatterAggregate(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "col", Type: sqltypes.Int32}, - {Name: "sum(foo)", Type: sqltypes.Decimal}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "sum(foo)", Type: sqltypes.Decimal, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, } @@ -2058,9 +2059,9 @@ func TestStreamSelectScatterAggregate(t *testing.T) { sbc := hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_PRIMARY, true, 1, nil) sbc.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "col", Type: sqltypes.Int32}, - {Name: "sum(foo)", Type: sqltypes.Int32}, - {Name: "weight_string(col)", Type: sqltypes.VarBinary}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "sum(foo)", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "weight_string(col)", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2087,8 +2088,8 @@ func TestStreamSelectScatterAggregate(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "col", Type: sqltypes.Int32}, - {Name: "sum(foo)", Type: sqltypes.Decimal}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "sum(foo)", Type: sqltypes.Decimal, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, } for i := 0; i < 4; i++ { @@ -2118,9 +2119,9 @@ func TestSelectScatterLimit(t *testing.T) { sbc := hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_PRIMARY, true, 1, nil) sbc.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "col1", Type: sqltypes.Int32}, - {Name: "col2", Type: sqltypes.Int32}, - {Name: "weight_string(col2)", Type: sqltypes.VarBinary}, + {Name: "col1", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col2", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "weight_string(col2)", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2147,8 +2148,8 @@ func TestSelectScatterLimit(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "col1", Type: sqltypes.Int32}, - {Name: "col2", Type: sqltypes.Int32}, + {Name: "col1", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col2", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, } @@ -2186,9 +2187,9 @@ func TestStreamSelectScatterLimit(t *testing.T) { sbc := hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_PRIMARY, true, 1, nil) sbc.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "col1", Type: sqltypes.Int32}, - {Name: "col2", Type: sqltypes.Int32}, - {Name: "weight_string(col2)", Type: sqltypes.VarBinary}, + {Name: "col1", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col2", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "weight_string(col2)", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2215,8 +2216,8 @@ func TestStreamSelectScatterLimit(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "col1", Type: sqltypes.Int32}, - {Name: "col2", Type: sqltypes.Int32}, + {Name: "col1", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col2", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, } wantResult.Rows = append(wantResult.Rows, @@ -2342,8 +2343,8 @@ func TestVarJoin(t *testing.T) { result1 := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2377,8 +2378,8 @@ func TestVarJoinStream(t *testing.T) { result1 := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2411,8 +2412,8 @@ func TestLeftJoin(t *testing.T) { defer QueryLogger.Unsubscribe(logChan) result1 := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2422,7 +2423,7 @@ func TestLeftJoin(t *testing.T) { }} emptyResult := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }} sbc1.SetResults(result1) @@ -2452,8 +2453,8 @@ func TestLeftJoinStream(t *testing.T) { executor, sbc1, sbc2, _ := createExecutorEnv() result1 := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2463,7 +2464,7 @@ func TestLeftJoinStream(t *testing.T) { }} emptyResult := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }} sbc1.SetResults(result1) @@ -2494,13 +2495,13 @@ func TestEmptyJoin(t *testing.T) { // which is sent to shard 0. sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }, { Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }}) result, err := executorExec(executor, "select u1.id, u2.id from user u1 join user u2 on u2.id = u1.col where u1.id = 1", nil) @@ -2517,8 +2518,8 @@ func TestEmptyJoin(t *testing.T) { utils.MustMatch(t, wantQueries, sbc1.Queries) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, } if !result.Equal(wantResult) { @@ -2532,11 +2533,11 @@ func TestEmptyJoinStream(t *testing.T) { // which is sent to shard 0. sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }, { Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }}) result, err := executorStream(executor, "select u1.id, u2.id from user u1 join user u2 on u2.id = u1.col where u1.id = 1") @@ -2553,8 +2554,8 @@ func TestEmptyJoinStream(t *testing.T) { utils.MustMatch(t, wantQueries, sbc1.Queries) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, } if !result.Equal(wantResult) { @@ -2567,16 +2568,16 @@ func TestEmptyJoinRecursive(t *testing.T) { // Make sure it also works recursively. sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }, { Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }, { Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }}) result, err := executorExec(executor, "select u1.id, u2.id, u3.id from user u1 join (user u2 join user u3 on u3.id = u2.col) where u1.id = 1", nil) @@ -2596,9 +2597,9 @@ func TestEmptyJoinRecursive(t *testing.T) { utils.MustMatch(t, wantQueries, sbc1.Queries) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "id", Type: sqltypes.Int32}, - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, } if !result.Equal(wantResult) { @@ -2611,16 +2612,16 @@ func TestEmptyJoinRecursiveStream(t *testing.T) { // Make sure it also works recursively. sbc1.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }, { Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }, { Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }}) result, err := executorStream(executor, "select u1.id, u2.id, u3.id from user u1 join (user u2 join user u3 on u3.id = u2.col) where u1.id = 1") @@ -2640,9 +2641,9 @@ func TestEmptyJoinRecursiveStream(t *testing.T) { utils.MustMatch(t, wantQueries, sbc1.Queries) wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "id", Type: sqltypes.Int32}, - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, } if !result.Equal(wantResult) { @@ -2654,8 +2655,8 @@ func TestCrossShardSubquery(t *testing.T) { executor, sbc1, sbc2, _ := createExecutorEnv() result1 := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2680,7 +2681,7 @@ func TestCrossShardSubquery(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), @@ -2695,8 +2696,8 @@ func TestSubQueryAndQueryWithLimit(t *testing.T) { executor, sbc1, sbc2, _ := createExecutorEnv() result1 := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2706,8 +2707,8 @@ func TestSubQueryAndQueryWithLimit(t *testing.T) { }} result2 := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2737,8 +2738,8 @@ func TestCrossShardSubqueryStream(t *testing.T) { executor, sbc1, sbc2, _ := createExecutorEnv() result1 := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -2763,7 +2764,7 @@ func TestCrossShardSubqueryStream(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), @@ -2778,13 +2779,13 @@ func TestCrossShardSubqueryGetFields(t *testing.T) { executor, sbc1, _, sbclookup := createExecutorEnv() sbclookup.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "col", Type: sqltypes.Int32}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }}) result1 := []*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "col", Type: sqltypes.Int32}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, }} sbc1.SetResults(result1) @@ -2803,8 +2804,8 @@ func TestCrossShardSubqueryGetFields(t *testing.T) { wantResult := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "col", Type: sqltypes.Int32}, - {Name: "id", Type: sqltypes.Int32}, + {Name: "col", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, } if !result.Equal(wantResult) { @@ -3054,9 +3055,9 @@ func TestSelectScatterFails(t *testing.T) { sbc := hc.AddTestTablet(cell, shard, 1, "TestExecutor", shard, topodatapb.TabletType_PRIMARY, true, 1, nil) sbc.SetResults([]*sqltypes.Result{{ Fields: []*querypb.Field{ - {Name: "col1", Type: sqltypes.Int32}, - {Name: "col2", Type: sqltypes.Int32}, - {Name: "weight_string(col2)", Type: sqltypes.VarBinary}, + {Name: "col1", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "col2", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "weight_string(col2)", Type: sqltypes.VarBinary, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_BINARY_FLAG)}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ diff --git a/go/vt/vtgate/executor_test.go b/go/vt/vtgate/executor_test.go index f327af5f106..3a0ce80a2e9 100644 --- a/go/vt/vtgate/executor_test.go +++ b/go/vt/vtgate/executor_test.go @@ -35,6 +35,8 @@ import ( "github.com/stretchr/testify/require" "google.golang.org/protobuf/proto" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/cache" "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/sqltypes" @@ -539,7 +541,7 @@ func TestExecutorShow(t *testing.T) { showResults := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "Tables_in_keyspace", Type: sqltypes.VarChar}, + {Name: "Tables_in_keyspace", Type: sqltypes.VarChar, Charset: uint32(collations.SystemCollation.Collation)}, }, RowsAffected: 1, InsertID: 0, @@ -646,7 +648,7 @@ func TestExecutorShow(t *testing.T) { qr, err := executor.Execute(ctx, nil, "TestExecute", session, query, nil) require.NoError(t, err) wantqr := &sqltypes.Result{ - Fields: append(buildVarCharFields("Charset", "Description", "Default collation"), &querypb.Field{Name: "Maxlen", Type: sqltypes.Int32}), + Fields: append(buildVarCharFields("Charset", "Description", "Default collation"), &querypb.Field{Name: "Maxlen", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG | querypb.MySqlFlag_NO_DEFAULT_VALUE_FLAG)}), Rows: [][]sqltypes.Value{ append(buildVarCharRow( "utf8", @@ -667,7 +669,7 @@ func TestExecutorShow(t *testing.T) { qr, err := executor.Execute(ctx, nil, "TestExecute", session, query, nil) require.NoError(t, err) wantqr := &sqltypes.Result{ - Fields: append(buildVarCharFields("Charset", "Description", "Default collation"), &querypb.Field{Name: "Maxlen", Type: sqltypes.Int32}), + Fields: append(buildVarCharFields("Charset", "Description", "Default collation"), &querypb.Field{Name: "Maxlen", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG | querypb.MySqlFlag_NO_DEFAULT_VALUE_FLAG)}), Rows: [][]sqltypes.Value{}, RowsAffected: 0, } @@ -679,7 +681,7 @@ func TestExecutorShow(t *testing.T) { qr, err := executor.Execute(ctx, nil, "TestExecute", session, query, nil) require.NoError(t, err) wantqr := &sqltypes.Result{ - Fields: append(buildVarCharFields("Charset", "Description", "Default collation"), &querypb.Field{Name: "Maxlen", Type: sqltypes.Int32}), + Fields: append(buildVarCharFields("Charset", "Description", "Default collation"), &querypb.Field{Name: "Maxlen", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG | querypb.MySqlFlag_NO_DEFAULT_VALUE_FLAG)}), Rows: [][]sqltypes.Value{ append(buildVarCharRow( "utf8", @@ -695,7 +697,7 @@ func TestExecutorShow(t *testing.T) { qr, err := executor.Execute(ctx, nil, "TestExecute", session, query, nil) require.NoError(t, err) wantqr := &sqltypes.Result{ - Fields: append(buildVarCharFields("Charset", "Description", "Default collation"), &querypb.Field{Name: "Maxlen", Type: sqltypes.Int32}), + Fields: append(buildVarCharFields("Charset", "Description", "Default collation"), &querypb.Field{Name: "Maxlen", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG | querypb.MySqlFlag_NO_DEFAULT_VALUE_FLAG)}), Rows: [][]sqltypes.Value{ append(buildVarCharRow( "utf8mb4", @@ -745,8 +747,8 @@ func TestExecutorShow(t *testing.T) { require.NoError(t, err) wantqr = &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "value", Type: sqltypes.VarChar}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "value", Type: sqltypes.VarChar, Charset: uint32(collations.Default())}, }, Rows: [][]sqltypes.Value{ {sqltypes.NewInt32(1), sqltypes.NewVarChar("foo")}, @@ -946,9 +948,9 @@ func TestExecutorShow(t *testing.T) { require.NoError(t, err) wantqr = &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "Level", Type: sqltypes.VarChar}, - {Name: "Code", Type: sqltypes.Uint16}, - {Name: "Message", Type: sqltypes.VarChar}, + {Name: "Level", Type: sqltypes.VarChar, Charset: uint32(collations.SystemCollation.Collation)}, + {Name: "Code", Type: sqltypes.Uint16, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG)}, + {Name: "Message", Type: sqltypes.VarChar, Charset: uint32(collations.SystemCollation.Collation)}, }, Rows: [][]sqltypes.Value{}, } @@ -960,9 +962,9 @@ func TestExecutorShow(t *testing.T) { require.NoError(t, err) wantqr = &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "Level", Type: sqltypes.VarChar}, - {Name: "Code", Type: sqltypes.Uint16}, - {Name: "Message", Type: sqltypes.VarChar}, + {Name: "Level", Type: sqltypes.VarChar, Charset: uint32(collations.SystemCollation.Collation)}, + {Name: "Code", Type: sqltypes.Uint16, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG)}, + {Name: "Message", Type: sqltypes.VarChar, Charset: uint32(collations.SystemCollation.Collation)}, }, Rows: [][]sqltypes.Value{}, } @@ -977,9 +979,9 @@ func TestExecutorShow(t *testing.T) { require.NoError(t, err) wantqr = &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "Level", Type: sqltypes.VarChar}, - {Name: "Code", Type: sqltypes.Uint16}, - {Name: "Message", Type: sqltypes.VarChar}, + {Name: "Level", Type: sqltypes.VarChar, Charset: uint32(collations.SystemCollation.Collation)}, + {Name: "Code", Type: sqltypes.Uint16, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG)}, + {Name: "Message", Type: sqltypes.VarChar, Charset: uint32(collations.SystemCollation.Collation)}, }, Rows: [][]sqltypes.Value{ diff --git a/go/vt/vtgate/legacy_scatter_conn_test.go b/go/vt/vtgate/legacy_scatter_conn_test.go index 601b07f94c6..ee5e9a2f5f6 100644 --- a/go/vt/vtgate/legacy_scatter_conn_test.go +++ b/go/vt/vtgate/legacy_scatter_conn_test.go @@ -26,6 +26,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/test/utils" "vitess.io/vitess/go/vt/discovery" @@ -509,7 +511,7 @@ func TestAppendResult(t *testing.T) { } innerqr2 := &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "foo", Type: sqltypes.Int8}, + {Name: "foo", Type: sqltypes.Int8, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, }, RowsAffected: 1, InsertID: 1, diff --git a/go/vt/vtgate/plan_execute.go b/go/vt/vtgate/plan_execute.go index 182e20d1980..6ec72d919cb 100644 --- a/go/vt/vtgate/plan_execute.go +++ b/go/vt/vtgate/plan_execute.go @@ -95,7 +95,7 @@ func (e *Executor) newExecute( } // 3: Prepare for execution - err = e.addNeededBindVars(plan.BindVarNeeds, bindVars, safeSession) + err = e.addNeededBindVars(vcursor, plan.BindVarNeeds, bindVars, safeSession) if err != nil { logStats.Error = err return err diff --git a/go/vt/vtgate/planbuilder/expression_converter.go b/go/vt/vtgate/planbuilder/expression_converter.go index 5cdd44feda3..0946f6ef479 100644 --- a/go/vt/vtgate/planbuilder/expression_converter.go +++ b/go/vt/vtgate/planbuilder/expression_converter.go @@ -63,8 +63,7 @@ func booleanValues(astExpr sqlparser.Expr) evalengine.Expr { func identifierAsStringValue(astExpr sqlparser.Expr) evalengine.Expr { colName, isColName := astExpr.(*sqlparser.ColName) if isColName { - // TODO@collations: proper collation for column name - return evalengine.NewLiteralString([]byte(colName.Name.Lowered()), collations.TypedCollation{}) + return evalengine.NewLiteralString([]byte(colName.Name.Lowered()), collations.SystemCollation) } return nil } diff --git a/go/vt/vtgate/planbuilder/from.go b/go/vt/vtgate/planbuilder/from.go index 669b92346e7..b252623d583 100644 --- a/go/vt/vtgate/planbuilder/from.go +++ b/go/vt/vtgate/planbuilder/from.go @@ -287,7 +287,7 @@ func (pb *primitiveBuilder) buildTablePrimitive(tableExpr *sqlparser.AliasedTabl eroute = engine.NewSimpleRoute(engine.EqualUnique, vschemaTable.Keyspace) vindex, _ = vindexes.CreateVindex("binary", "binary", nil) eroute.Vindex = vindex - lit := evalengine.NewLiteralString(vschemaTable.Pinned, collations.TypedCollation{}) + lit := evalengine.NewLiteralString(vschemaTable.Pinned, collations.SystemCollation) eroute.Values = []evalengine.Expr{lit} } eroute.TableName = sqlparser.String(vschemaTable.Name) diff --git a/go/vt/vtgate/planbuilder/operators/route.go b/go/vt/vtgate/planbuilder/operators/route.go index ae2e2e432e1..185032eae6c 100644 --- a/go/vt/vtgate/planbuilder/operators/route.go +++ b/go/vt/vtgate/planbuilder/operators/route.go @@ -20,6 +20,7 @@ import ( "fmt" "strings" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine" @@ -169,7 +170,7 @@ func isConstantFalse(expr sqlparser.Expr) bool { if err != nil { return false } - if eres.Value().IsNull() { + if eres.Value(collations.Default()).IsNull() { return false } b, err := eres.ToBooleanStrict() diff --git a/go/vt/vtgate/planbuilder/operators/sharded_routing.go b/go/vt/vtgate/planbuilder/operators/sharded_routing.go index e565684507e..a49b4e6ba3a 100644 --- a/go/vt/vtgate/planbuilder/operators/sharded_routing.go +++ b/go/vt/vtgate/planbuilder/operators/sharded_routing.go @@ -67,7 +67,7 @@ func newShardedRouting(vtable *vindexes.Table, id semantics.TableSet) Routing { vindex, _ := vindexes.CreateVindex("binary", "binary", nil) routing.Selected = &VindexOption{ Ready: true, - Values: []evalengine.Expr{evalengine.NewLiteralString(vtable.Pinned, collations.TypedCollation{})}, + Values: []evalengine.Expr{evalengine.NewLiteralString(vtable.Pinned, collations.SystemCollation)}, ValueExprs: nil, Predicates: nil, OpCode: engine.EqualUnique, diff --git a/go/vt/vtgate/planbuilder/plan_test.go b/go/vt/vtgate/planbuilder/plan_test.go index 592645f2fef..f5e9636a24d 100644 --- a/go/vt/vtgate/planbuilder/plan_test.go +++ b/go/vt/vtgate/planbuilder/plan_test.go @@ -694,7 +694,7 @@ func (vw *vschemaWrapper) GetSrvVschema() *vschemapb.SrvVSchema { } func (vw *vschemaWrapper) ConnCollation() collations.ID { - return collations.CollationUtf8ID + return collations.Default() } func (vw *vschemaWrapper) PlannerWarning(_ string) { diff --git a/go/vt/vtgate/planbuilder/predicate_rewrite_test.go b/go/vt/vtgate/planbuilder/predicate_rewrite_test.go index 71c8a6f8621..36709f5d591 100644 --- a/go/vt/vtgate/planbuilder/predicate_rewrite_test.go +++ b/go/vt/vtgate/planbuilder/predicate_rewrite_test.go @@ -26,6 +26,8 @@ import ( "github.com/stretchr/testify/assert" "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/evalengine" @@ -135,7 +137,7 @@ func testValues(t *testing.T, env *evalengine.ExpressionEnv, i int, original, si require.NoError(t, err) v2, err := env.Evaluate(simpler) require.NoError(t, err) - assert.Equal(t, v1.Value(), v2.Value()) + assert.Equal(t, v1.Value(collations.Default()), v2.Value(collations.Default())) if len(env.Row) > i+1 { testValues(t, env, i+1, original, simpler) } diff --git a/go/vt/vtgate/planbuilder/project.go b/go/vt/vtgate/planbuilder/project.go index aa6faec524b..11f93b707b7 100644 --- a/go/vt/vtgate/planbuilder/project.go +++ b/go/vt/vtgate/planbuilder/project.go @@ -19,6 +19,7 @@ package planbuilder import ( "fmt" + "vitess.io/vitess/go/mysql/collations" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" @@ -161,8 +162,10 @@ func planProjection(pb *primitiveBuilder, in logicalPlan, expr *sqlparser.Aliase rc := newResultColumn(expr, node) node.resultColumns = append(node.resultColumns, rc) node.eVindexFunc.Fields = append(node.eVindexFunc.Fields, &querypb.Field{ - Name: rc.alias.String(), - Type: querypb.Type_VARBINARY, + Name: rc.alias.String(), + Type: querypb.Type_VARBINARY, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }) node.eVindexFunc.Cols = append(node.eVindexFunc.Cols, col.Metadata.(*column).colNumber) return node, rc, len(node.resultColumns) - 1, nil diff --git a/go/vt/vtgate/planbuilder/show.go b/go/vt/vtgate/planbuilder/show.go index 7a4c5abc14c..27b184b2bcc 100644 --- a/go/vt/vtgate/planbuilder/show.go +++ b/go/vt/vtgate/planbuilder/show.go @@ -134,7 +134,7 @@ func buildShowTargetPlan(vschema plancontext.VSchema) (engine.Primitive, error) func buildCharsetPlan(show *sqlparser.ShowBasic) (engine.Primitive, error) { fields := buildVarCharFields("Charset", "Description", "Default collation") - maxLenField := &querypb.Field{Name: "Maxlen", Type: sqltypes.Int32} + maxLenField := &querypb.Field{Name: "Maxlen", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG | querypb.MySqlFlag_NO_DEFAULT_VALUE_FLAG)} fields = append(fields, maxLenField) charsets := []string{utf8, utf8mb4} @@ -340,7 +340,7 @@ func buildVarCharFields(names ...string) []*querypb.Field { fields[i] = &querypb.Field{ Name: v, Type: sqltypes.VarChar, - Charset: collations.CollationUtf8ID, + Charset: uint32(collations.SystemCollation.Collation), Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG), } } @@ -596,9 +596,9 @@ func buildWarnings() (engine.Primitive, error) { f := func(sa engine.SessionActions) (*sqltypes.Result, error) { fields := []*querypb.Field{ - {Name: "Level", Type: sqltypes.VarChar}, - {Name: "Code", Type: sqltypes.Uint16}, - {Name: "Message", Type: sqltypes.VarChar}, + {Name: "Level", Type: sqltypes.VarChar, Charset: uint32(collations.SystemCollation.Collation)}, + {Name: "Code", Type: sqltypes.Uint16, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG | querypb.MySqlFlag_UNSIGNED_FLAG)}, + {Name: "Message", Type: sqltypes.VarChar, Charset: uint32(collations.SystemCollation.Collation)}, } warns := sa.GetWarnings() diff --git a/go/vt/vtgate/planbuilder/vindex_func.go b/go/vt/vtgate/planbuilder/vindex_func.go index e72a35e7814..e289f7da858 100644 --- a/go/vt/vtgate/planbuilder/vindex_func.go +++ b/go/vt/vtgate/planbuilder/vindex_func.go @@ -19,6 +19,7 @@ package planbuilder import ( "fmt" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" "vitess.io/vitess/go/vt/vtgate/semantics" @@ -130,8 +131,10 @@ func (vf *vindexFunc) SupplyCol(col *sqlparser.ColName) (rc *resultColumn, colNu vf.resultColumns = append(vf.resultColumns, &resultColumn{column: c}) vf.eVindexFunc.Fields = append(vf.eVindexFunc.Fields, &querypb.Field{ - Name: col.Name.String(), - Type: querypb.Type_VARBINARY, + Name: col.Name.String(), + Type: querypb.Type_VARBINARY, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }) // columns that reference vindexFunc will have their colNumber set. @@ -163,8 +166,10 @@ func (vf *vindexFunc) SupplyProjection(expr *sqlparser.AliasedExpr, reuse bool) } vf.eVindexFunc.Fields = append(vf.eVindexFunc.Fields, &querypb.Field{ - Name: expr.ColumnName(), - Type: querypb.Type_VARBINARY, + Name: expr.ColumnName(), + Type: querypb.Type_VARBINARY, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }) vf.eVindexFunc.Cols = append(vf.eVindexFunc.Cols, enum) return len(vf.eVindexFunc.Cols) - 1, nil diff --git a/go/vt/vtgate/semantics/early_rewriter.go b/go/vt/vtgate/semantics/early_rewriter.go index 94dacc529e9..b3553a2de73 100644 --- a/go/vt/vtgate/semantics/early_rewriter.go +++ b/go/vt/vtgate/semantics/early_rewriter.go @@ -20,6 +20,7 @@ import ( "strconv" "strings" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/vtgate/evalengine" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" @@ -310,7 +311,7 @@ func rewriteOrFalse(orExpr sqlparser.OrExpr) sqlparser.Expr { return false } - boolValue, err := res.Value().ToBool() + boolValue, err := res.Value(collations.Default()).ToBool() if err != nil { return false } diff --git a/go/vt/vtgate/simplifier/simplifier_test.go b/go/vt/vtgate/simplifier/simplifier_test.go index 5757f4cf194..63f43a7febb 100644 --- a/go/vt/vtgate/simplifier/simplifier_test.go +++ b/go/vt/vtgate/simplifier/simplifier_test.go @@ -20,6 +20,7 @@ import ( "fmt" "testing" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/vtgate/evalengine" @@ -119,7 +120,7 @@ func TestSimplifyEvalEngineExpr(t *testing.T) { if err != nil { return false } - toInt64, err := res.Value().ToInt64() + toInt64, err := res.Value(collations.Default()).ToInt64() if err != nil { return false } diff --git a/go/vt/vtgate/tabletgateway_flaky_test.go b/go/vt/vtgate/tabletgateway_flaky_test.go index c0e87dfc31e..dd3b9c21790 100644 --- a/go/vt/vtgate/tabletgateway_flaky_test.go +++ b/go/vt/vtgate/tabletgateway_flaky_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/discovery" querypb "vitess.io/vitess/go/vt/proto/query" @@ -63,8 +65,9 @@ func TestGatewayBufferingWhenPrimarySwitchesServingState(t *testing.T) { // add a result to the sandbox connection sqlResult1 := &sqltypes.Result{ Fields: []*querypb.Field{{ - Name: "col1", - Type: sqltypes.VarChar, + Name: "col1", + Type: sqltypes.VarChar, + Charset: uint32(collations.Default()), }}, RowsAffected: 1, Rows: [][]sqltypes.Value{{ @@ -150,8 +153,9 @@ func TestGatewayBufferingWhileReparenting(t *testing.T) { // add a result to the sandbox connection sqlResult1 := &sqltypes.Result{ Fields: []*querypb.Field{{ - Name: "col1", - Type: sqltypes.VarChar, + Name: "col1", + Type: sqltypes.VarChar, + Charset: uint32(collations.Default()), }}, RowsAffected: 1, Rows: [][]sqltypes.Value{{ @@ -277,8 +281,9 @@ func TestInconsistentStateDetectedBuffering(t *testing.T) { // add a result to the sandbox connection sqlResult1 := &sqltypes.Result{ Fields: []*querypb.Field{{ - Name: "col1", - Type: sqltypes.VarChar, + Name: "col1", + Type: sqltypes.VarChar, + Charset: uint32(collations.Default()), }}, RowsAffected: 1, Rows: [][]sqltypes.Value{{ diff --git a/go/vt/vttablet/sandboxconn/sandboxconn.go b/go/vt/vttablet/sandboxconn/sandboxconn.go index d5f6e497339..e51636ec195 100644 --- a/go/vt/vttablet/sandboxconn/sandboxconn.go +++ b/go/vt/vttablet/sandboxconn/sandboxconn.go @@ -25,6 +25,7 @@ import ( "sync/atomic" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/log" "vitess.io/vitess/go/vt/sqlparser" @@ -686,8 +687,10 @@ func getSingleRowResult() *sqltypes.Result { fields := SingleRowResult.Fields for _, field := range fields { singleRowResult.Fields = append(singleRowResult.Fields, &querypb.Field{ - Name: field.Name, - Type: field.Type, + Name: field.Name, + Type: field.Type, + Charset: field.Charset, + Flags: field.Flags, }) } @@ -697,8 +700,8 @@ func getSingleRowResult() *sqltypes.Result { // SingleRowResult is returned when there is no pre-stored result. var SingleRowResult = &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "value", Type: sqltypes.VarChar}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "value", Type: sqltypes.VarChar, Charset: collations.CollationUtf8mb4ID}, }, InsertID: 0, Rows: [][]sqltypes.Value{{ @@ -711,8 +714,8 @@ var SingleRowResult = &sqltypes.Result{ // StreamRowResult is SingleRowResult with RowsAffected set to 0. var StreamRowResult = &sqltypes.Result{ Fields: []*querypb.Field{ - {Name: "id", Type: sqltypes.Int32}, - {Name: "value", Type: sqltypes.VarChar}, + {Name: "id", Type: sqltypes.Int32, Charset: collations.CollationBinaryID, Flags: uint32(querypb.MySqlFlag_NUM_FLAG)}, + {Name: "value", Type: sqltypes.VarChar, Charset: collations.CollationUtf8mb4ID}, }, Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), diff --git a/go/vt/vttablet/tabletserver/query_executor.go b/go/vt/vttablet/tabletserver/query_executor.go index 0ac3dc78155..813f8d1b7b7 100644 --- a/go/vt/vttablet/tabletserver/query_executor.go +++ b/go/vt/vttablet/tabletserver/query_executor.go @@ -26,6 +26,8 @@ import ( "google.golang.org/protobuf/proto" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql" "vitess.io/vitess/go/pools" "vitess.io/vitess/go/sqltypes" @@ -611,7 +613,7 @@ func (qre *QueryExecutor) execNextval() (*sqltypes.Result, error) { return nil, err } tableName := qre.plan.TableName() - v := result.Value() + v := result.Value(collations.Default()) inc, err := v.ToInt64() if err != nil || inc < 1 { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "invalid increment for sequence %s: %s", tableName, v.String()) diff --git a/go/vt/vttablet/tabletserver/schema/historian_test.go b/go/vt/vttablet/tabletserver/schema/historian_test.go index 5fb52d2cd9a..f66306966de 100644 --- a/go/vt/vttablet/tabletserver/schema/historian_test.go +++ b/go/vt/vttablet/tabletserver/schema/historian_test.go @@ -23,6 +23,9 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" binlogdatapb "vitess.io/vitess/go/vt/proto/binlogdata" querypb "vitess.io/vitess/go/vt/proto/query" @@ -35,10 +38,14 @@ func getTable(name string, fieldNames []string, fieldTypes []querypb.Type, pks [ } fields := []*querypb.Field{} for i := range fieldNames { + typ := fieldTypes[i] + cs := collations.DefaultCollationForType(typ) fields = append(fields, &querypb.Field{ - Name: fieldNames[i], - Type: fieldTypes[i], - Table: name, + Name: fieldNames[i], + Type: typ, + Charset: uint32(cs), + Flags: mysql.FlagsForColumn(typ, cs), + Table: name, }) } table := &binlogdatapb.MinimalTable{ @@ -119,7 +126,7 @@ func TestHistorian(t *testing.T) { }, }) require.Nil(t, se.RegisterVersionEvent()) - exp1 := `name:"t1" fields:{name:"id1" type:INT32 table:"t1"} fields:{name:"id2" type:INT32 table:"t1"} p_k_columns:0` + exp1 := `name:"t1" fields:{name:"id1" type:INT32 table:"t1" charset:63 flags:32768} fields:{name:"id2" type:INT32 table:"t1" charset:63 flags:32768} p_k_columns:0` tab, err = se.GetTableForPos(sqlparser.NewIdentifierCS("t1"), gtid1) require.NoError(t, err) require.Equal(t, exp1, fmt.Sprintf("%v", tab)) @@ -139,7 +146,7 @@ func TestHistorian(t *testing.T) { }, }) require.Nil(t, se.RegisterVersionEvent()) - exp2 := `name:"t1" fields:{name:"id1" type:INT32 table:"t1"} fields:{name:"id2" type:VARBINARY table:"t1"} p_k_columns:0` + exp2 := `name:"t1" fields:{name:"id1" type:INT32 table:"t1" charset:63 flags:32768} fields:{name:"id2" type:VARBINARY table:"t1" charset:63 flags:128} p_k_columns:0` tab, err = se.GetTableForPos(sqlparser.NewIdentifierCS("t1"), gtid2) require.NoError(t, err) require.Equal(t, exp2, fmt.Sprintf("%v", tab)) @@ -159,7 +166,7 @@ func TestHistorian(t *testing.T) { }, }) require.Nil(t, se.RegisterVersionEvent()) - exp3 := `name:"t1" fields:{name:"id1" type:INT32 table:"t1"} fields:{name:"id2" type:VARBINARY table:"t1"} fields:{name:"id3" type:INT32 table:"t1"} p_k_columns:0` + exp3 := `name:"t1" fields:{name:"id1" type:INT32 table:"t1" charset:63 flags:32768} fields:{name:"id2" type:VARBINARY table:"t1" charset:63 flags:128} fields:{name:"id3" type:INT32 table:"t1" charset:63 flags:32768} p_k_columns:0` tab, err = se.GetTableForPos(sqlparser.NewIdentifierCS("t1"), gtid3) require.NoError(t, err) require.Equal(t, exp3, fmt.Sprintf("%v", tab)) @@ -242,7 +249,7 @@ func TestHistorianPurgeOldSchemas(t *testing.T) { }, }) require.Nil(t, se.RegisterVersionEvent()) - exp2 := `name:"t1" fields:{name:"id1" type:INT32 table:"t1"} fields:{name:"id2" type:VARBINARY table:"t1"} p_k_columns:0` + exp2 := `name:"t1" fields:{name:"id1" type:INT32 table:"t1" charset:63 flags:32768} fields:{name:"id2" type:VARBINARY table:"t1" charset:63 flags:128} p_k_columns:0` tab, err := se.GetTableForPos(sqlparser.NewIdentifierCS("t1"), gtid2) require.NoError(t, err) require.Equal(t, exp2, fmt.Sprintf("%v", tab)) diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go index 1ef1517995d..07f0a981dd0 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder.go @@ -541,7 +541,7 @@ func (plan *Plan) analyzeWhere(vschema *localVSchema, where *sqlparser.Where) er plan.Filters = append(plan.Filters, Filter{ Opcode: opcode, ColNum: colnum, - Value: resolved.Value(), + Value: resolved.Value(collations.Default()), }) case *sqlparser.FuncExpr: if !expr.Name.EqualString("in_keyrange") { @@ -629,8 +629,10 @@ func (plan *Plan) analyzeExpr(vschema *localVSchema, selExpr sqlparser.SelectExp } return ColExpr{ Field: &querypb.Field{ - Name: "keyspace_id", - Type: sqltypes.VarBinary, + Name: "keyspace_id", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, Vindex: cv.Vindex, VindexColumns: vindexColumns, @@ -653,8 +655,10 @@ func (plan *Plan) analyzeExpr(vschema *localVSchema, selExpr sqlparser.SelectExp } return ColExpr{ Field: &querypb.Field{ - Name: "keyspace_id", - Type: sqltypes.VarBinary, + Name: "keyspace_id", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, Vindex: cv.Vindex, VindexColumns: vindexColumns, @@ -689,8 +693,10 @@ func (plan *Plan) analyzeExpr(vschema *localVSchema, selExpr sqlparser.SelectExp } return ColExpr{ Field: &querypb.Field{ - Name: "1", - Type: querypb.Type_INT64, + Name: "1", + Type: querypb.Type_INT64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NOT_NULL_FLAG | querypb.MySqlFlag_NUM_FLAG), }, ColNum: -1, FixedValue: sqltypes.NewInt64(num), diff --git a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go index 849ad2cec1a..aed178998cb 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/planbuilder_test.go @@ -178,42 +178,58 @@ func TestPlanBuilder(t *testing.T) { t1 := &Table{ Name: "t1", Fields: []*querypb.Field{{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, { - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }}, } // t1alt has no id column t1alt := &Table{ Name: "t1", Fields: []*querypb.Field{{ - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }}, } t2 := &Table{ Name: "t2", Fields: []*querypb.Field{{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, { - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }}, } regional := &Table{ Name: "regional", Fields: []*querypb.Field{{ - Name: "region", - Type: sqltypes.Int64, + Name: "region", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, { - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, { - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }}, } @@ -229,14 +245,18 @@ func TestPlanBuilder(t *testing.T) { ColExprs: []ColExpr{{ ColNum: 0, Field: &querypb.Field{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }, { ColNum: 1, Field: &querypb.Field{ - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, }}, }, @@ -247,14 +267,18 @@ func TestPlanBuilder(t *testing.T) { ColExprs: []ColExpr{{ ColNum: 0, Field: &querypb.Field{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }, { ColNum: 1, Field: &querypb.Field{ - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, }}, Filters: []Filter{{ @@ -273,14 +297,18 @@ func TestPlanBuilder(t *testing.T) { ColExprs: []ColExpr{{ ColNum: 0, Field: &querypb.Field{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }, { ColNum: 1, Field: &querypb.Field{ - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, }}, }, @@ -291,14 +319,18 @@ func TestPlanBuilder(t *testing.T) { ColExprs: []ColExpr{{ ColNum: 0, Field: &querypb.Field{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }, { ColNum: 1, Field: &querypb.Field{ - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, }}, }, @@ -309,14 +341,18 @@ func TestPlanBuilder(t *testing.T) { ColExprs: []ColExpr{{ ColNum: 1, Field: &querypb.Field{ - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, }, { ColNum: 0, Field: &querypb.Field{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }}, }, @@ -327,14 +363,18 @@ func TestPlanBuilder(t *testing.T) { ColExprs: []ColExpr{{ ColNum: 1, Field: &querypb.Field{ - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, }, { ColNum: 0, Field: &querypb.Field{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }}, Filters: []Filter{{ @@ -353,14 +393,18 @@ func TestPlanBuilder(t *testing.T) { ColExprs: []ColExpr{{ ColNum: 1, Field: &querypb.Field{ - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, }, { ColNum: 0, Field: &querypb.Field{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }}, Filters: []Filter{{ @@ -379,14 +423,18 @@ func TestPlanBuilder(t *testing.T) { ColExprs: []ColExpr{{ ColNum: 1, Field: &querypb.Field{ - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, }, { ColNum: 0, Field: &querypb.Field{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }}, Filters: []Filter{{ @@ -408,14 +456,18 @@ func TestPlanBuilder(t *testing.T) { ColExprs: []ColExpr{{ ColNum: 2, Field: &querypb.Field{ - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, }, { ColNum: 1, Field: &querypb.Field{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }}, Filters: []Filter{{ @@ -434,13 +486,17 @@ func TestPlanBuilder(t *testing.T) { ColExprs: []ColExpr{{ ColNum: 1, Field: &querypb.Field{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_NUM_FLAG), }, }, { Field: &querypb.Field{ - Name: "keyspace_id", - Type: sqltypes.VarBinary, + Name: "keyspace_id", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }, Vindex: testLocalVSchema.vschema.Keyspaces["ks"].Vindexes["region_vdx"], VindexColumns: []int{0, 1}, @@ -590,11 +646,15 @@ func TestPlanBuilderFilterComparison(t *testing.T) { t1 := &Table{ Name: "t1", Fields: []*querypb.Field{{ - Name: "id", - Type: sqltypes.Int64, + Name: "id", + Type: sqltypes.Int64, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG | querypb.MySqlFlag_NUM_FLAG), }, { - Name: "val", - Type: sqltypes.VarBinary, + Name: "val", + Type: sqltypes.VarBinary, + Charset: collations.CollationBinaryID, + Flags: uint32(querypb.MySqlFlag_BINARY_FLAG), }}, } hashVindex, err := vindexes.CreateVindex("hash", "hash", nil) diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go index e05df724db6..31e13427af3 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer.go @@ -303,21 +303,18 @@ func (rs *rowStreamer) streamQuery(conn *snapshotConn, send func(*binlogdatapb.V return err } - // first call the callback with the fields - flds, err := conn.Fields() - if err != nil { - return err - } pkfields := make([]*querypb.Field, len(rs.pkColumns)) for i, pk := range rs.pkColumns { pkfields[i] = &querypb.Field{ - Name: flds[pk].Name, - Type: flds[pk].Type, + Name: rs.plan.Table.Fields[pk].Name, + Type: rs.plan.Table.Fields[pk].Type, + Charset: rs.plan.Table.Fields[pk].Charset, + Flags: rs.plan.Table.Fields[pk].Flags, } } - charsets := make([]collations.ID, len(flds)) - for i, fld := range flds { + charsets := make([]collations.ID, len(rs.plan.Table.Fields)) + for i, fld := range rs.plan.Table.Fields { charsets[i] = collations.ID(fld.Charset) } diff --git a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go index 4115a006c37..6ba5a3a5d02 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/rowstreamer_test.go @@ -64,7 +64,7 @@ func TestStreamRowsScan(t *testing.T) { // t1: simulates rollup wantStream := []string{ - `fields:{name:"1" type:INT64} pkfields:{name:"id" type:INT32}`, + `fields:{name:"1" type:INT64 charset:63} pkfields:{name:"id" type:INT32 charset:63}`, `rows:{lengths:1 values:"1"} rows:{lengths:1 values:"1"} lastpk:{lengths:1 values:"2"}`, } wantQuery := "select id, val from t1 order by id" @@ -72,7 +72,7 @@ func TestStreamRowsScan(t *testing.T) { // t1: simulates rollup, with non-pk column wantStream = []string{ - `fields:{name:"1" type:INT64} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32}`, + `fields:{name:"1" type:INT64 charset:63} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32 charset:63}`, `rows:{lengths:1 lengths:3 values:"1aaa"} rows:{lengths:1 lengths:3 values:"1bbb"} lastpk:{lengths:1 values:"2"}`, } wantQuery = "select id, val from t1 order by id" @@ -80,7 +80,7 @@ func TestStreamRowsScan(t *testing.T) { // t1: simulates rollup, with pk and non-pk column wantStream = []string{ - `fields:{name:"1" type:INT64} fields:{name:"id" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32}`, + `fields:{name:"1" type:INT64 charset:63} fields:{name:"id" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32 charset:63}`, `rows:{lengths:1 lengths:1 lengths:3 values:"11aaa"} rows:{lengths:1 lengths:1 lengths:3 values:"12bbb"} lastpk:{lengths:1 values:"2"}`, } wantQuery = "select id, val from t1 order by id" @@ -88,7 +88,7 @@ func TestStreamRowsScan(t *testing.T) { // t1: no pk in select list wantStream = []string{ - `fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32}`, + `fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32 charset:63}`, `rows:{lengths:3 values:"aaa"} rows:{lengths:3 values:"bbb"} lastpk:{lengths:1 values:"2"}`, } wantQuery = "select id, val from t1 order by id" @@ -96,7 +96,7 @@ func TestStreamRowsScan(t *testing.T) { // t1: all rows wantStream = []string{ - `fields:{name:"id" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32}`, + `fields:{name:"id" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32 charset:63}`, `rows:{lengths:1 lengths:3 values:"1aaa"} rows:{lengths:1 lengths:3 values:"2bbb"} lastpk:{lengths:1 values:"2"}`, } wantQuery = "select id, val from t1 order by id" @@ -104,7 +104,7 @@ func TestStreamRowsScan(t *testing.T) { // t1: lastpk=1 wantStream = []string{ - `fields:{name:"id" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32}`, + `fields:{name:"id" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32 charset:63}`, `rows:{lengths:1 lengths:3 values:"2bbb"} lastpk:{lengths:1 values:"2"}`, } wantQuery = "select id, val from t1 where (id > 1) order by id" @@ -112,7 +112,7 @@ func TestStreamRowsScan(t *testing.T) { // t1: different column ordering wantStream = []string{ - `fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} fields:{name:"id" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} pkfields:{name:"id" type:INT32}`, + `fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} fields:{name:"id" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} pkfields:{name:"id" type:INT32 charset:63}`, `rows:{lengths:3 lengths:1 values:"aaa1"} rows:{lengths:3 lengths:1 values:"bbb2"} lastpk:{lengths:1 values:"2"}`, } wantQuery = "select id, val from t1 order by id" @@ -120,7 +120,7 @@ func TestStreamRowsScan(t *testing.T) { // t2: all rows wantStream = []string{ - `fields:{name:"id1" type:INT32 table:"t2" org_table:"t2" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"id2" type:INT32 table:"t2" org_table:"t2" database:"vttest" org_name:"id2" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t2" org_table:"t2" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32} pkfields:{name:"id2" type:INT32}`, + `fields:{name:"id1" type:INT32 table:"t2" org_table:"t2" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"id2" type:INT32 table:"t2" org_table:"t2" database:"vttest" org_name:"id2" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t2" org_table:"t2" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32 charset:63} pkfields:{name:"id2" type:INT32 charset:63}`, `rows:{lengths:1 lengths:1 lengths:3 values:"12aaa"} rows:{lengths:1 lengths:1 lengths:3 values:"13bbb"} lastpk:{lengths:1 lengths:1 values:"13"}`, } wantQuery = "select id1, id2, val from t2 order by id1, id2" @@ -128,7 +128,7 @@ func TestStreamRowsScan(t *testing.T) { // t2: lastpk=1,2 wantStream = []string{ - `fields:{name:"id1" type:INT32 table:"t2" org_table:"t2" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"id2" type:INT32 table:"t2" org_table:"t2" database:"vttest" org_name:"id2" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t2" org_table:"t2" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32} pkfields:{name:"id2" type:INT32}`, + `fields:{name:"id1" type:INT32 table:"t2" org_table:"t2" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"id2" type:INT32 table:"t2" org_table:"t2" database:"vttest" org_name:"id2" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t2" org_table:"t2" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32 charset:63} pkfields:{name:"id2" type:INT32 charset:63}`, `rows:{lengths:1 lengths:1 lengths:3 values:"13bbb"} lastpk:{lengths:1 lengths:1 values:"13"}`, } wantQuery = "select id1, id2, val from t2 where (id1 = 1 and id2 > 2) or (id1 > 1) order by id1, id2" @@ -136,7 +136,7 @@ func TestStreamRowsScan(t *testing.T) { // t3: all rows wantStream = []string{ - `fields:{name:"id" type:INT32 table:"t3" org_table:"t3" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t3" org_table:"t3" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32} pkfields:{name:"val" type:VARBINARY}`, + `fields:{name:"id" type:INT32 table:"t3" org_table:"t3" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t3" org_table:"t3" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32 charset:63} pkfields:{name:"val" type:VARBINARY charset:63}`, `rows:{lengths:1 lengths:3 values:"1aaa"} rows:{lengths:1 lengths:3 values:"2bbb"} lastpk:{lengths:1 lengths:3 values:"2bbb"}`, } wantQuery = "select id, val from t3 order by id, val" @@ -144,7 +144,7 @@ func TestStreamRowsScan(t *testing.T) { // t3: lastpk: 1,'aaa' wantStream = []string{ - `fields:{name:"id" type:INT32 table:"t3" org_table:"t3" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t3" org_table:"t3" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32} pkfields:{name:"val" type:VARBINARY}`, + `fields:{name:"id" type:INT32 table:"t3" org_table:"t3" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t3" org_table:"t3" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32 charset:63} pkfields:{name:"val" type:VARBINARY charset:63}`, `rows:{lengths:1 lengths:3 values:"2bbb"} lastpk:{lengths:1 lengths:3 values:"2bbb"}`, } wantQuery = "select id, val from t3 where (id = 1 and val > 'aaa') or (id > 1) order by id, val" @@ -152,7 +152,7 @@ func TestStreamRowsScan(t *testing.T) { // t4: all rows wantStream = []string{ - `fields:{name:"id1" type:INT32 table:"t4" org_table:"t4" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"id2" type:INT32 table:"t4" org_table:"t4" database:"vttest" org_name:"id2" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"id3" type:INT32 table:"t4" org_table:"t4" database:"vttest" org_name:"id3" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t4" org_table:"t4" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32} pkfields:{name:"id2" type:INT32} pkfields:{name:"id3" type:INT32}`, + `fields:{name:"id1" type:INT32 table:"t4" org_table:"t4" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"id2" type:INT32 table:"t4" org_table:"t4" database:"vttest" org_name:"id2" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"id3" type:INT32 table:"t4" org_table:"t4" database:"vttest" org_name:"id3" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t4" org_table:"t4" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32 charset:63} pkfields:{name:"id2" type:INT32 charset:63} pkfields:{name:"id3" type:INT32 charset:63}`, `rows:{lengths:1 lengths:1 lengths:1 lengths:3 values:"123aaa"} rows:{lengths:1 lengths:1 lengths:1 lengths:3 values:"234bbb"} lastpk:{lengths:1 lengths:1 lengths:1 values:"234"}`, } wantQuery = "select id1, id2, id3, val from t4 order by id1, id2, id3" @@ -160,7 +160,7 @@ func TestStreamRowsScan(t *testing.T) { // t4: lastpk: 1,2,3 wantStream = []string{ - `fields:{name:"id1" type:INT32 table:"t4" org_table:"t4" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"id2" type:INT32 table:"t4" org_table:"t4" database:"vttest" org_name:"id2" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"id3" type:INT32 table:"t4" org_table:"t4" database:"vttest" org_name:"id3" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t4" org_table:"t4" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32} pkfields:{name:"id2" type:INT32} pkfields:{name:"id3" type:INT32}`, + `fields:{name:"id1" type:INT32 table:"t4" org_table:"t4" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"id2" type:INT32 table:"t4" org_table:"t4" database:"vttest" org_name:"id2" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"id3" type:INT32 table:"t4" org_table:"t4" database:"vttest" org_name:"id3" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t4" org_table:"t4" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32 charset:63} pkfields:{name:"id2" type:INT32 charset:63} pkfields:{name:"id3" type:INT32 charset:63}`, `rows:{lengths:1 lengths:1 lengths:1 lengths:3 values:"234bbb"} lastpk:{lengths:1 lengths:1 lengths:1 values:"234"}`, } wantQuery = "select id1, id2, id3, val from t4 where (id1 = 1 and id2 = 2 and id3 > 3) or (id1 = 1 and id2 > 2) or (id1 > 1) order by id1, id2, id3" @@ -255,7 +255,7 @@ func TestStreamRowsKeyRange(t *testing.T) { // Only the first row should be returned, but lastpk should be 6. wantStream := []string{ - `fields:{name:"id1" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32}`, + `fields:{name:"id1" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32 charset:63}`, `rows:{lengths:1 lengths:3 values:"1aaa"} lastpk:{lengths:1 values:"6"}`, } wantQuery := "select id1, val from t1 order by id1" @@ -287,7 +287,7 @@ func TestStreamRowsFilterInt(t *testing.T) { time.Sleep(1 * time.Second) wantStream := []string{ - `fields:{name:"id1" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32}`, + `fields:{name:"id1" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32 charset:63}`, `rows:{lengths:1 lengths:3 values:"1aaa"} rows:{lengths:1 lengths:3 values:"4ddd"} lastpk:{lengths:1 values:"5"}`, } wantQuery := "select id1, id2, val from t1 order by id1" @@ -320,7 +320,7 @@ func TestStreamRowsFilterVarBinary(t *testing.T) { time.Sleep(1 * time.Second) wantStream := []string{ - `fields:{name:"id1" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32}`, + `fields:{name:"id1" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id1" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id1" type:INT32 charset:63}`, `rows:{lengths:1 lengths:6 values:"2newton"} rows:{lengths:1 lengths:6 values:"3newton"} rows:{lengths:1 lengths:6 values:"5newton"} lastpk:{lengths:1 values:"6"}`, } wantQuery := "select id1, val from t1 order by id1" @@ -346,7 +346,7 @@ func TestStreamRowsMultiPacket(t *testing.T) { engine.se.Reload(context.Background()) wantStream := []string{ - `fields:{name:"id" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32}`, + `fields:{name:"id" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} pkfields:{name:"id" type:INT32 charset:63}`, `rows:{lengths:1 lengths:3 values:"1234"} rows:{lengths:1 lengths:4 values:"26789"} rows:{lengths:1 lengths:1 values:"31"} lastpk:{lengths:1 values:"3"}`, `rows:{lengths:1 lengths:10 values:"42345678901"} lastpk:{lengths:1 values:"4"}`, `rows:{lengths:1 lengths:1 values:"52"} lastpk:{lengths:1 values:"5"}`, diff --git a/go/vt/vttablet/tabletserver/vstreamer/uvstreamer_flaky_test.go b/go/vt/vttablet/tabletserver/vstreamer/uvstreamer_flaky_test.go index 1d23cf5a98b..91505ba14af 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/uvstreamer_flaky_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/uvstreamer_flaky_test.go @@ -51,6 +51,7 @@ import ( "testing" "time" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/dbconfigs" "vitess.io/vitess/go/vt/vttablet/tabletserver/throttle/throttlerapp" @@ -153,7 +154,7 @@ func TestVStreamCopyFilterValidations(t *testing.T) { tablePKs := []*binlogdatapb.TableLastPK{{ TableName: "t1", - Lastpk: getQRFromLastPK([]*query.Field{{Name: "id11", Type: query.Type_INT32}}, []sqltypes.Value{sqltypes.NewInt32(10)}), + Lastpk: getQRFromLastPK([]*query.Field{{Name: "id11", Type: query.Type_INT32, Charset: collations.CollationBinaryID, Flags: uint32(query.MySqlFlag_BINARY_FLAG | query.MySqlFlag_NUM_FLAG)}}, []sqltypes.Value{sqltypes.NewInt32(10)}), }} testCases = append(testCases, &TestCase{[]*binlogdatapb.Rule{{Match: "t1"}}, tablePKs, []string{"t1"}, ""}) @@ -407,7 +408,7 @@ func getRule(table string) *binlogdatapb.Rule { } func getTablePK(table string, idx int) *binlogdatapb.TableLastPK { - fields := []*query.Field{{Name: fmt.Sprintf("id%d1", idx), Type: query.Type_INT32}} + fields := []*query.Field{{Name: fmt.Sprintf("id%d1", idx), Type: query.Type_INT32, Charset: collations.CollationBinaryID, Flags: uint32(query.MySqlFlag_BINARY_FLAG | query.MySqlFlag_NUM_FLAG)}} lastPK := []sqltypes.Value{sqltypes.NewInt32(0)} return &binlogdatapb.TableLastPK{ @@ -479,7 +480,7 @@ var expectedEvents = []string{ "type:ROW row_event:{table_name:\"t1\" row_changes:{after:{lengths:1 lengths:2 values:\"880\"}}}", "type:ROW row_event:{table_name:\"t1\" row_changes:{after:{lengths:1 lengths:2 values:\"990\"}}}", "type:ROW row_event:{table_name:\"t1\" row_changes:{after:{lengths:2 lengths:3 values:\"10100\"}}}", - "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t1\" lastpk:{fields:{name:\"id11\" type:INT32} rows:{lengths:2 values:\"10\"}}}}", + "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t1\" lastpk:{fields:{name:\"id11\" type:INT32 charset:63 flags:53251} rows:{lengths:2 values:\"10\"}}}}", "type:COMMIT", "type:BEGIN", "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t1\"} completed:true}", @@ -508,7 +509,7 @@ var expectedEvents = []string{ "type:ROW row_event:{table_name:\"t2\" row_changes:{after:{lengths:1 lengths:3 values:\"9180\"}}}", "type:ROW row_event:{table_name:\"t2\" row_changes:{after:{lengths:2 lengths:3 values:\"10200\"}}}", "type:ROW row_event:{table_name:\"t2\" row_changes:{after:{lengths:2 lengths:3 values:\"11220\"}}}", - "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t2\" lastpk:{fields:{name:\"id21\" type:INT32} rows:{lengths:2 values:\"11\"}}}}", + "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t2\" lastpk:{fields:{name:\"id21\" type:INT32 charset:63 flags:53251} rows:{lengths:2 values:\"11\"}}}}", "type:COMMIT", "type:BEGIN", "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t2\"} completed:true}", @@ -536,7 +537,7 @@ var expectedEvents = []string{ "type:ROW row_event:{table_name:\"t3\" row_changes:{after:{lengths:1 lengths:3 values:\"8240\"}}}", "type:ROW row_event:{table_name:\"t3\" row_changes:{after:{lengths:1 lengths:3 values:\"9270\"}}}", "type:ROW row_event:{table_name:\"t3\" row_changes:{after:{lengths:2 lengths:3 values:\"10300\"}}}", - "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t3\" lastpk:{fields:{name:\"id31\" type:INT32} rows:{lengths:2 values:\"10\"}}}}", + "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t3\" lastpk:{fields:{name:\"id31\" type:INT32 charset:63 flags:53251} rows:{lengths:2 values:\"10\"}}}}", "type:COMMIT", "type:BEGIN", "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t3\"} completed:true}", diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go index 72c56207c95..9c821bdc506 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer.go @@ -771,8 +771,10 @@ func (vs *vstreamer) buildTableColumns(tm *mysql.TableMap) ([]*querypb.Field, er return nil, fmt.Errorf("unsupported type: %d, position: %d", typ, i) } fields = append(fields, &querypb.Field{ - Name: fmt.Sprintf("@%d", i+1), - Type: t, + Name: fmt.Sprintf("@%d", i+1), + Type: t, + Charset: uint32(collations.DefaultCollationForType(t)), + Flags: mysql.FlagsForColumn(t, collations.DefaultCollationForType(t)), }) } st, err := vs.se.GetTableForPos(sqlparser.NewIdentifierCS(tm.Name), mysql.EncodePosition(vs.pos)) diff --git a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_flaky_test.go b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_flaky_test.go index a98088181ac..8eafad07e01 100644 --- a/go/vt/vttablet/tabletserver/vstreamer/vstreamer_flaky_test.go +++ b/go/vt/vttablet/tabletserver/vstreamer/vstreamer_flaky_test.go @@ -575,7 +575,7 @@ func TestVStreamCopyWithDifferentFilters(t *testing.T) { "type:FIELD field_event:{table_name:\"t1\" fields:{name:\"id1\" type:INT32 table:\"t1\" org_table:\"t1\" database:\"vttest\" org_name:\"id1\" column_length:11 charset:63 column_type:\"int(11)\"} fields:{name:\"id2\" type:INT32 table:\"t1\" org_table:\"t1\" database:\"vttest\" org_name:\"id2\" column_length:11 charset:63 column_type:\"int(11)\"}}", "type:GTID", "type:ROW row_event:{table_name:\"t1\" row_changes:{after:{lengths:1 lengths:1 values:\"12\"}}}", - "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t1\" lastpk:{fields:{name:\"id1\" type:INT32} rows:{lengths:1 values:\"1\"}}}}", + "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t1\" lastpk:{fields:{name:\"id1\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\"1\"}}}}", "type:COMMIT", "type:BEGIN", "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t1\"} completed:true}", @@ -583,7 +583,7 @@ func TestVStreamCopyWithDifferentFilters(t *testing.T) { "type:BEGIN", "type:FIELD field_event:{table_name:\"t2a\" fields:{name:\"id1\" type:INT32 table:\"t2a\" org_table:\"t2a\" database:\"vttest\" org_name:\"id1\" column_length:11 charset:63 column_type:\"int(11)\"} fields:{name:\"id2\" type:INT32 table:\"t2a\" org_table:\"t2a\" database:\"vttest\" org_name:\"id2\" column_length:11 charset:63 column_type:\"int(11)\"}}", "type:ROW row_event:{table_name:\"t2a\" row_changes:{after:{lengths:1 lengths:1 values:\"14\"}}}", - "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t2a\" lastpk:{fields:{name:\"id1\" type:INT32} rows:{lengths:1 values:\"1\"}}}}", + "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t2a\" lastpk:{fields:{name:\"id1\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\"1\"}}}}", "type:COMMIT", "type:BEGIN", "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t2a\"} completed:true}", @@ -592,7 +592,7 @@ func TestVStreamCopyWithDifferentFilters(t *testing.T) { "type:FIELD field_event:{table_name:\"t2b\" fields:{name:\"id1\" type:VARCHAR table:\"t2b\" org_table:\"t2b\" database:\"vttest\" org_name:\"id1\" column_length:80 charset:45 column_type:\"varchar(20)\"} fields:{name:\"id2\" type:INT32 table:\"t2b\" org_table:\"t2b\" database:\"vttest\" org_name:\"id2\" column_length:11 charset:63 column_type:\"int(11)\"}}", "type:ROW row_event:{table_name:\"t2b\" row_changes:{after:{lengths:1 lengths:1 values:\"a5\"}}}", "type:ROW row_event:{table_name:\"t2b\" row_changes:{after:{lengths:1 lengths:1 values:\"b6\"}}}", - "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t2b\" lastpk:{fields:{name:\"id1\" type:VARCHAR} rows:{lengths:1 values:\"b\"}}}}", + "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t2b\" lastpk:{fields:{name:\"id1\" type:VARCHAR charset:45 flags:20483} rows:{lengths:1 values:\"b\"}}}}", "type:COMMIT", "type:BEGIN", "type:LASTPK last_p_k_event:{table_last_p_k:{table_name:\"t2b\"} completed:true}", @@ -1230,7 +1230,7 @@ func TestInKeyRangeMultiColumn(t *testing.T) { execStatements(t, input) expectLog(ctx, t, input, ch, [][]string{{ `begin`, - `type:FIELD field_event:{table_name:"t1" fields:{name:"id" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"region" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"region" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} fields:{name:"keyspace_id" type:VARBINARY}}`, + `type:FIELD field_event:{table_name:"t1" fields:{name:"id" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"id" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"region" type:INT32 table:"t1" org_table:"t1" database:"vttest" org_name:"region" column_length:11 charset:63 column_type:"int(11)"} fields:{name:"val" type:VARBINARY table:"t1" org_table:"t1" database:"vttest" org_name:"val" column_length:128 charset:63 column_type:"varbinary(128)"} fields:{name:"keyspace_id" type:VARBINARY charset:63}}`, `type:ROW row_event:{table_name:"t1" row_changes:{after:{lengths:1 lengths:1 lengths:3 lengths:9 values:"11aaa\x01\x16k@\xb4J\xbaK\xd6"}}}`, `type:ROW row_event:{table_name:"t1" row_changes:{before:{lengths:1 lengths:1 lengths:3 lengths:9 values:"11aaa\x01\x16k@\xb4J\xbaK\xd6"} ` + `after:{lengths:1 lengths:1 lengths:3 lengths:9 values:"12aaa\x02\x16k@\xb4J\xbaK\xd6"}}}`, @@ -1609,7 +1609,7 @@ func TestBestEffortNameInFieldEvent(t *testing.T) { // information returned by binlog for val column == varchar (rather than varbinary). output: [][]string{{ `begin`, - `type:FIELD field_event:{table_name:"vitess_test" fields:{name:"@1" type:INT32} fields:{name:"@2" type:VARCHAR}}`, + `type:FIELD field_event:{table_name:"vitess_test" fields:{name:"@1" type:INT32 charset:63} fields:{name:"@2" type:VARCHAR charset:255}}`, `type:ROW row_event:{table_name:"vitess_test" row_changes:{after:{lengths:1 lengths:3 values:"1abc"}}}`, `gtid`, `commit`, From 4666428699ae78a8e976081102b6c1ec45986e2c Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Sat, 15 Jul 2023 17:22:41 +0200 Subject: [PATCH 03/13] normalizer: Simplify normalization This removes the runtime regexp compilation as that's not necessary. It also normalizes the casing to match MySQL to upcase things and adds a leading `0` if needed. We move the helpers for this to a new utility package. Signed-off-by: Dirkjan Bussink --- go/mysql/hex/hex.go | 64 +++++++++++++++++ go/mysql/json/marshal.go | 5 +- go/vt/sqlparser/ast_funcs.go | 34 --------- go/vt/sqlparser/normalizer.go | 26 ++++--- go/vt/sqlparser/normalizer_test.go | 46 ++++++++++--- go/vt/vtgate/evalengine/compiler_asm.go | 30 ++++---- go/vt/vtgate/evalengine/fn_hex.go | 92 +++++-------------------- 7 files changed, 153 insertions(+), 144 deletions(-) create mode 100644 go/mysql/hex/hex.go diff --git a/go/mysql/hex/hex.go b/go/mysql/hex/hex.go new file mode 100644 index 00000000000..4e6cf103905 --- /dev/null +++ b/go/mysql/hex/hex.go @@ -0,0 +1,64 @@ +package hex + +import ( + "encoding/hex" + "math/bits" +) + +const hextable = "0123456789ABCDEF" + +func EncodeBytes(src []byte) []byte { + j := 0 + dst := make([]byte, len(src)*2) + for _, v := range src { + dst[j] = hextable[v>>4] + dst[j+1] = hextable[v&0x0f] + j += 2 + } + return dst +} + +func EncodeUint(u uint64) []byte { + var a [16 + 1]byte + i := len(a) + shift := uint(bits.TrailingZeros(uint(16))) & 7 + b := uint64(16) + m := uint(16) - 1 // == 1<= b { + i-- + a[i] = hextable[uint(u)&m] + u >>= shift + } + + // u < base + i-- + a[i] = hextable[uint(u)] + return a[i:] +} + +func DecodeUint(u uint64) []byte { + if u == 0 { + return []byte{0} + } + var decoded []byte + for u > 0 { + c1 := u % 10 + c2 := u % 100 / 10 + decoded = append([]byte{byte(c1 + c2<<4)}, decoded...) + u /= 100 + } + return decoded +} + +func DecodedLen(src []byte) int { + return (len(src) + 1) / 2 +} + +func DecodeBytes(dst, src []byte) bool { + if len(src)&1 == 1 { + src = append([]byte{'0'}, src...) + } + _, err := hex.Decode(dst, src) + return err == nil +} diff --git a/go/mysql/json/marshal.go b/go/mysql/json/marshal.go index d9e76c5bb76..77d30285a69 100644 --- a/go/mysql/json/marshal.go +++ b/go/mysql/json/marshal.go @@ -17,10 +17,11 @@ limitations under the License. package json import ( - "encoding/hex" "fmt" "math/big" + "vitess.io/vitess/go/hack" + "vitess.io/vitess/go/mysql/hex" querypb "vitess.io/vitess/go/vt/proto/query" "vitess.io/vitess/go/sqltypes" @@ -106,7 +107,7 @@ func (v *Value) marshalSQLInternal(top bool, dst []byte) []byte { dst = append(dst, "CAST("...) } dst = append(dst, "x'"...) - dst = append(dst, hex.EncodeToString([]byte(v.s))...) + dst = append(dst, hex.EncodeBytes(hack.StringBytes(v.s))...) dst = append(dst, "'"...) if top { dst = append(dst, " as JSON)"...) diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go index 521ebf0f486..96ebea71ec6 100644 --- a/go/vt/sqlparser/ast_funcs.go +++ b/go/vt/sqlparser/ast_funcs.go @@ -17,12 +17,10 @@ limitations under the License. package sqlparser import ( - "bytes" "encoding/hex" "encoding/json" "fmt" "io" - "regexp" "strconv" "strings" @@ -594,38 +592,6 @@ func (node *Literal) SQLType() sqltypes.Type { } } -// encodeHexOrBitValToMySQLQueryFormat encodes the hexval or bitval back into the query format -// for passing on to MySQL as a bind var -func (node *Literal) encodeHexOrBitValToMySQLQueryFormat() ([]byte, error) { - nb := node.Bytes() - if node.Type != HexVal && node.Type != BitVal { - return nb, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "Literal value is not a HexVal") - } - - prefix := 'x' - regex := "^x'.*'$" - if node.Type == BitVal { - prefix = 'b' - regex = "^b'.*'$" - } - // Let's make this idempotent in case it's called more than once - match, err := regexp.Match(regex, nb) - if err != nil { - return nb, err - } - if match { - return nb, nil - } - - var bb bytes.Buffer - bb.WriteByte(byte(prefix)) - bb.WriteByte('\'') - bb.WriteString(string(nb)) - bb.WriteByte('\'') - nb = bb.Bytes() - return nb, nil -} - // Equal returns true if the column names match. func (node *ColName) Equal(c *ColName) bool { // Failsafe: ColName should not be empty. diff --git a/go/vt/sqlparser/normalizer.go b/go/vt/sqlparser/normalizer.go index 82c32f33fd7..299f58e016d 100644 --- a/go/vt/sqlparser/normalizer.go +++ b/go/vt/sqlparser/normalizer.go @@ -17,10 +17,11 @@ limitations under the License. package sqlparser import ( - "fmt" + "bytes" "math/big" "vitess.io/vitess/go/mysql/datetime" + "vitess.io/vitess/go/mysql/hex" "vitess.io/vitess/go/sqltypes" vtrpcpb "vitess.io/vitess/go/vt/proto/vtrpc" "vitess.io/vitess/go/vt/vterrors" @@ -352,16 +353,18 @@ func SQLToBindvar(node SQLNode) *querypb.BindVariable { case DecimalVal: v, err = sqltypes.NewValue(sqltypes.Decimal, node.Bytes()) case HexNum: - v, err = sqltypes.NewValue(sqltypes.HexNum, node.Bytes()) + buf := make([]byte, 0, len(node.Bytes())) + buf = append(buf, "0x"...) + buf = append(buf, bytes.ToUpper(node.Bytes()[2:])...) + v, err = sqltypes.NewValue(sqltypes.HexNum, buf) case HexVal: // We parse the `x'7b7d'` string literal into a hex encoded string of `7b7d` in the parser // We need to re-encode it back to the original MySQL query format before passing it on as a bindvar value to MySQL - var vbytes []byte - vbytes, err = node.encodeHexOrBitValToMySQLQueryFormat() - if err != nil { - return nil - } - v, err = sqltypes.NewValue(sqltypes.HexVal, vbytes) + buf := make([]byte, 0, len(node.Bytes())+3) + buf = append(buf, 'x', '\'') + buf = append(buf, bytes.ToUpper(node.Bytes())...) + buf = append(buf, '\'') + v, err = sqltypes.NewValue(sqltypes.HexVal, buf) case BitVal: // Convert bit value to hex number in parameterized query format var i big.Int @@ -369,7 +372,12 @@ func SQLToBindvar(node SQLNode) *querypb.BindVariable { if !ok { return nil } - v, err = sqltypes.NewValue(sqltypes.HexNum, []byte(fmt.Sprintf("0x%s", i.Text(16)))) + + buf := i.Bytes() + out := make([]byte, 0, (len(buf)*2)+2) + out = append(out, '0', 'x') + out = append(out, hex.EncodeBytes(buf)...) + v, err = sqltypes.NewValue(sqltypes.HexNum, out) case DateVal: v, err = sqltypes.NewValue(sqltypes.Date, node.Bytes()) case TimeVal: diff --git a/go/vt/sqlparser/normalizer_test.go b/go/vt/sqlparser/normalizer_test.go index d18cda63e26..8e40dfe9f1a 100644 --- a/go/vt/sqlparser/normalizer_test.go +++ b/go/vt/sqlparser/normalizer_test.go @@ -139,19 +139,47 @@ func TestNormalize(t *testing.T) { outbv: map[string]*querypb.BindVariable{ "foo": sqltypes.HexNumBindVariable([]byte("0x1234")), }, + }, { + // Hex number values are normalized to a consistent case + in: "select * from t where foo = 0xdeadbeef", + outstmt: "select * from t where foo = :foo /* HEXNUM */", + outbv: map[string]*querypb.BindVariable{ + "foo": sqltypes.HexNumBindVariable([]byte("0xDEADBEEF")), + }, + }, { + // Hex number values are normalized to a consistent case + in: "select * from t where foo = 0xDEADBEEF", + outstmt: "select * from t where foo = :foo /* HEXNUM */", + outbv: map[string]*querypb.BindVariable{ + "foo": sqltypes.HexNumBindVariable([]byte("0xDEADBEEF")), + }, }, { // Hex encoded string values should work for selects in: "select * from t where foo = x'7b7d'", outstmt: "select * from t where foo = :foo /* HEXVAL */", outbv: map[string]*querypb.BindVariable{ - "foo": sqltypes.HexValBindVariable([]byte("x'7b7d'")), + "foo": sqltypes.HexValBindVariable([]byte("x'7B7D'")), + }, + }, { + // Hex encoded string are converted to a consistent case + in: "select * from t where foo = x'7b7D'", + outstmt: "select * from t where foo = :foo /* HEXVAL */", + outbv: map[string]*querypb.BindVariable{ + "foo": sqltypes.HexValBindVariable([]byte("x'7B7D'")), + }, + }, { + // Hex encoded string values should work for selects + in: "select * from t where foo = x'7B7D'", + outstmt: "select * from t where foo = :foo /* HEXVAL */", + outbv: map[string]*querypb.BindVariable{ + "foo": sqltypes.HexValBindVariable([]byte("x'7B7D'")), }, }, { // Ensure that hex notation bind vars work with collation based conversions in: "select convert(x'7b7d' using utf8mb4) from dual", outstmt: "select convert(:bv1 /* HEXVAL */ using utf8mb4) from dual", outbv: map[string]*querypb.BindVariable{ - "bv1": sqltypes.HexValBindVariable([]byte("x'7b7d'")), + "bv1": sqltypes.HexValBindVariable([]byte("x'7B7D'")), }, }, { // Hex number values should work for DMLs @@ -165,21 +193,21 @@ func TestNormalize(t *testing.T) { in: "select * from t where foo = b'11'", outstmt: "select * from t where foo = :foo /* HEXNUM */", outbv: map[string]*querypb.BindVariable{ - "foo": sqltypes.HexNumBindVariable([]byte("0x3")), + "foo": sqltypes.HexNumBindVariable([]byte("0x03")), }, }, { // Large bin values work fine in: "select * from t where foo = b'11101010100101010010101010101010101010101000100100100100100101001101010101010101000001'", outstmt: "select * from t where foo = :foo /* HEXNUM */", outbv: map[string]*querypb.BindVariable{ - "foo": sqltypes.HexNumBindVariable([]byte("0x3aa54aaaaaa24925355541")), + "foo": sqltypes.HexNumBindVariable([]byte("0x3AA54AAAAAA24925355541")), }, }, { // Bin value does not convert for DMLs in: "update a set v1 = b'11'", outstmt: "update a set v1 = :v1 /* HEXNUM */", outbv: map[string]*querypb.BindVariable{ - "v1": sqltypes.HexNumBindVariable([]byte("0x3")), + "v1": sqltypes.HexNumBindVariable([]byte("0x03")), }, }, { // ORDER BY column_position @@ -284,10 +312,10 @@ func TestNormalize(t *testing.T) { in: `select b'1', 0b01, b'1010', 0b1111111`, outstmt: `select :bv1 /* HEXNUM */, :bv2 /* HEXNUM */, :bv3 /* HEXNUM */, :bv4 /* HEXNUM */ from dual`, outbv: map[string]*querypb.BindVariable{ - "bv1": sqltypes.HexNumBindVariable([]byte("0x1")), - "bv2": sqltypes.HexNumBindVariable([]byte("0x1")), - "bv3": sqltypes.HexNumBindVariable([]byte("0xa")), - "bv4": sqltypes.HexNumBindVariable([]byte("0x7f")), + "bv1": sqltypes.HexNumBindVariable([]byte("0x01")), + "bv2": sqltypes.HexNumBindVariable([]byte("0x01")), + "bv3": sqltypes.HexNumBindVariable([]byte("0x0A")), + "bv4": sqltypes.HexNumBindVariable([]byte("0x7F")), }, }, { // DateVal should also be normalized diff --git a/go/vt/vtgate/evalengine/compiler_asm.go b/go/vt/vtgate/evalengine/compiler_asm.go index b56801a0dde..74741c7b1cc 100644 --- a/go/vt/vtgate/evalengine/compiler_asm.go +++ b/go/vt/vtgate/evalengine/compiler_asm.go @@ -24,7 +24,7 @@ import ( "crypto/sha256" "crypto/sha512" "encoding/binary" - "encoding/hex" + gohex "encoding/hex" "errors" "hash/crc32" "math" @@ -35,6 +35,8 @@ import ( "github.com/google/uuid" + "vitess.io/vitess/go/mysql/hex" + "vitess.io/vitess/go/mysql/icuregex" "vitess.io/vitess/go/hack" @@ -2047,7 +2049,7 @@ func (asm *assembler) Fn_FROM_BASE64(t sqltypes.Type) { func (asm *assembler) Fn_HEX_c(t sqltypes.Type, col collations.TypedCollation) { asm.emit(func(env *ExpressionEnv) int { arg := env.vm.stack[env.vm.sp-1].(*evalBytes) - encoded := env.vm.arena.newEvalText(hexEncodeBytes(arg.bytes), col) + encoded := env.vm.arena.newEvalText(hex.EncodeBytes(arg.bytes), col) encoded.tt = int16(t) env.vm.stack[env.vm.sp-1] = encoded return 1 @@ -2057,7 +2059,7 @@ func (asm *assembler) Fn_HEX_c(t sqltypes.Type, col collations.TypedCollation) { func (asm *assembler) Fn_HEX_d(col collations.TypedCollation) { asm.emit(func(env *ExpressionEnv) int { arg := env.vm.stack[env.vm.sp-1].(evalNumeric) - env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalText(hexEncodeUint(uint64(arg.toInt64().i)), col) + env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalText(hex.EncodeUint(uint64(arg.toInt64().i)), col) return 1 }, "FN HEX NUMERIC(SP-1)") } @@ -2069,7 +2071,7 @@ func (asm *assembler) Fn_UNHEX_i(tt sqltypes.Type) { env.vm.stack[env.vm.sp-1] = nil return 1 } - env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalRaw(hexDecodeUint(uint64(arg.toInt64().i)), tt, collationBinary) + env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalRaw(hex.DecodeUint(uint64(arg.toInt64().i)), tt, collationBinary) return 1 }, "FN UNHEX INT64(SP-1)") } @@ -2077,7 +2079,7 @@ func (asm *assembler) Fn_UNHEX_i(tt sqltypes.Type) { func (asm *assembler) Fn_UNHEX_u(tt sqltypes.Type) { asm.emit(func(env *ExpressionEnv) int { arg := env.vm.stack[env.vm.sp-1].(*evalUint64) - env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalRaw(hexDecodeUint(uint64(arg.u)), tt, collationBinary) + env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalRaw(hex.DecodeUint(uint64(arg.u)), tt, collationBinary) return 1 }, "FN UNHEX UINT64(SP-1)") } @@ -2090,7 +2092,7 @@ func (asm *assembler) Fn_UNHEX_f(tt sqltypes.Type) { env.vm.stack[env.vm.sp-1] = nil return 1 } - env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalRaw(hexDecodeUint(uint64(arg.f)), tt, collationBinary) + env.vm.stack[env.vm.sp-1] = env.vm.arena.newEvalRaw(hex.DecodeUint(uint64(arg.f)), tt, collationBinary) return 1 }, "FN UNHEX FLOAT64(SP-1)") } @@ -2098,9 +2100,9 @@ func (asm *assembler) Fn_UNHEX_f(tt sqltypes.Type) { func (asm *assembler) Fn_UNHEX_b(tt sqltypes.Type) { asm.emit(func(env *ExpressionEnv) int { arg := env.vm.stack[env.vm.sp-1].(*evalBytes) - decoded := make([]byte, hexDecodedLen(arg.bytes)) + decoded := make([]byte, hex.DecodedLen(arg.bytes)) - ok := hexDecodeBytes(decoded, arg.bytes) + ok := hex.DecodeBytes(decoded, arg.bytes) if !ok { env.vm.stack[env.vm.sp-1] = nil return 1 @@ -3417,8 +3419,8 @@ func (asm *assembler) Fn_MD5(col collations.TypedCollation) { arg := env.vm.stack[env.vm.sp-1].(*evalBytes) sum := md5.Sum(arg.bytes) - buf := make([]byte, hex.EncodedLen(len(sum))) - hex.Encode(buf, sum[:]) + buf := make([]byte, gohex.EncodedLen(len(sum))) + gohex.Encode(buf, sum[:]) arg.tt = int16(sqltypes.VarChar) arg.bytes = buf @@ -3432,8 +3434,8 @@ func (asm *assembler) Fn_SHA1(col collations.TypedCollation) { arg := env.vm.stack[env.vm.sp-1].(*evalBytes) sum := sha1.Sum(arg.bytes) - buf := make([]byte, hex.EncodedLen(len(sum))) - hex.Encode(buf, sum[:]) + buf := make([]byte, gohex.EncodedLen(len(sum))) + gohex.Encode(buf, sum[:]) arg.tt = int16(sqltypes.VarChar) arg.bytes = buf @@ -3467,8 +3469,8 @@ func (asm *assembler) Fn_SHA2(col collations.TypedCollation) { env.vm.sp-- return 1 } - buf := make([]byte, hex.EncodedLen(len(sum))) - hex.Encode(buf, sum[:]) + buf := make([]byte, gohex.EncodedLen(len(sum))) + gohex.Encode(buf, sum[:]) arg.tt = int16(sqltypes.VarChar) arg.bytes = buf diff --git a/go/vt/vtgate/evalengine/fn_hex.go b/go/vt/vtgate/evalengine/fn_hex.go index 5bc73cacff4..3aa395f74cc 100644 --- a/go/vt/vtgate/evalengine/fn_hex.go +++ b/go/vt/vtgate/evalengine/fn_hex.go @@ -17,10 +17,8 @@ limitations under the License. package evalengine import ( - "encoding/hex" - "math/bits" - "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/mysql/hex" "vitess.io/vitess/go/mysql/json" "vitess.io/vitess/go/sqltypes" querypb "vitess.io/vitess/go/vt/proto/query" @@ -45,11 +43,11 @@ func (call *builtinHex) eval(env *ExpressionEnv) (eval, error) { var encoded []byte switch arg := arg.(type) { case *evalBytes: - encoded = hexEncodeBytes(arg.bytes) + encoded = hex.EncodeBytes(arg.bytes) case evalNumeric: - encoded = hexEncodeUint(uint64(arg.toInt64().i)) + encoded = hex.EncodeUint(uint64(arg.toInt64().i)) default: - encoded = hexEncodeBytes(arg.ToRawBytes()) + encoded = hex.EncodeBytes(arg.ToRawBytes()) } if arg.SQLType() == sqltypes.Blob || arg.SQLType() == sqltypes.TypeJSON { return newEvalRaw(sqltypes.Text, encoded, defaultCoercionCollation(call.collate)), nil @@ -93,64 +91,6 @@ func (call *builtinHex) compile(c *compiler) (ctype, error) { return ctype{Type: t, Col: col}, nil } -const hextable = "0123456789ABCDEF" - -func hexEncodeBytes(src []byte) []byte { - j := 0 - dst := make([]byte, len(src)*2) - for _, v := range src { - dst[j] = hextable[v>>4] - dst[j+1] = hextable[v&0x0f] - j += 2 - } - return dst -} - -func hexEncodeUint(u uint64) []byte { - var a [16 + 1]byte - i := len(a) - shift := uint(bits.TrailingZeros(uint(16))) & 7 - b := uint64(16) - m := uint(16) - 1 // == 1<= b { - i-- - a[i] = hextable[uint(u)&m] - u >>= shift - } - - // u < base - i-- - a[i] = hextable[uint(u)] - return a[i:] -} - -func hexDecodeUint(u uint64) []byte { - if u == 0 { - return []byte{0} - } - var decoded []byte - for u > 0 { - c1 := u % 10 - c2 := u % 100 / 10 - decoded = append([]byte{byte(c1 + c2<<4)}, decoded...) - u /= 100 - } - return decoded -} - -func hexDecodedLen(src []byte) int { - return (len(src) + 1) / 2 -} - -func hexDecodeBytes(dst, src []byte) bool { - if len(src)&1 == 1 { - src = append([]byte{'0'}, src...) - } - _, err := hex.Decode(dst, src) - return err == nil -} - type builtinUnhex struct { CallExpr } @@ -162,14 +102,14 @@ func hexDecodeJSON(j *evalJSON) ([]byte, bool) { case json.TypeNumber: u, ok := j.Uint64() if ok { - return hexDecodeUint(u), true + return hex.DecodeUint(u), true } else { return nil, false } default: b := j.ToRawBytes() - decoded := make([]byte, hexDecodedLen(b)) - ok := hexDecodeBytes(decoded, b) + decoded := make([]byte, hex.DecodedLen(b)) + ok := hex.DecodeBytes(decoded, b) if !ok { return nil, false } @@ -189,8 +129,8 @@ func (call *builtinUnhex) eval(env *ExpressionEnv) (eval, error) { var decoded []byte switch arg := arg.(type) { case *evalBytes: - decoded = make([]byte, hexDecodedLen(arg.bytes)) - ok := hexDecodeBytes(decoded, arg.bytes) + decoded = make([]byte, hex.DecodedLen(arg.bytes)) + ok := hex.DecodeBytes(decoded, arg.bytes) if !ok { return nil, nil } @@ -198,13 +138,13 @@ func (call *builtinUnhex) eval(env *ExpressionEnv) (eval, error) { if arg.i < 0 { return nil, nil } - decoded = hexDecodeUint(uint64(arg.i)) + decoded = hex.DecodeUint(uint64(arg.i)) case *evalUint64: - decoded = hexDecodeUint(arg.u) + decoded = hex.DecodeUint(arg.u) case *evalDecimal: b := arg.ToRawBytes() - decoded = make([]byte, hexDecodedLen(b)) - ok := hexDecodeBytes(decoded, b) + decoded = make([]byte, hex.DecodedLen(b)) + ok := hex.DecodeBytes(decoded, b) if !ok { return nil, nil } @@ -213,7 +153,7 @@ func (call *builtinUnhex) eval(env *ExpressionEnv) (eval, error) { if f != float64(int64(f)) { return nil, nil } - decoded = hexDecodeUint(uint64(arg.f)) + decoded = hex.DecodeUint(uint64(arg.f)) case *evalJSON: var ok bool decoded, ok = hexDecodeJSON(arg) @@ -222,8 +162,8 @@ func (call *builtinUnhex) eval(env *ExpressionEnv) (eval, error) { } default: b := evalToBinary(arg) - decoded = make([]byte, hexDecodedLen(b.bytes)) - ok := hexDecodeBytes(decoded, b.bytes) + decoded = make([]byte, hex.DecodedLen(b.bytes)) + ok := hex.DecodeBytes(decoded, b.bytes) if !ok { return nil, nil } From d2c421f0054fac37056322030de02c99761864a4 Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Sat, 15 Jul 2023 17:48:24 +0200 Subject: [PATCH 04/13] Fix tests Signed-off-by: Dirkjan Bussink --- go/mysql/binlog/binlog_json_test.go | 4 +- go/vt/vtgate/evalengine/cached_size.go | 12 ++++ go/vt/vttablet/endtoend/vstreamer_test.go | 2 +- .../vreplication/vcopier_test.go | 55 +++++++++---------- 4 files changed, 42 insertions(+), 31 deletions(-) diff --git a/go/mysql/binlog/binlog_json_test.go b/go/mysql/binlog/binlog_json_test.go index 7e72c43cc99..f6d4fe7fcf2 100644 --- a/go/mysql/binlog/binlog_json_test.go +++ b/go/mysql/binlog/binlog_json_test.go @@ -450,12 +450,12 @@ func TestMarshalJSONToSQL(t *testing.T) { { name: `opaque string [2 202 254]`, data: []byte{15, 15, 2, 202, 254}, - expected: `CAST(x'02cafe' as JSON)`, + expected: `CAST(x'02CAFE' as JSON)`, }, { name: `opaque blob [2 202 254]`, data: []byte{15, 252, 2, 202, 254}, - expected: `CAST(x'02cafe' as JSON)`, + expected: `CAST(x'02CAFE' as JSON)`, }, } for _, tc := range testcases { diff --git a/go/vt/vtgate/evalengine/cached_size.go b/go/vt/vtgate/evalengine/cached_size.go index ea525e46a25..9955fc85c09 100644 --- a/go/vt/vtgate/evalengine/cached_size.go +++ b/go/vt/vtgate/evalengine/cached_size.go @@ -245,6 +245,18 @@ func (cached *IntervalExpr) CachedSize(alloc bool) int64 { size += cached.CallExpr.CachedSize(false) return size } +func (cached *IntroducerExpr) CachedSize(alloc bool) int64 { + if cached == nil { + return int64(0) + } + size := int64(0) + if alloc { + size += int64(24) + } + // field UnaryExpr vitess.io/vitess/go/vt/vtgate/evalengine.UnaryExpr + size += cached.UnaryExpr.CachedSize(false) + return size +} func (cached *IsExpr) CachedSize(alloc bool) int64 { if cached == nil { return int64(0) diff --git a/go/vt/vttablet/endtoend/vstreamer_test.go b/go/vt/vttablet/endtoend/vstreamer_test.go index 673f5801e45..75095588d39 100644 --- a/go/vt/vttablet/endtoend/vstreamer_test.go +++ b/go/vt/vttablet/endtoend/vstreamer_test.go @@ -310,7 +310,7 @@ func TestSchemaVersioning(t *testing.T) { `version`, `gtid`, /*at this point we only have latest schema so we have types (int32, int32, varbinary, varbinary) so the types don't match. Hence the @ fieldnames*/ - `type:FIELD field_event:{table_name:"vitess_version" fields:{name:"@1" type:INT32} fields:{name:"@2" type:INT32} fields:{name:"@3" type:INT32}}`, + `type:FIELD field_event:{table_name:"vitess_version" fields:{name:"@1" type:INT32 charset:63} fields:{name:"@2" type:INT32 charset:63} fields:{name:"@3" type:INT32 charset:63}}`, `type:ROW row_event:{table_name:"vitess_version" row_changes:{after:{lengths:1 lengths:2 lengths:3 values:"220200"}}}`, `gtid`, `gtid`, diff --git a/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go b/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go index 6c000add85d..e1b1686b0d7 100644 --- a/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go +++ b/go/vt/vttablet/tabletmanager/vreplication/vcopier_test.go @@ -17,6 +17,7 @@ limitations under the License. package vreplication import ( + "context" "fmt" "os" "strings" @@ -27,8 +28,6 @@ import ( "vitess.io/vitess/go/vt/vttablet" - "context" - "vitess.io/vitess/go/vt/log" "github.com/stretchr/testify/require" @@ -177,10 +176,10 @@ func testPlayerCopyCharPK(t *testing.T) { "/insert into _vt.copy_state", "/update _vt.vreplication set state='Copying'", "insert into dst(idc,val) values ('a\\0',1)", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"idc\\" type:BINARY} rows:{lengths:2 values:\\"a\\\\x00\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"idc\\" type:BINARY charset:63 flags:20611} rows:{lengths:2 values:\\"a\\\\x00\\"}'.*`, `update dst set val=3 where idc='a\0' and ('a\0') <= ('a\0')`, "insert into dst(idc,val) values ('c\\0',2)", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"idc\\" type:BINARY} rows:{lengths:2 values:\\"c\\\\x00\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"idc\\" type:BINARY charset:63 flags:20611} rows:{lengths:2 values:\\"c\\\\x00\\"}'.*`, "/delete cs, pca from _vt.copy_state as cs left join _vt.post_copy_action as pca on cs.vrepl_id=pca.vrepl_id and cs.table_name=pca.table_name.*dst", "/update _vt.vreplication set state='Running", )) @@ -285,7 +284,7 @@ func testPlayerCopyVarcharPKCaseInsensitive(t *testing.T) { "/update _vt.vreplication set state='Copying'", // Copy mode. "insert into dst(idc,val) values ('a',1)", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"idc\\" type:VARCHAR} rows:{lengths:1 values:\\"a\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"idc\\" type:VARCHAR charset:45 flags:20483} rows:{lengths:1 values:\\"a\\"}'.*`, // Copy-catchup mode. `/insert into dst\(idc,val\) select 'B', 3 from dual where \( .* 'B' COLLATE .* \) <= \( .* 'a' COLLATE .* \)`, ).Then(func(expect qh.ExpectationSequencer) qh.ExpectationSequencer { @@ -295,11 +294,11 @@ func testPlayerCopyVarcharPKCaseInsensitive(t *testing.T) { //upd1 := expect. upd1 := expect.Then(qh.Eventually( "insert into dst(idc,val) values ('B',3)", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"idc\\" type:VARCHAR} rows:{lengths:1 values:\\"B\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"idc\\" type:VARCHAR charset:45 flags:20483} rows:{lengths:1 values:\\"B\\"}'.*`, )) upd2 := expect.Then(qh.Eventually( "insert into dst(idc,val) values ('c',2)", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"idc\\" type:VARCHAR} rows:{lengths:1 values:\\"c\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"idc\\" type:VARCHAR charset:45 flags:20483} rows:{lengths:1 values:\\"c\\"}'.*`, )) upd1.Then(upd2.Eventually()) return upd2 @@ -408,12 +407,12 @@ func testPlayerCopyVarcharCompositePKCaseSensitiveCollation(t *testing.T) { "/update _vt.vreplication set state='Copying'", // Copy mode. "insert into dst(id,idc,idc2,val) values (1,'a','a',1)", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} fields:{name:\\"idc\\" type:VARBINARY} fields:{name:\\"idc2\\" type:VARBINARY} rows:{lengths:1 lengths:1 lengths:1 values:\\"1aa\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} fields:{name:\\"idc\\" type:VARBINARY charset:63 flags:20611} fields:{name:\\"idc2\\" type:VARBINARY charset:63 flags:20611} rows:{lengths:1 lengths:1 lengths:1 values:\\"1aa\\"}'.*`, // Copy-catchup mode. `insert into dst(id,idc,idc2,val) select 1, 'B', 'B', 3 from dual where (1,'B','B') <= (1,'a','a')`, // Copy mode. "insert into dst(id,idc,idc2,val) values (1,'c','c',2)", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} fields:{name:\\"idc\\" type:VARBINARY} fields:{name:\\"idc2\\" type:VARBINARY} rows:{lengths:1 lengths:1 lengths:1 values:\\"1cc\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} fields:{name:\\"idc\\" type:VARBINARY charset:63 flags:20611} fields:{name:\\"idc2\\" type:VARBINARY charset:63 flags:20611} rows:{lengths:1 lengths:1 lengths:1 values:\\"1cc\\"}'.*`, // Wrap-up. "/delete cs, pca from _vt.copy_state as cs left join _vt.post_copy_action as pca on cs.vrepl_id=pca.vrepl_id and cs.table_name=pca.table_name.*dst", "/update _vt.vreplication set state='Running'", @@ -498,7 +497,7 @@ func testPlayerCopyTablesWithFK(t *testing.T) { // Inserts may happen out-of-order. Update happen in-order. "begin", "insert into dst1(id,id2) values (1,1), (2,2)", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"2\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"2\\"}'.*`, "commit", )).Then(qh.Immediately( "set foreign_key_checks=0;", @@ -519,7 +518,7 @@ func testPlayerCopyTablesWithFK(t *testing.T) { // copy dst2 "begin", "insert into dst2(id,id2) values (1,21), (2,22)", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"2\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"2\\"}'.*`, "commit", )).Then(qh.Immediately( "set foreign_key_checks=0;", @@ -619,7 +618,7 @@ func testPlayerCopyTables(t *testing.T) { "/update _vt.vreplication set pos=", "begin", "insert into dst1(id,val,val2,d,j) values (1,'aaa','aaa',0,JSON_ARRAY(123456789012345678901234567890, _utf8mb4'abcd')), (2,'bbb','bbb',1,JSON_OBJECT(_utf8mb4'foo', _utf8mb4'bar'))", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"2\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"2\\"}'.*`, "commit", // copy of dst1 is done: delete from copy_state. "/delete cs, pca from _vt.copy_state as cs left join _vt.post_copy_action as pca on cs.vrepl_id=pca.vrepl_id and cs.table_name=pca.table_name.*dst1", @@ -753,7 +752,7 @@ func testPlayerCopyBigTable(t *testing.T) { // The first fast-forward has no starting point. So, it just saves the current position. "/update _vt.vreplication set state='Copying'", "insert into dst(id,val) values (1,'aaa')", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"1\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"1\\"}'.*`, // The next catchup executes the new row insert, but will be a no-op. "insert into dst(id,val) select 3, 'ccc' from dual where (3) <= (1)", // fastForward has nothing to add. Just saves position. @@ -763,12 +762,12 @@ func testPlayerCopyBigTable(t *testing.T) { ).Then(func(expect qh.ExpectationSequencer) qh.ExpectationSequencer { ins1 := expect.Then(qh.Eventually("insert into dst(id,val) values (2,'bbb')")) upd1 := ins1.Then(qh.Eventually( - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"2\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"2\\"}'.*`, )) // Third row copied without going back to catchup state. ins3 := expect.Then(qh.Eventually("insert into dst(id,val) values (3,'ccc')")) upd3 := ins3.Then(qh.Eventually( - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"3\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"3\\"}'.*`, )) upd1.Then(upd3.Eventually()) return upd3 @@ -883,7 +882,7 @@ func testPlayerCopyWildcardRule(t *testing.T) { "/update _vt.vreplication set state='Copying'", // The first fast-forward has no starting point. So, it just saves the current position. "insert into src(id,val) values (1,'aaa')", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"1\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"1\\"}'.*`, // The next catchup executes the new row insert, but will be a no-op. "insert into src(id,val) select 3, 'ccc' from dual where (3) <= (1)", // fastForward has nothing to add. Just saves position. @@ -893,12 +892,12 @@ func testPlayerCopyWildcardRule(t *testing.T) { ).Then(func(expect qh.ExpectationSequencer) qh.ExpectationSequencer { ins1 := expect.Then(qh.Eventually("insert into src(id,val) values (2,'bbb')")) upd1 := ins1.Then(qh.Eventually( - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"2\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"2\\"}'.*`, )) // Third row copied without going back to catchup state. ins3 := expect.Then(qh.Eventually("insert into src(id,val) values (3,'ccc')")) upd3 := ins3.Then(qh.Eventually( - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"3\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"3\\"}'.*`, )) upd1.Then(upd3.Eventually()) return upd3 @@ -1058,13 +1057,13 @@ func testPlayerCopyTableContinuation(t *testing.T) { ).Then(qh.Immediately( "insert into dst1(id,val) values (7,'insert out'), (8,'no change'), (10,'updated'), (12,'move out')", )).Then(qh.Eventually( - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id1\\" type:INT32} fields:{name:\\"id2\\" type:INT32} rows:{lengths:2 lengths:1 values:\\"126\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id1\\" type:INT32 charset:63 flags:53251} fields:{name:\\"id2\\" type:INT32 charset:63 flags:53251} rows:{lengths:2 lengths:1 values:\\"126\\"}'.*`, )).Then(qh.Immediately( "/delete cs, pca from _vt.copy_state as cs left join _vt.post_copy_action as pca on cs.vrepl_id=pca.vrepl_id and cs.table_name=pca.table_name.*dst1", "insert into not_copied(id,val) values (1,'bbb')", )).Then(qh.Eventually( // Copy again. There should be no events for catchup. - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\\"id\\\" type:INT32} rows:{lengths:1 values:\\\"1\\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\\"id\\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\\"1\\\"}'.*`, )).Then(qh.Immediately( "/delete cs, pca from _vt.copy_state as cs left join _vt.post_copy_action as pca on cs.vrepl_id=pca.vrepl_id and cs.table_name=pca.table_name.*not_copied", "/update _vt.vreplication set state='Running'", @@ -1383,7 +1382,7 @@ func testPlayerCopyTablesStopAfterCopy(t *testing.T) { ).Then(qh.Eventually( "begin", "insert into dst1(id,val) values (1,'aaa'), (2,'bbb')", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"2\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"2\\"}'.*`, "commit", )).Then(qh.Immediately( // copy of dst1 is done: delete from copy_state. @@ -1472,7 +1471,7 @@ func testPlayerCopyTablesGIPK(t *testing.T) { ).Then(qh.Eventually( "begin", "insert into dst1(my_row_id,val) values (1,'aaa'), (2,'bbb')", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"my_row_id\\" type:UINT64} rows:{lengths:1 values:\\"2\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"my_row_id\\" type:UINT64 charset:63 flags:49699} rows:{lengths:1 values:\\"2\\"}'.*`, "commit", )).Then(qh.Immediately( // copy of dst1 is done: delete from copy_state. @@ -1483,7 +1482,7 @@ func testPlayerCopyTablesGIPK(t *testing.T) { "commit", "begin", "insert into dst2(my_row_id,val) values (1,'aaa'), (2,'bbb')", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"my_row_id\\" type:UINT64} rows:{lengths:1 values:\\"2\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"my_row_id\\" type:UINT64 charset:63 flags:49699} rows:{lengths:1 values:\\"2\\"}'.*`, "commit", )).Then(qh.Immediately( // copy of dst2 is done: delete from copy_state. @@ -1573,7 +1572,7 @@ func testPlayerCopyTableCancel(t *testing.T) { ).Then(qh.Eventually( "begin", "insert into dst1(id,val) values (1,'aaa'), (2,'bbb')", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"2\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"2\\"}'.*`, "commit", )).Then(qh.Immediately( // copy of dst1 is done: delete from copy_state. @@ -1647,11 +1646,11 @@ func testPlayerCopyTablesWithGeneratedColumn(t *testing.T) { "/update _vt.vreplication set state", // The first fast-forward has no starting point. So, it just saves the current position. "insert into dst1(id,val,val3,id2) values (1,'aaa','aaa1',10), (2,'bbb','bbb2',20)", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"2\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"2\\"}'.*`, // copy of dst1 is done: delete from copy_state. "/delete cs, pca from _vt.copy_state as cs left join _vt.post_copy_action as pca on cs.vrepl_id=pca.vrepl_id and cs.table_name=pca.table_name.*dst1", "insert into dst2(val3,val,id2) values ('aaa1','aaa',10), ('bbb2','bbb',20)", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"2\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"2\\"}'.*`, // copy of dst2 is done: delete from copy_state. "/delete cs, pca from _vt.copy_state as cs left join _vt.post_copy_action as pca on cs.vrepl_id=pca.vrepl_id and cs.table_name=pca.table_name.*dst2", "/update _vt.vreplication set state", @@ -1725,7 +1724,7 @@ func testCopyTablesWithInvalidDates(t *testing.T) { ).Then(qh.Eventually( "begin", "insert into dst1(id,dt) values (1,'2020-01-12'), (2,'0000-00-00')", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} rows:{lengths:1 values:\\"2\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 values:\\"2\\"}'.*`, "commit", )).Then(qh.Immediately( // copy of dst1 is done: delete from copy_state. @@ -1815,7 +1814,7 @@ func testCopyInvisibleColumns(t *testing.T) { "/update _vt.vreplication set state='Copying'", // The first fast-forward has no starting point. So, it just saves the current position. "insert into dst1(id,id2,inv1,inv2) values (1,10,100,1000), (2,20,200,2000)", - `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32} fields:{name:\\"inv1\\" type:INT32} rows:{lengths:1 lengths:3 values:\\"2200\\"}'.*`, + `/insert into _vt.copy_state \(lastpk, vrepl_id, table_name\) values \('fields:{name:\\"id\\" type:INT32 charset:63 flags:53251} fields:{name:\\"inv1\\" type:INT32 charset:63 flags:53251} rows:{lengths:1 lengths:3 values:\\"2200\\"}'.*`, // copy of dst1 is done: delete from copy_state. "/delete cs, pca from _vt.copy_state as cs left join _vt.post_copy_action as pca on cs.vrepl_id=pca.vrepl_id and cs.table_name=pca.table_name.*dst1", "/update _vt.vreplication set state='Running'", From 41ab596c9c44d456f9a9a0edd3f81c65316b2b71 Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Sun, 16 Jul 2023 17:26:43 +0200 Subject: [PATCH 05/13] Use explicit types and collations for columns and bindvars This way we can ensure we always initialize these with the proper explicit typing information. Signed-off-by: Dirkjan Bussink --- go/mysql/collations/local.go | 6 +- .../without_schematracker/orderby_test.go | 2 +- go/vt/vtgate/engine/delete_test.go | 5 +- go/vt/vtgate/engine/limit_test.go | 12 +- go/vt/vtgate/engine/memory_sort_test.go | 14 +- go/vt/vtgate/engine/set_test.go | 3 +- go/vt/vtgate/engine/update_test.go | 3 +- go/vt/vtgate/evalengine/api_compare_test.go | 204 +++++++++--------- go/vt/vtgate/evalengine/api_literal.go | 36 ++-- go/vt/vtgate/evalengine/expr_bvar.go | 12 +- go/vt/vtgate/evalengine/expr_column.go | 2 +- go/vt/vtgate/evalengine/expr_column_test.go | 2 +- go/vt/vtgate/evalengine/translate.go | 35 +-- go/vt/vtgate/evalengine/translate_test.go | 14 +- .../planbuilder/expression_converter.go | 2 +- .../planbuilder/expression_converter_test.go | 4 +- .../planbuilder/operator_transformers.go | 10 +- .../operators/info_schema_planning.go | 17 +- .../planbuilder/operators/sharded_routing.go | 5 +- go/vt/vtgate/planbuilder/postprocess.go | 4 +- .../planbuilder/predicate_rewrite_test.go | 2 + go/vt/vtgate/planbuilder/projection.go | 1 + go/vt/vtgate/planbuilder/route.go | 3 +- go/vt/vtgate/planbuilder/system_tables.go | 6 +- go/vt/vtgate/planbuilder/vindex_op.go | 3 +- go/vt/vtgate/semantics/real_table.go | 8 +- go/vt/vtgate/semantics/typer.go | 6 +- 27 files changed, 240 insertions(+), 181 deletions(-) diff --git a/go/mysql/collations/local.go b/go/mysql/collations/local.go index 15adc1a9e05..4bbe9a35a9c 100644 --- a/go/mysql/collations/local.go +++ b/go/mysql/collations/local.go @@ -49,10 +49,10 @@ func Default() ID { } func DefaultCollationForType(t sqltypes.Type) ID { - switch t { - case sqltypes.VarChar, sqltypes.Char, sqltypes.Text: + switch { + case sqltypes.IsText(t): return Default() - case sqltypes.TypeJSON: + case t == sqltypes.TypeJSON: return CollationUtf8mb4ID default: return CollationBinaryID diff --git a/go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go b/go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go index 77a2ceeb519..a20c7ad54c6 100644 --- a/go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go +++ b/go/test/endtoend/vtgate/queries/orderby/without_schematracker/orderby_test.go @@ -54,5 +54,5 @@ func TestSimpleOrderBy(t *testing.T) { mcmp.Exec("insert into user(id, name) values (0,'Apa'),(1,'Banan'),(3,'Ceasar'),(4,'David')") mcmp.AssertMatches(`SELECT name FROM user WHERE id in (0,4) ORDER BY name ASC`, - `[[VARCHAR("Apa")] [VARCHAR("David)]]`) + `[[VARCHAR("Apa")] [VARCHAR("David")]]`) } diff --git a/go/vt/vtgate/engine/delete_test.go b/go/vt/vtgate/engine/delete_test.go index 47ecf8306af..10f5bcb3449 100644 --- a/go/vt/vtgate/engine/delete_test.go +++ b/go/vt/vtgate/engine/delete_test.go @@ -21,6 +21,7 @@ import ( "errors" "testing" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/vtgate/evalengine" "github.com/stretchr/testify/require" @@ -89,7 +90,7 @@ func TestDeleteEqual(t *testing.T) { }) // Failure case - expr := evalengine.NewBindVar("aa") + expr := evalengine.NewBindVar("aa", -1, collations.CollationBinaryID) del.Values = []evalengine.Expr{expr} _, err = del.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false) require.EqualError(t, err, "query arguments missing for aa") @@ -121,7 +122,7 @@ func TestDeleteEqualMultiCol(t *testing.T) { }) // Failure case - expr := evalengine.NewBindVar("aa") + expr := evalengine.NewBindVar("aa", -1, collations.CollationBinaryID) del.Values = []evalengine.Expr{expr} _, err = del.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false) require.EqualError(t, err, "query arguments missing for aa") diff --git a/go/vt/vtgate/engine/limit_test.go b/go/vt/vtgate/engine/limit_test.go index 8101c64d3a8..ba15306685a 100644 --- a/go/vt/vtgate/engine/limit_test.go +++ b/go/vt/vtgate/engine/limit_test.go @@ -23,6 +23,8 @@ import ( "github.com/stretchr/testify/assert" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/vt/vtgate/evalengine" "github.com/stretchr/testify/require" @@ -128,7 +130,7 @@ func TestLimitExecute(t *testing.T) { results: []*sqltypes.Result{inputResult}, } l = &Limit{ - Count: evalengine.NewBindVar("l"), + Count: evalengine.NewBindVar("l", sqltypes.Int64, collations.CollationBinaryID), Input: fp, } @@ -341,8 +343,8 @@ func TestLimitOffsetExecute(t *testing.T) { } l = &Limit{ - Count: evalengine.NewBindVar("l"), - Offset: evalengine.NewBindVar("o"), + Count: evalengine.NewBindVar("l", sqltypes.Int64, collations.CollationBinaryID), + Offset: evalengine.NewBindVar("o", sqltypes.Int64, collations.CollationBinaryID), Input: fp, } result, err = l.TryExecute(context.Background(), &noopVCursor{}, map[string]*querypb.BindVariable{"l": sqltypes.Int64BindVariable(1), "o": sqltypes.Int64BindVariable(1)}, false) @@ -394,7 +396,7 @@ func TestLimitStreamExecute(t *testing.T) { // Test with bind vars. fp.rewind() - l.Count = evalengine.NewBindVar("l") + l.Count = evalengine.NewBindVar("l", sqltypes.Int64, collations.CollationBinaryID) results = nil err = l.TryStreamExecute(context.Background(), &noopVCursor{}, map[string]*querypb.BindVariable{"l": sqltypes.Int64BindVariable(2)}, true, func(qr *sqltypes.Result) error { results = append(results, qr) @@ -538,7 +540,7 @@ func TestLimitInputFail(t *testing.T) { func TestLimitInvalidCount(t *testing.T) { l := &Limit{ - Count: evalengine.NewBindVar("l"), + Count: evalengine.NewBindVar("l", sqltypes.Int64, collations.CollationBinaryID), } _, _, err := l.getCountAndOffset(context.Background(), &noopVCursor{}, nil) assert.EqualError(t, err, "query arguments missing for l") diff --git a/go/vt/vtgate/engine/memory_sort_test.go b/go/vt/vtgate/engine/memory_sort_test.go index bb06291ad6d..9028f9188f9 100644 --- a/go/vt/vtgate/engine/memory_sort_test.go +++ b/go/vt/vtgate/engine/memory_sort_test.go @@ -77,7 +77,7 @@ func TestMemorySortExecute(t *testing.T) { utils.MustMatch(t, wantResult, result) fp.rewind() - ms.UpperLimit = evalengine.NewBindVar("__upper_limit") + ms.UpperLimit = evalengine.NewBindVar("__upper_limit", sqltypes.Int64, collations.CollationBinaryID) bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} result, err = ms.TryExecute(context.Background(), &noopVCursor{}, bv, false) @@ -138,7 +138,7 @@ func TestMemorySortStreamExecuteWeightString(t *testing.T) { t.Run("Limit test", func(t *testing.T) { fp.rewind() - ms.UpperLimit = evalengine.NewBindVar("__upper_limit") + ms.UpperLimit = evalengine.NewBindVar("__upper_limit", sqltypes.Int64, collations.CollationBinaryID) bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} results = nil @@ -196,7 +196,7 @@ func TestMemorySortExecuteWeightString(t *testing.T) { utils.MustMatch(t, wantResult, result) fp.rewind() - ms.UpperLimit = evalengine.NewBindVar("__upper_limit") + ms.UpperLimit = evalengine.NewBindVar("__upper_limit", sqltypes.Int64, collations.CollationBinaryID) bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} result, err = ms.TryExecute(context.Background(), &noopVCursor{}, bv, false) @@ -279,7 +279,7 @@ func TestMemorySortStreamExecuteCollation(t *testing.T) { t.Run("Limit test", func(t *testing.T) { fp.rewind() - ms.UpperLimit = evalengine.NewBindVar("__upper_limit") + ms.UpperLimit = evalengine.NewBindVar("__upper_limit", sqltypes.Int64, collations.CollationBinaryID) bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} results = nil @@ -338,7 +338,7 @@ func TestMemorySortExecuteCollation(t *testing.T) { utils.MustMatch(t, wantResult, result) fp.rewind() - ms.UpperLimit = evalengine.NewBindVar("__upper_limit") + ms.UpperLimit = evalengine.NewBindVar("__upper_limit", sqltypes.Int64, collations.CollationBinaryID) bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} result, err = ms.TryExecute(context.Background(), &noopVCursor{}, bv, false) @@ -395,7 +395,7 @@ func TestMemorySortStreamExecute(t *testing.T) { utils.MustMatch(t, wantResults, results) fp.rewind() - ms.UpperLimit = evalengine.NewBindVar("__upper_limit") + ms.UpperLimit = evalengine.NewBindVar("__upper_limit", sqltypes.Int64, collations.CollationBinaryID) bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} results = nil @@ -554,7 +554,7 @@ func TestMemorySortMultiColumn(t *testing.T) { utils.MustMatch(t, wantResult, result) fp.rewind() - ms.UpperLimit = evalengine.NewBindVar("__upper_limit") + ms.UpperLimit = evalengine.NewBindVar("__upper_limit", sqltypes.Int64, collations.CollationBinaryID) bv := map[string]*querypb.BindVariable{"__upper_limit": sqltypes.Int64BindVariable(3)} result, err = ms.TryExecute(context.Background(), &noopVCursor{}, bv, false) diff --git a/go/vt/vtgate/engine/set_test.go b/go/vt/vtgate/engine/set_test.go index eb37473eb61..a8e8aba2043 100644 --- a/go/vt/vtgate/engine/set_test.go +++ b/go/vt/vtgate/engine/set_test.go @@ -22,6 +22,7 @@ import ( "fmt" "testing" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/srvtopo" @@ -107,7 +108,7 @@ func TestSetTable(t *testing.T) { setOps: []SetOp{ &UserDefinedVariable{ Name: "x", - Expr: evalengine.NewColumn(0), + Expr: evalengine.NewColumn(0, -1, collations.CollationBinaryID), }, }, qr: []*sqltypes.Result{sqltypes.MakeTestResult( diff --git a/go/vt/vtgate/engine/update_test.go b/go/vt/vtgate/engine/update_test.go index d4fab943a00..59adebfa1e6 100644 --- a/go/vt/vtgate/engine/update_test.go +++ b/go/vt/vtgate/engine/update_test.go @@ -21,6 +21,7 @@ import ( "errors" "testing" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/vtgate/evalengine" topodatapb "vitess.io/vitess/go/vt/proto/topodata" @@ -93,7 +94,7 @@ func TestUpdateEqual(t *testing.T) { }) // Failure case - upd.Values = []evalengine.Expr{evalengine.NewBindVar("aa")} + upd.Values = []evalengine.Expr{evalengine.NewBindVar("aa", -1, collations.CollationBinaryID)} _, err = upd.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false) require.EqualError(t, err, `query arguments missing for aa`) } diff --git a/go/vt/vtgate/evalengine/api_compare_test.go b/go/vt/vtgate/evalengine/api_compare_test.go index 10213694788..603f3ebe676 100644 --- a/go/vt/vtgate/evalengine/api_compare_test.go +++ b/go/vt/vtgate/evalengine/api_compare_test.go @@ -107,7 +107,7 @@ func TestCompareIntegers(t *testing.T) { tests := []testCase{ { name: "integers are equal (1)", - v1: NewColumn(0), v2: NewColumn(0), + v1: NewColumn(0, sqltypes.Int64, collations.CollationBinaryID), v2: NewColumn(0, sqltypes.Int64, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewInt64(18)}, }, @@ -128,25 +128,25 @@ func TestCompareIntegers(t *testing.T) { }, { name: "integers are not equal (3)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Int64, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Int64, collations.CollationBinaryID), out: &F, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewInt64(18), sqltypes.NewInt64(98)}, }, { name: "unsigned integers are equal", - v1: NewColumn(0), v2: NewColumn(0), + v1: NewColumn(0, sqltypes.Uint64, collations.CollationBinaryID), v2: NewColumn(0, sqltypes.Uint64, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewUint64(18)}, }, { name: "unsigned integer and integer are equal", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Uint64, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Int64, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewUint64(18), sqltypes.NewInt64(18)}, }, { name: "unsigned integer and integer are not equal", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Uint64, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Int64, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewUint64(18), sqltypes.NewInt64(42)}, }, @@ -204,7 +204,7 @@ func TestCompareFloats(t *testing.T) { tests := []testCase{ { name: "floats are equal (1)", - v1: NewColumn(0), v2: NewColumn(0), + v1: NewColumn(0, sqltypes.Float64, collations.CollationBinaryID), v2: NewColumn(0, sqltypes.Float64, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewFloat64(18)}, }, @@ -225,7 +225,7 @@ func TestCompareFloats(t *testing.T) { }, { name: "floats are not equal (3)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Float64, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Float64, collations.CollationBinaryID), out: &F, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewFloat64(16516.84), sqltypes.NewFloat64(219541.01)}, }, @@ -283,37 +283,37 @@ func TestCompareDecimals(t *testing.T) { tests := []testCase{ { name: "decimals are equal", - v1: NewColumn(0), v2: NewColumn(0), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewDecimal("12.9019")}, }, { name: "decimals are not equal", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Decimal, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewDecimal("12.9019"), sqltypes.NewDecimal("489.156849")}, }, { name: "decimal is greater than decimal", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Decimal, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewDecimal("192.129"), sqltypes.NewDecimal("192.128")}, }, { name: "decimal is not greater than decimal", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Decimal, collations.CollationBinaryID), out: &F, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewDecimal("192.128"), sqltypes.NewDecimal("192.129")}, }, { name: "decimal is less than decimal", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Decimal, collations.CollationBinaryID), out: &T, op: sqlparser.LessThanOp, row: []sqltypes.Value{sqltypes.NewDecimal("192.128"), sqltypes.NewDecimal("192.129")}, }, { name: "decimal is not less than decimal", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Decimal, collations.CollationBinaryID), out: &F, op: sqlparser.LessThanOp, row: []sqltypes.Value{sqltypes.NewDecimal("192.129"), sqltypes.NewDecimal("192.128")}, }, @@ -331,151 +331,151 @@ func TestCompareNumerics(t *testing.T) { tests := []testCase{ { name: "decimal and float are equal", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Float64, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Decimal, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewFloat64(189.6), sqltypes.NewDecimal("189.6")}, }, { name: "decimal and float with negative values are equal", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Float64, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Decimal, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewFloat64(-98.1839), sqltypes.NewDecimal("-98.1839")}, }, { name: "decimal and float with negative values are not equal (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Float64, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Decimal, collations.CollationBinaryID), out: &F, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewFloat64(-98.9381), sqltypes.NewDecimal("-98.1839")}, }, { name: "decimal and float with negative values are not equal (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Float64, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Decimal, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewFloat64(-98.9381), sqltypes.NewDecimal("-98.1839")}, }, { name: "decimal and integer are equal (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Int64, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Decimal, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewInt64(8979), sqltypes.NewDecimal("8979")}, }, { name: "decimal and integer are equal (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Int64, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewDecimal("8979.0000"), sqltypes.NewInt64(8979)}, }, { name: "decimal and unsigned integer are equal (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Uint64, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Decimal, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewUint64(901), sqltypes.NewDecimal("901")}, }, { name: "decimal and unsigned integer are equal (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Uint64, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewDecimal("901.00"), sqltypes.NewUint64(901)}, }, { name: "decimal and unsigned integer are not equal (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Uint64, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewDecimal("192.129"), sqltypes.NewUint64(192)}, }, { name: "decimal and unsigned integer are not equal (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Uint64, collations.CollationBinaryID), out: &F, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewDecimal("192.129"), sqltypes.NewUint64(192)}, }, { name: "decimal is greater than integer", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Int64, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewDecimal("1.01"), sqltypes.NewInt64(1)}, }, { name: "decimal is greater-equal to integer", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Int64, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterEqualOp, row: []sqltypes.Value{sqltypes.NewDecimal("1.00"), sqltypes.NewInt64(1)}, }, { name: "decimal is less than integer", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Int64, collations.CollationBinaryID), out: &T, op: sqlparser.LessThanOp, row: []sqltypes.Value{sqltypes.NewDecimal(".99"), sqltypes.NewInt64(1)}, }, { name: "decimal is less-equal to integer", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Int64, collations.CollationBinaryID), out: &T, op: sqlparser.LessEqualOp, row: []sqltypes.Value{sqltypes.NewDecimal("1.00"), sqltypes.NewInt64(1)}, }, { name: "decimal is greater than float", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Float64, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewDecimal("849.896"), sqltypes.NewFloat64(86.568)}, }, { name: "decimal is not greater than float", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Float64, collations.CollationBinaryID), out: &F, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewDecimal("15.23"), sqltypes.NewFloat64(8689.5)}, }, { name: "decimal is greater-equal to float (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Float64, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterEqualOp, row: []sqltypes.Value{sqltypes.NewDecimal("65"), sqltypes.NewFloat64(65)}, }, { name: "decimal is greater-equal to float (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Float64, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterEqualOp, row: []sqltypes.Value{sqltypes.NewDecimal("65"), sqltypes.NewFloat64(60)}, }, { name: "decimal is less than float", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Float64, collations.CollationBinaryID), out: &T, op: sqlparser.LessThanOp, row: []sqltypes.Value{sqltypes.NewDecimal("0.998"), sqltypes.NewFloat64(0.999)}, }, { name: "decimal is less-equal to float", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Decimal, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Float64, collations.CollationBinaryID), out: &T, op: sqlparser.LessEqualOp, row: []sqltypes.Value{sqltypes.NewDecimal("1.000101"), sqltypes.NewFloat64(1.00101)}, }, { name: "different int types are equal for 8 bit", - v1: NewColumn(0), v2: NewLiteralInt(0), + v1: NewColumn(0, sqltypes.Int8, collations.CollationBinaryID), v2: NewLiteralInt(0), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewInt8(0)}, }, { name: "different int types are equal for 32 bit", - v1: NewColumn(0), v2: NewLiteralInt(0), + v1: NewColumn(0, sqltypes.Int32, collations.CollationBinaryID), v2: NewLiteralInt(0), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewInt32(0)}, }, { name: "different int types are equal for float32 bit", - v1: NewColumn(0), v2: NewLiteralFloat(1.0), + v1: NewColumn(0, sqltypes.Float32, collations.CollationBinaryID), v2: NewLiteralFloat(1.0), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.MakeTrusted(sqltypes.Float32, []byte("1.0"))}, }, { name: "different unsigned int types are equal for 8 bit", - v1: NewColumn(0), v2: NewLiteralInt(0), + v1: NewColumn(0, sqltypes.Uint8, collations.CollationBinaryID), v2: NewLiteralInt(0), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.MakeTrusted(sqltypes.Uint8, []byte("0"))}, }, { name: "different unsigned int types are equal for 32 bit", - v1: NewColumn(0), v2: NewLiteralInt(0), + v1: NewColumn(0, sqltypes.Uint32, collations.CollationBinaryID), v2: NewLiteralInt(0), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewUint32(0)}, }, @@ -493,73 +493,73 @@ func TestCompareDatetime(t *testing.T) { tests := []testCase{ { name: "datetimes are equal", - v1: NewColumn(0), v2: NewColumn(0), + v1: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), v2: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewDatetime("2021-10-22 12:00:00")}, }, { name: "datetimes are not equal (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &F, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewDatetime("2021-10-22 12:00:00"), sqltypes.NewDatetime("2020-10-22 12:00:00")}, }, { name: "datetimes are not equal (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &F, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewDatetime("2021-10-22 12:00:00"), sqltypes.NewDatetime("2021-10-22 10:23:56")}, }, { name: "datetimes are not equal (3)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewDatetime("2021-10-01 00:00:00"), sqltypes.NewDatetime("2021-02-01 00:00:00")}, }, { name: "datetime is greater than datetime", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewDatetime("2021-10-30 10:42:50"), sqltypes.NewDatetime("2021-10-01 13:10:02")}, }, { name: "datetime is not greater than datetime", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &F, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewDatetime("2021-10-01 13:10:02"), sqltypes.NewDatetime("2021-10-30 10:42:50")}, }, { name: "datetime is less than datetime", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.LessThanOp, row: []sqltypes.Value{sqltypes.NewDatetime("2021-10-01 13:10:02"), sqltypes.NewDatetime("2021-10-30 10:42:50")}, }, { name: "datetime is not less than datetime", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &F, op: sqlparser.LessThanOp, row: []sqltypes.Value{sqltypes.NewDatetime("2021-10-30 10:42:50"), sqltypes.NewDatetime("2021-10-01 13:10:02")}, }, { name: "datetime is greater-equal to datetime (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterEqualOp, row: []sqltypes.Value{sqltypes.NewDatetime("2021-10-30 10:42:50"), sqltypes.NewDatetime("2021-10-30 10:42:50")}, }, { name: "datetime is greater-equal to datetime (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterEqualOp, row: []sqltypes.Value{sqltypes.NewDatetime("2021-10-30 10:42:50"), sqltypes.NewDatetime("2021-10-01 13:10:02")}, }, { name: "datetime is less-equal to datetime (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.LessEqualOp, row: []sqltypes.Value{sqltypes.NewDatetime("2021-10-30 10:42:50"), sqltypes.NewDatetime("2021-10-30 10:42:50")}, }, { name: "datetime is less-equal to datetime (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Datetime, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.LessEqualOp, row: []sqltypes.Value{sqltypes.NewDatetime("2021-10-01 13:10:02"), sqltypes.NewDatetime("2021-10-30 10:42:50")}, }, @@ -577,73 +577,73 @@ func TestCompareTimestamp(t *testing.T) { tests := []testCase{ { name: "timestamps are equal", - v1: NewColumn(0), v2: NewColumn(0), + v1: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), v2: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewTimestamp("2021-10-22 12:00:00")}, }, { name: "timestamps are not equal (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &F, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewTimestamp("2021-10-22 12:00:00"), sqltypes.NewTimestamp("2020-10-22 12:00:00")}, }, { name: "timestamps are not equal (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &F, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewTimestamp("2021-10-22 12:00:00"), sqltypes.NewTimestamp("2021-10-22 10:23:56")}, }, { name: "timestamps are not equal (3)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewTimestamp("2021-10-01 00:00:00"), sqltypes.NewTimestamp("2021-02-01 00:00:00")}, }, { name: "timestamp is greater than timestamp", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewTimestamp("2021-10-30 10:42:50"), sqltypes.NewTimestamp("2021-10-01 13:10:02")}, }, { name: "timestamp is not greater than timestamp", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &F, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewTimestamp("2021-10-01 13:10:02"), sqltypes.NewTimestamp("2021-10-30 10:42:50")}, }, { name: "timestamp is less than timestamp", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &T, op: sqlparser.LessThanOp, row: []sqltypes.Value{sqltypes.NewTimestamp("2021-10-01 13:10:02"), sqltypes.NewTimestamp("2021-10-30 10:42:50")}, }, { name: "timestamp is not less than timestamp", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &F, op: sqlparser.LessThanOp, row: []sqltypes.Value{sqltypes.NewTimestamp("2021-10-30 10:42:50"), sqltypes.NewTimestamp("2021-10-01 13:10:02")}, }, { name: "timestamp is greater-equal to timestamp (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterEqualOp, row: []sqltypes.Value{sqltypes.NewTimestamp("2021-10-30 10:42:50"), sqltypes.NewTimestamp("2021-10-30 10:42:50")}, }, { name: "timestamp is greater-equal to timestamp (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterEqualOp, row: []sqltypes.Value{sqltypes.NewTimestamp("2021-10-30 10:42:50"), sqltypes.NewTimestamp("2021-10-01 13:10:02")}, }, { name: "timestamp is less-equal to timestamp (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &T, op: sqlparser.LessEqualOp, row: []sqltypes.Value{sqltypes.NewTimestamp("2021-10-30 10:42:50"), sqltypes.NewTimestamp("2021-10-30 10:42:50")}, }, { name: "timestamp is less-equal to timestamp (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Timestamp, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &T, op: sqlparser.LessEqualOp, row: []sqltypes.Value{sqltypes.NewTimestamp("2021-10-01 13:10:02"), sqltypes.NewTimestamp("2021-10-30 10:42:50")}, }, @@ -661,67 +661,67 @@ func TestCompareDate(t *testing.T) { tests := []testCase{ { name: "dates are equal", - v1: NewColumn(0), v2: NewColumn(0), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-22")}, }, { name: "dates are not equal (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Date, collations.CollationBinaryID), out: &F, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-22"), sqltypes.NewDate("2020-10-21")}, }, { name: "dates are not equal (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Date, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-01"), sqltypes.NewDate("2021-02-01")}, }, { name: "date is greater than date", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Date, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-30"), sqltypes.NewDate("2021-10-01")}, }, { name: "date is not greater than date", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Date, collations.CollationBinaryID), out: &F, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-01"), sqltypes.NewDate("2021-10-30")}, }, { name: "date is less than date", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Date, collations.CollationBinaryID), out: &T, op: sqlparser.LessThanOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-01"), sqltypes.NewDate("2021-10-30")}, }, { name: "date is not less than date", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Date, collations.CollationBinaryID), out: &F, op: sqlparser.LessThanOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-30"), sqltypes.NewDate("2021-10-01")}, }, { name: "date is greater-equal to date (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Date, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterEqualOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-30"), sqltypes.NewDate("2021-10-30")}, }, { name: "date is greater-equal to date (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Date, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterEqualOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-30"), sqltypes.NewDate("2021-10-01")}, }, { name: "date is less-equal to date (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Date, collations.CollationBinaryID), out: &T, op: sqlparser.LessEqualOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-30"), sqltypes.NewDate("2021-10-30")}, }, { name: "date is less-equal to date (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Date, collations.CollationBinaryID), out: &T, op: sqlparser.LessEqualOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-01"), sqltypes.NewDate("2021-10-30")}, }, @@ -739,67 +739,67 @@ func TestCompareTime(t *testing.T) { tests := []testCase{ { name: "times are equal", - v1: NewColumn(0), v2: NewColumn(0), + v1: NewColumn(0, sqltypes.Time, collations.CollationBinaryID), v2: NewColumn(0, sqltypes.Time, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewTime("12:00:00")}, }, { name: "times are not equal (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Time, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &F, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewTime("12:00:00"), sqltypes.NewTime("10:23:56")}, }, { name: "times are not equal (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Time, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewTime("00:00:00"), sqltypes.NewTime("10:15:00")}, }, { name: "time is greater than time", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Time, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewTime("18:14:35"), sqltypes.NewTime("13:01:38")}, }, { name: "time is not greater than time", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Time, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &F, op: sqlparser.GreaterThanOp, row: []sqltypes.Value{sqltypes.NewTime("02:46:02"), sqltypes.NewTime("10:42:50")}, }, { name: "time is less than time", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Time, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &T, op: sqlparser.LessThanOp, row: []sqltypes.Value{sqltypes.NewTime("04:30:00"), sqltypes.NewTime("09:23:48")}, }, { name: "time is not less than time", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Time, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &F, op: sqlparser.LessThanOp, row: []sqltypes.Value{sqltypes.NewTime("15:21:00"), sqltypes.NewTime("10:00:00")}, }, { name: "time is greater-equal to time (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Time, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterEqualOp, row: []sqltypes.Value{sqltypes.NewTime("10:42:50"), sqltypes.NewTime("10:42:50")}, }, { name: "time is greater-equal to time (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Time, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &T, op: sqlparser.GreaterEqualOp, row: []sqltypes.Value{sqltypes.NewTime("19:42:50"), sqltypes.NewTime("13:10:02")}, }, { name: "time is less-equal to time (1)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Time, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &T, op: sqlparser.LessEqualOp, row: []sqltypes.Value{sqltypes.NewTime("10:42:50"), sqltypes.NewTime("10:42:50")}, }, { name: "time is less-equal to time (2)", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Time, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &T, op: sqlparser.LessEqualOp, row: []sqltypes.Value{sqltypes.NewTime("10:10:02"), sqltypes.NewTime("10:42:50")}, }, @@ -817,13 +817,13 @@ func TestCompareDates(t *testing.T) { tests := []testCase{ { name: "date equal datetime", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-22"), sqltypes.NewDatetime("2021-10-22 00:00:00")}, }, { name: "date equal datetime through bind variables", - v1: NewBindVar("k1"), v2: NewBindVar("k2"), + v1: NewBindVar("k1", sqltypes.Date, collations.CollationBinaryID), v2: NewBindVar("k2", sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, bv: map[string]*querypb.BindVariable{ "k1": {Type: sqltypes.Date, Value: []byte("2021-10-22")}, @@ -832,7 +832,7 @@ func TestCompareDates(t *testing.T) { }, { name: "date not equal datetime through bind variables", - v1: NewBindVar("k1"), v2: NewBindVar("k2"), + v1: NewBindVar("k1", sqltypes.Date, collations.CollationBinaryID), v2: NewBindVar("k2", sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, bv: map[string]*querypb.BindVariable{ "k1": {Type: sqltypes.Date, Value: []byte("2021-02-20")}, @@ -841,73 +841,73 @@ func TestCompareDates(t *testing.T) { }, { name: "date not equal datetime", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-22"), sqltypes.NewDatetime("2021-10-20 00:06:00")}, }, { name: "date equal timestamp", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-22"), sqltypes.NewTimestamp("2021-10-22 00:00:00")}, }, { name: "date not equal timestamp", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewDate("2021-10-22"), sqltypes.NewTimestamp("2021-10-22 16:00:00")}, }, { name: "date equal time", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &F, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewDate(time.Now().Format("2006-01-02")), sqltypes.NewTime("00:00:00")}, }, { name: "date not equal time", - v1: NewColumn(0), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewDate(time.Now().Format("2006-01-02")), sqltypes.NewTime("12:00:00")}, }, { name: "string equal datetime", - v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.VarChar, collations.CollationUtf8mb4ID), v2: NewColumn(1, sqltypes.Datetime, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewVarChar("2021-10-22"), sqltypes.NewDatetime("2021-10-22 00:00:00")}, }, { name: "string equal timestamp", - v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.VarChar, collations.CollationUtf8mb4ID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewVarChar("2021-10-22 00:00:00"), sqltypes.NewTimestamp("2021-10-22 00:00:00")}, }, { name: "string not equal timestamp", - v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.VarChar, collations.CollationUtf8mb4ID), v2: NewColumn(1, sqltypes.Timestamp, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewVarChar("2021-10-22 06:00:30"), sqltypes.NewTimestamp("2021-10-20 15:02:10")}, }, { name: "string equal time", - v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.VarChar, collations.CollationUtf8mb4ID), v2: NewColumn(1, sqltypes.Time, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewVarChar("00:05:12"), sqltypes.NewTime("00:05:12")}, }, { name: "string equal date", - v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.VarChar, collations.CollationUtf8mb4ID), v2: NewColumn(1, sqltypes.Date, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewVarChar("2021-02-22"), sqltypes.NewDate("2021-02-22")}, }, { name: "string not equal date (1, date on the RHS)", - v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumn(1), + v1: NewColumn(0, sqltypes.VarChar, collations.CollationUtf8mb4ID), v2: NewColumn(1, sqltypes.Date, collations.CollationBinaryID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewVarChar("2021-02-20"), sqltypes.NewDate("2021-03-30")}, }, { name: "string not equal date (2, date on the LHS)", - v1: NewColumn(0), v2: NewColumnWithCollation(1, defaultCollation()), + v1: NewColumn(0, sqltypes.Date, collations.CollationBinaryID), v2: NewColumn(1, sqltypes.VarChar, collations.CollationUtf8mb4ID), out: &T, op: sqlparser.NotEqualOp, row: []sqltypes.Value{sqltypes.NewDate("2021-03-30"), sqltypes.NewVarChar("2021-02-20")}, }, @@ -925,13 +925,13 @@ func TestCompareStrings(t *testing.T) { tests := []testCase{ { name: "string equal string", - v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumnWithCollation(1, defaultCollation()), + v1: NewColumn(0, sqltypes.VarChar, collations.Default()), v2: NewColumn(1, sqltypes.VarChar, collations.Default()), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewVarChar("toto"), sqltypes.NewVarChar("toto")}, }, { name: "string equal number", - v1: NewColumnWithCollation(0, defaultCollation()), v2: NewColumnWithCollation(1, defaultCollation()), + v1: NewColumn(0, sqltypes.VarChar, collations.Default()), v2: NewColumn(1, sqltypes.Int64, collations.CollationBinaryID), out: &T, op: sqlparser.EqualOp, row: []sqltypes.Value{sqltypes.NewVarChar("1"), sqltypes.NewInt64(1)}, }, @@ -1180,7 +1180,7 @@ func TestNullsafeCompareCollate(t *testing.T) { collation: getCollationID("utf8mb4_ja_0900_as_cs_ks"), }, { - // non breaking space + // non-breaking space v1: "abc ", v2: "abc\u00a0", out: -1, diff --git a/go/vt/vtgate/evalengine/api_literal.go b/go/vt/vtgate/evalengine/api_literal.go index 996e4b2f505..2371dcd73bd 100644 --- a/go/vt/vtgate/evalengine/api_literal.go +++ b/go/vt/vtgate/evalengine/api_literal.go @@ -192,33 +192,45 @@ func NewLiteralBinaryFromBit(val []byte) (*Literal, error) { } // NewBindVar returns a bind variable -func NewBindVar(key string) *BindVariable { +func NewBindVar(key string, typ sqltypes.Type, col collations.ID) *BindVariable { return &BindVariable{ - Key: key, + Key: key, + Type: typ, Collation: collations.TypedCollation{ - Collation: collations.Unknown, + Collation: col, Coercibility: collations.CoerceCoercible, Repertoire: collations.RepertoireUnicode, }, + typed: typ >= 0, } } // NewBindVarTuple returns a bind variable containing a tuple -func NewBindVarTuple(key string) *BindVariable { +func NewBindVarTuple(key string, col collations.ID) *BindVariable { return &BindVariable{ - Key: key, - Type: sqltypes.Tuple, + Key: key, + Type: sqltypes.Tuple, + Collation: collations.TypedCollation{ + Collation: col, + Coercibility: collations.CoerceCoercible, + Repertoire: collations.RepertoireUnicode, + }, typed: true, } } // NewColumn returns a column expression -func NewColumn(offset int) *Column { - return NewColumnWithCollation(offset, collations.SystemCollation) -} - -func NewColumnWithCollation(offset int, coll collations.TypedCollation) *Column { - return &Column{Offset: offset, Type: -1, Collation: coll} +func NewColumn(offset int, typ sqltypes.Type, col collations.ID) *Column { + return &Column{ + Offset: offset, + Type: typ, + Collation: collations.TypedCollation{ + Collation: col, + Coercibility: collations.CoerceImplicit, + Repertoire: collations.RepertoireUnicode, + }, + typed: typ >= 0, + } } // NewTupleExpr returns a tuple expression diff --git a/go/vt/vtgate/evalengine/expr_bvar.go b/go/vt/vtgate/evalengine/expr_bvar.go index 2cc305e9982..fb8c56bea32 100644 --- a/go/vt/vtgate/evalengine/expr_bvar.go +++ b/go/vt/vtgate/evalengine/expr_bvar.go @@ -58,7 +58,11 @@ func (bv *BindVariable) eval(env *ExpressionEnv) (eval, error) { tuple := make([]eval, 0, len(bvar.Values)) for _, value := range bvar.Values { - e, err := valueToEval(sqltypes.MakeTrusted(value.Type, value.Value), collations.TypedCollation{}) + e, err := valueToEval(sqltypes.MakeTrusted(value.Type, value.Value), collations.TypedCollation{ + Collation: collations.DefaultCollationForType(value.Type), + Coercibility: collations.CoerceCoercible, + Repertoire: collations.RepertoireUnicode, + }) if err != nil { return nil, err } @@ -74,7 +78,11 @@ func (bv *BindVariable) eval(env *ExpressionEnv) (eval, error) { if bv.typed { typ = bv.Type } - return valueToEval(sqltypes.MakeTrusted(typ, bvar.Value), bv.Collation) + return valueToEval(sqltypes.MakeTrusted(typ, bvar.Value), collations.TypedCollation{ + Collation: collations.DefaultCollationForType(typ), + Coercibility: collations.CoerceCoercible, + Repertoire: collations.RepertoireUnicode, + }) } } diff --git a/go/vt/vtgate/evalengine/expr_column.go b/go/vt/vtgate/evalengine/expr_column.go index 969368ea3c2..f58df0fab3c 100644 --- a/go/vt/vtgate/evalengine/expr_column.go +++ b/go/vt/vtgate/evalengine/expr_column.go @@ -56,7 +56,7 @@ func (c *Column) typeof(env *ExpressionEnv, fields []*querypb.Field) (sqltypes.T if c.typed { return c.Type, flagNullable } - return sqltypes.Null, flagAmbiguousType + return -1, flagAmbiguousType } func (column *Column) compile(c *compiler) (ctype, error) { diff --git a/go/vt/vtgate/evalengine/expr_column_test.go b/go/vt/vtgate/evalengine/expr_column_test.go index e11a9313f90..99f96ce8b3f 100644 --- a/go/vt/vtgate/evalengine/expr_column_test.go +++ b/go/vt/vtgate/evalengine/expr_column_test.go @@ -65,7 +65,7 @@ func TestTypeOf(t *testing.T) { t.Run("Check when offset is out of bounds", func(t *testing.T) { c.Offset = 10 typ, flag := c.typeof(env, fields) - if typ != sqltypes.Null || flag != flagAmbiguousType { + if typ != -1 || flag != flagAmbiguousType { t.Errorf("typeof() failed, expected sqltypes.Null and flagAmbiguousType, got %v and %v", typ, flag) } }) diff --git a/go/vt/vtgate/evalengine/translate.go b/go/vt/vtgate/evalengine/translate.go index a49faded2c5..be0538fc4b7 100644 --- a/go/vt/vtgate/evalengine/translate.go +++ b/go/vt/vtgate/evalengine/translate.go @@ -192,26 +192,25 @@ func defaultCoercionCollation(id collations.ID) collations.TypedCollation { } func (ast *astCompiler) translateBindVar(arg *sqlparser.Argument) (Expr, error) { - bvar := NewBindVar(arg.Name) - bvar.Collation.Collation = ast.cfg.Collation + bvar := NewBindVar(arg.Name, arg.Type, ast.cfg.Collation) - if arg.Type >= 0 { - bvar.Type = arg.Type - bvar.typed = true - } else { + if !bvar.typed { ast.untyped++ } return bvar, nil } func (ast *astCompiler) translateColOffset(col *sqlparser.Offset) (Expr, error) { - column := NewColumn(col.V) + var typ querypb.Type = -1 + var coll collations.ID if ast.cfg.ResolveType != nil { - column.Type, column.Collation.Collation, column.typed = ast.cfg.ResolveType(col.Original) + typ, coll, _ = ast.cfg.ResolveType(col.Original) } - if column.Collation.Collation == collations.Unknown { - column.Collation.Collation = ast.cfg.Collation + if coll == collations.Unknown { + coll = ast.cfg.Collation } + + column := NewColumn(col.V, typ, coll) if !column.typed { ast.untyped++ } @@ -226,13 +225,17 @@ func (ast *astCompiler) translateColName(colname *sqlparser.ColName) (Expr, erro if err != nil { return nil, err } - column := NewColumn(idx) + var typ querypb.Type = -1 + var coll collations.ID if ast.cfg.ResolveType != nil { - column.Type, column.Collation.Collation, column.typed = ast.cfg.ResolveType(colname) + typ, coll, _ = ast.cfg.ResolveType(colname) } - if column.Collation.Collation == collations.Unknown { - column.Collation.Collation = ast.cfg.Collation + if coll == collations.Unknown { + coll = ast.cfg.Collation } + + column := NewColumn(idx, typ, coll) + if !column.typed { ast.untyped++ } @@ -509,7 +512,7 @@ func (ast *astCompiler) translateExpr(e sqlparser.Expr) (Expr, error) { case *sqlparser.Argument: return ast.translateBindVar(node) case sqlparser.ListArg: - return NewBindVarTuple(string(node)), nil + return NewBindVarTuple(string(node), ast.cfg.Collation), nil case *sqlparser.Literal: return translateLiteral(node, ast.cfg.Collation) case *sqlparser.AndExpr: @@ -637,5 +640,5 @@ func (fields FieldResolver) Type(expr sqlparser.Expr) (sqltypes.Type, collations } } } - return 0, 0, false + return -1, collations.Unknown, false } diff --git a/go/vt/vtgate/evalengine/translate_test.go b/go/vt/vtgate/evalengine/translate_test.go index 11e59a5de3c..d9ce3812abb 100644 --- a/go/vt/vtgate/evalengine/translate_test.go +++ b/go/vt/vtgate/evalengine/translate_test.go @@ -87,8 +87,8 @@ func TestTranslateSimplification(t *testing.T) { {"coalesce(NULL, 2, NULL, 4)", ok("COALESCE(NULL, INT64(2), NULL, INT64(4))"), ok("INT64(2)")}, {"coalesce(NULL, NULL)", ok("COALESCE(NULL, NULL)"), ok("NULL")}, {"coalesce(NULL)", ok("COALESCE(NULL)"), ok("NULL")}, - {"weight_string('foobar')", ok(`WEIGHT_STRING(VARCHAR("foobar"))`), ok(`VARBINARY("\x00F\x00O\x00O\x00B\x00A\x00R")`)}, - {"weight_string('foobar' as char(12))", ok(`WEIGHT_STRING(VARCHAR("foobar") AS CHAR(12))`), ok(`VARBINARY("\x00F\x00O\x00O\x00B\x00A\x00R\x00 \x00 \x00 \x00 \x00 \x00 ")`)}, + {"weight_string('foobar')", ok(`WEIGHT_STRING(VARCHAR("foobar"))`), ok("VARBINARY(\"\\x1c\\xe5\\x1d\\xdd\\x1d\\xdd\\x1c`\\x1cG\\x1e3\")")}, + {"weight_string('foobar' as char(12))", ok(`WEIGHT_STRING(VARCHAR("foobar") AS CHAR(12))`), ok("VARBINARY(\"\\x1c\\xe5\\x1d\\xdd\\x1d\\xdd\\x1c`\\x1cG\\x1e3\")")}, {"case when 1 = 1 then 2 else 3 end", ok("CASE WHEN INT64(1) = INT64(1) THEN INT64(2) ELSE INT64(3)"), ok("INT64(2)")}, {"case when null then 2 when 12 = 4 then 'ohnoes' else 42 end", ok(`CASE WHEN NULL THEN INT64(2) WHEN INT64(12) = INT64(4) THEN VARCHAR("ohnoes") ELSE INT64(42)`), ok(`VARCHAR("42")`)}, {"convert('a', char(2) character set utf8mb4)", ok(`CONVERT(VARCHAR("a"), CHAR(2) CHARACTER SET utf8mb4_0900_ai_ci)`), ok(`VARCHAR("a")`)}, @@ -125,7 +125,7 @@ func TestTranslateSimplification(t *testing.T) { cfg := &Config{ ResolveColumn: fields.Column, - Collation: 45, + Collation: collations.Default(), Optimization: OptimizationLevelNone, } @@ -300,7 +300,7 @@ func TestEvaluate(t *testing.T) { stmt, err := sqlparser.Parse("select " + test.expression) require.NoError(t, err) astExpr := stmt.(*sqlparser.Select).SelectExprs[0].(*sqlparser.AliasedExpr).Expr - sqltypesExpr, err := Translate(astExpr, &Config{Collation: 45}) + sqltypesExpr, err := Translate(astExpr, &Config{Collation: collations.Default()}) require.Nil(t, err) require.NotNil(t, sqltypesExpr) env := NewExpressionEnv(context.Background(), map[string]*querypb.BindVariable{ @@ -345,7 +345,7 @@ func TestEvaluateTuple(t *testing.T) { stmt, err := sqlparser.Parse("select " + test.expression) require.NoError(t, err) astExpr := stmt.(*sqlparser.Select).SelectExprs[0].(*sqlparser.AliasedExpr).Expr - sqltypesExpr, err := Translate(astExpr, &Config{Collation: 45}) + sqltypesExpr, err := Translate(astExpr, &Config{Collation: collations.Default()}) require.Nil(t, err) require.NotNil(t, sqltypesExpr) @@ -382,7 +382,7 @@ func TestTranslationFailures(t *testing.T) { stmt, err := sqlparser.Parse("select " + testcase.expression) require.NoError(t, err) astExpr := stmt.(*sqlparser.Select).SelectExprs[0].(*sqlparser.AliasedExpr).Expr - _, err = Translate(astExpr, &Config{Collation: 45}) + _, err = Translate(astExpr, &Config{Collation: collations.Default()}) require.EqualError(t, err, testcase.expectedErr) }) } @@ -418,7 +418,7 @@ func TestCardinalityWithBindVariables(t *testing.T) { } astExpr := stmt.(*sqlparser.Select).SelectExprs[0].(*sqlparser.AliasedExpr).Expr - _, err = Translate(astExpr, &Config{Collation: 45}) + _, err = Translate(astExpr, &Config{Collation: collations.Default()}) return err }() diff --git a/go/vt/vtgate/planbuilder/expression_converter.go b/go/vt/vtgate/planbuilder/expression_converter.go index 0946f6ef479..e6972f5acd4 100644 --- a/go/vt/vtgate/planbuilder/expression_converter.go +++ b/go/vt/vtgate/planbuilder/expression_converter.go @@ -86,7 +86,7 @@ func (ec *expressionConverter) convert(astExpr sqlparser.Expr, boolean, identifi if !strings.Contains(err.Error(), evalengine.ErrTranslateExprNotSupported) { return nil, err } - evalExpr = evalengine.NewColumn(len(ec.tabletExpressions)) + evalExpr = evalengine.NewColumn(len(ec.tabletExpressions), -1, collations.Default()) ec.tabletExpressions = append(ec.tabletExpressions, astExpr) } return evalExpr, nil diff --git a/go/vt/vtgate/planbuilder/expression_converter_test.go b/go/vt/vtgate/planbuilder/expression_converter_test.go index 2ff05d188fa..8bc8100ebc6 100644 --- a/go/vt/vtgate/planbuilder/expression_converter_test.go +++ b/go/vt/vtgate/planbuilder/expression_converter_test.go @@ -21,6 +21,8 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/evalengine" ) @@ -40,7 +42,7 @@ func TestConversion(t *testing.T) { expressionsOut: e(evalengine.NewLiteralInt(1)), }, { expressionsIn: "@@foo", - expressionsOut: e(evalengine.NewColumn(0)), + expressionsOut: e(evalengine.NewColumn(0, -1, collations.Default())), }} for _, tc := range queries { diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 307936d3c08..5d6a67ed167 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -177,12 +177,11 @@ func transformProjection(ctx *plancontext.PlanningContext, op *operators.Project case operators.Eval: return e.EExpr case operators.Offset: - t := ctx.SemTable.ExprTypes[e.Expr] - return &evalengine.Column{ - Offset: e.Offset, - Type: t.Type, - Collation: collations.TypedCollation{}, + t, found := ctx.SemTable.ExprTypes[e.Expr] + if !found { + return evalengine.NewColumn(e.Offset, -1, collations.CollationBinaryID) } + return evalengine.NewColumn(e.Offset, t.Type, t.Collation) default: failed = true return nil @@ -251,6 +250,7 @@ func transformFilter(ctx *plancontext.PlanningContext, op *operators.Filter) (lo // this might already have been done on the operators if predicate == nil { predicate, err = evalengine.Translate(ast, &evalengine.Config{ + ResolveType: ctx.SemTable.TypeForExpr, ResolveColumn: resolveFromPlan(ctx, plan, true), Collation: ctx.SemTable.Collation, }) diff --git a/go/vt/vtgate/planbuilder/operators/info_schema_planning.go b/go/vt/vtgate/planbuilder/operators/info_schema_planning.go index d0cfbbe521a..20b3d29e427 100644 --- a/go/vt/vtgate/planbuilder/operators/info_schema_planning.go +++ b/go/vt/vtgate/planbuilder/operators/info_schema_planning.go @@ -22,6 +22,8 @@ import ( "golang.org/x/exp/maps" "golang.org/x/exp/slices" + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/servenv" "vitess.io/vitess/go/vt/sqlparser" @@ -45,7 +47,10 @@ type InfoSchemaRouting struct { func (isr *InfoSchemaRouting) UpdateRoutingParams(_ *plancontext.PlanningContext, rp *engine.RoutingParameters) error { rp.SysTableTableSchema = nil for _, expr := range isr.SysTableTableSchema { - eexpr, err := evalengine.Translate(expr, &evalengine.Config{ResolveColumn: NotImplementedSchemaInfoResolver}) + eexpr, err := evalengine.Translate(expr, &evalengine.Config{ + Collation: collations.SystemCollation.Collation, + ResolveColumn: NotImplementedSchemaInfoResolver, + }) if err != nil { return err } @@ -54,7 +59,10 @@ func (isr *InfoSchemaRouting) UpdateRoutingParams(_ *plancontext.PlanningContext rp.SysTableTableName = make(map[string]evalengine.Expr, len(isr.SysTableTableName)) for k, expr := range isr.SysTableTableName { - eexpr, err := evalengine.Translate(expr, &evalengine.Config{ResolveColumn: NotImplementedSchemaInfoResolver}) + eexpr, err := evalengine.Translate(expr, &evalengine.Config{ + Collation: collations.SystemCollation.Collation, + ResolveColumn: NotImplementedSchemaInfoResolver, + }) if err != nil { return err } @@ -125,7 +133,10 @@ func extractInfoSchemaRoutingPredicate(in sqlparser.Expr, reservedVars *sqlparse // here we are just checking if this query can be translated to an evalengine expression // we'll need to do this translation again later when building the engine.Route - _, err := evalengine.Translate(rhs, &evalengine.Config{ResolveColumn: NotImplementedSchemaInfoResolver}) + _, err := evalengine.Translate(rhs, &evalengine.Config{ + Collation: collations.SystemCollation.Collation, + ResolveColumn: NotImplementedSchemaInfoResolver, + }) if err != nil { // if we can't translate this to an evalengine expression, // we are not going to be able to route based on this expression, diff --git a/go/vt/vtgate/planbuilder/operators/sharded_routing.go b/go/vt/vtgate/planbuilder/operators/sharded_routing.go index a49b4e6ba3a..26c5a370150 100644 --- a/go/vt/vtgate/planbuilder/operators/sharded_routing.go +++ b/go/vt/vtgate/planbuilder/operators/sharded_routing.go @@ -652,7 +652,10 @@ func makeEvalEngineExpr(ctx *plancontext.PlanningContext, n sqlparser.Expr) eval expr = sqlparser.NewArgument(extractedSubquery.GetArgName()) } } - ee, _ := evalengine.Translate(expr, &evalengine.Config{Collation: ctx.SemTable.Collation}) + ee, _ := evalengine.Translate(expr, &evalengine.Config{ + Collation: ctx.SemTable.Collation, + ResolveType: ctx.SemTable.TypeForExpr, + }) if ee != nil { return ee } diff --git a/go/vt/vtgate/planbuilder/postprocess.go b/go/vt/vtgate/planbuilder/postprocess.go index c64ce9f493d..2161e1ef414 100644 --- a/go/vt/vtgate/planbuilder/postprocess.go +++ b/go/vt/vtgate/planbuilder/postprocess.go @@ -17,6 +17,8 @@ limitations under the License. package planbuilder import ( + "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/evalengine" @@ -104,7 +106,7 @@ func setUpperLimit(plan logicalPlan) (bool, logicalPlan, error) { case *join, *joinGen4, *hashJoin: return false, node, nil case *memorySort: - pv := evalengine.NewBindVar("__upper_limit") + pv := evalengine.NewBindVar("__upper_limit", sqltypes.Int64, collations.CollationBinaryID) node.eMemorySort.UpperLimit = pv // we don't want to go down to the rest of the tree return false, node, nil diff --git a/go/vt/vtgate/planbuilder/predicate_rewrite_test.go b/go/vt/vtgate/planbuilder/predicate_rewrite_test.go index 36709f5d591..369a99bf5d3 100644 --- a/go/vt/vtgate/planbuilder/predicate_rewrite_test.go +++ b/go/vt/vtgate/planbuilder/predicate_rewrite_test.go @@ -103,10 +103,12 @@ func TestFuzzRewriting(t *testing.T) { simplified := sqlparser.RewritePredicate(predicate) original, err := evalengine.Translate(predicate, &evalengine.Config{ + Collation: collations.Default(), ResolveColumn: resolveForFuzz, }) require.NoError(t, err) simpler, err := evalengine.Translate(simplified.(sqlparser.Expr), &evalengine.Config{ + Collation: collations.Default(), ResolveColumn: resolveForFuzz, }) require.NoError(t, err) diff --git a/go/vt/vtgate/planbuilder/projection.go b/go/vt/vtgate/planbuilder/projection.go index 2444b916d48..d211f16233a 100644 --- a/go/vt/vtgate/planbuilder/projection.go +++ b/go/vt/vtgate/planbuilder/projection.go @@ -53,6 +53,7 @@ func (p *projection) WireupGen4(ctx *plancontext.PlanningContext) error { for _, expr := range p.columns { convert, err := evalengine.Translate(expr, &evalengine.Config{ ResolveColumn: resolveFromPlan(ctx, p.source, false), + ResolveType: ctx.SemTable.TypeForExpr, Collation: ctx.SemTable.Collation, }) if err != nil { diff --git a/go/vt/vtgate/planbuilder/route.go b/go/vt/vtgate/planbuilder/route.go index 64f4f7cbc63..d36cc60f071 100644 --- a/go/vt/vtgate/planbuilder/route.go +++ b/go/vt/vtgate/planbuilder/route.go @@ -20,6 +20,7 @@ import ( "fmt" "strconv" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine" @@ -249,7 +250,7 @@ func (rb *route) procureValues(plan logicalPlan, jt *jointab, val sqlparser.Expr return evalengine.NewTupleExpr(exprs...), nil case *sqlparser.ColName: joinVar := jt.Procure(plan, typedVal, rb.Order()) - return evalengine.NewBindVar(joinVar), nil + return evalengine.NewBindVar(joinVar, -1, collations.CollationBinaryID), nil default: return evalengine.Translate(typedVal, nil) } diff --git a/go/vt/vtgate/planbuilder/system_tables.go b/go/vt/vtgate/planbuilder/system_tables.go index d8c429af6e9..1245fe458de 100644 --- a/go/vt/vtgate/planbuilder/system_tables.go +++ b/go/vt/vtgate/planbuilder/system_tables.go @@ -19,6 +19,7 @@ package planbuilder import ( "strings" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vtgate/evalengine" @@ -106,7 +107,10 @@ func extractInfoSchemaRoutingPredicate( return } - evalExpr, err = evalengine.Translate(other, &evalengine.Config{ResolveColumn: operators.NotImplementedSchemaInfoResolver}) + evalExpr, err = evalengine.Translate(other, &evalengine.Config{ + Collation: collations.SystemCollation.Collation, + ResolveColumn: operators.NotImplementedSchemaInfoResolver, + }) if err != nil { if strings.Contains(err.Error(), evalengine.ErrTranslateExprNotSupported) { // This just means we can't rewrite this particular expression, diff --git a/go/vt/vtgate/planbuilder/vindex_op.go b/go/vt/vtgate/planbuilder/vindex_op.go index bfd064d6921..4918a326076 100644 --- a/go/vt/vtgate/planbuilder/vindex_op.go +++ b/go/vt/vtgate/planbuilder/vindex_op.go @@ -33,7 +33,8 @@ func transformVindexPlan(ctx *plancontext.PlanningContext, op *operators.Vindex) } expr, err := evalengine.Translate(op.Value, &evalengine.Config{ - Collation: ctx.SemTable.Collation, + Collation: ctx.SemTable.Collation, + ResolveType: ctx.SemTable.TypeForExpr, }) if err != nil { return nil, err diff --git a/go/vt/vtgate/semantics/real_table.go b/go/vt/vtgate/semantics/real_table.go index a116095fd90..bd57ab81474 100644 --- a/go/vt/vtgate/semantics/real_table.go +++ b/go/vt/vtgate/semantics/real_table.go @@ -104,10 +104,14 @@ func vindexTableToColumnInfo(tbl *vindexes.Table) []ColumnInfo { nameMap := map[string]any{} cols := make([]ColumnInfo, 0, len(tbl.Columns)) for _, col := range tbl.Columns { - var collation collations.ID + collation := collations.DefaultCollationForType(col.Type) if sqltypes.IsText(col.Type) { - collation, _ = collations.Local().LookupID(col.CollationName) + coll, found := collations.Local().LookupID(col.CollationName) + if found { + collation = coll + } } + cols = append(cols, ColumnInfo{ Name: col.Name.String(), Type: Type{ diff --git a/go/vt/vtgate/semantics/typer.go b/go/vt/vtgate/semantics/typer.go index 6c820ff14a8..5802b539585 100644 --- a/go/vt/vtgate/semantics/typer.go +++ b/go/vt/vtgate/semantics/typer.go @@ -45,10 +45,10 @@ func newTyper() *typer { func (t *typer) up(cursor *sqlparser.Cursor) error { switch node := cursor.Node().(type) { case *sqlparser.Literal: - t.exprTypes[node] = Type{Type: node.SQLType()} + t.exprTypes[node] = Type{Type: node.SQLType(), Collation: collations.DefaultCollationForType(node.SQLType())} case *sqlparser.Argument: if node.Type >= 0 { - t.exprTypes[node] = Type{Type: node.Type} + t.exprTypes[node] = Type{Type: node.Type, Collation: collations.DefaultCollationForType(node.Type)} } case sqlparser.AggrFunc: code, ok := opcode.SupportedAggregates[node.AggrName()] @@ -63,7 +63,7 @@ func (t *typer) up(cursor *sqlparser.Cursor) error { } } typ, _ := code.Type(inputType) - t.exprTypes[node] = Type{Type: typ} + t.exprTypes[node] = Type{Type: typ, Collation: collations.DefaultCollationForType(typ)} } return nil } From 8da8cdfbb6a1725edb1bc40e156a83574a36a003 Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Mon, 17 Jul 2023 10:59:51 +0200 Subject: [PATCH 06/13] evalengine: Cleanup unneeded flag We know this value always based on whether the type is -1 or a valid value, so we don't need the additional boolean value here. Signed-off-by: Dirkjan Bussink --- go/vt/vtgate/evalengine/api_literal.go | 3 --- go/vt/vtgate/evalengine/expr_bvar.go | 11 +++++++---- go/vt/vtgate/evalengine/expr_column.go | 9 ++++++--- go/vt/vtgate/evalengine/expr_column_test.go | 7 ++++--- go/vt/vtgate/evalengine/translate.go | 6 +++--- 5 files changed, 20 insertions(+), 16 deletions(-) diff --git a/go/vt/vtgate/evalengine/api_literal.go b/go/vt/vtgate/evalengine/api_literal.go index 2371dcd73bd..271798a5c7f 100644 --- a/go/vt/vtgate/evalengine/api_literal.go +++ b/go/vt/vtgate/evalengine/api_literal.go @@ -201,7 +201,6 @@ func NewBindVar(key string, typ sqltypes.Type, col collations.ID) *BindVariable Coercibility: collations.CoerceCoercible, Repertoire: collations.RepertoireUnicode, }, - typed: typ >= 0, } } @@ -215,7 +214,6 @@ func NewBindVarTuple(key string, col collations.ID) *BindVariable { Coercibility: collations.CoerceCoercible, Repertoire: collations.RepertoireUnicode, }, - typed: true, } } @@ -229,7 +227,6 @@ func NewColumn(offset int, typ sqltypes.Type, col collations.ID) *Column { Coercibility: collations.CoerceImplicit, Repertoire: collations.RepertoireUnicode, }, - typed: typ >= 0, } } diff --git a/go/vt/vtgate/evalengine/expr_bvar.go b/go/vt/vtgate/evalengine/expr_bvar.go index fb8c56bea32..387438c4310 100644 --- a/go/vt/vtgate/evalengine/expr_bvar.go +++ b/go/vt/vtgate/evalengine/expr_bvar.go @@ -29,7 +29,6 @@ type ( Key string Type sqltypes.Type Collation collations.TypedCollation - typed bool } ) @@ -75,7 +74,7 @@ func (bv *BindVariable) eval(env *ExpressionEnv) (eval, error) { return nil, vterrors.Errorf(vtrpcpb.Code_INVALID_ARGUMENT, "query argument '%s' must be a tuple (is %s)", bv.Key, bvar.Type) } typ := bvar.Type - if bv.typed { + if bv.typed() { typ = bv.Type } return valueToEval(sqltypes.MakeTrusted(typ, bvar.Value), collations.TypedCollation{ @@ -89,7 +88,7 @@ func (bv *BindVariable) eval(env *ExpressionEnv) (eval, error) { // typeof implements the Expr interface func (bv *BindVariable) typeof(env *ExpressionEnv, _ []*querypb.Field) (sqltypes.Type, typeFlag) { var tt sqltypes.Type - if bv.typed { + if bv.typed() { tt = bv.Type } else { if bvar, err := env.lookupBindVar(bv.Key); err == nil { @@ -107,7 +106,7 @@ func (bv *BindVariable) typeof(env *ExpressionEnv, _ []*querypb.Field) (sqltypes } func (bvar *BindVariable) compile(c *compiler) (ctype, error) { - if !bvar.typed { + if !bvar.typed() { return ctype{}, c.unsupported(bvar) } @@ -143,3 +142,7 @@ func (bvar *BindVariable) compile(c *compiler) (ctype, error) { Col: bvar.Collation, }, nil } + +func (bvar *BindVariable) typed() bool { + return bvar.Type >= 0 +} diff --git a/go/vt/vtgate/evalengine/expr_column.go b/go/vt/vtgate/evalengine/expr_column.go index f58df0fab3c..e0212bb5b28 100644 --- a/go/vt/vtgate/evalengine/expr_column.go +++ b/go/vt/vtgate/evalengine/expr_column.go @@ -29,7 +29,6 @@ type ( Offset int Type sqltypes.Type Collation collations.TypedCollation - typed bool } ) @@ -53,14 +52,14 @@ func (c *Column) typeof(env *ExpressionEnv, fields []*querypb.Field) (sqltypes.T return fields[c.Offset].Type, flagNullable // we probably got here because the value was NULL, // so let's assume we are on a nullable field } - if c.typed { + if c.typed() { return c.Type, flagNullable } return -1, flagAmbiguousType } func (column *Column) compile(c *compiler) (ctype, error) { - if !column.typed { + if !column.typed() { return ctype{}, c.unsupported(column) } @@ -102,3 +101,7 @@ func (column *Column) compile(c *compiler) (ctype, error) { Col: col, }, nil } + +func (column *Column) typed() bool { + return column.Type >= 0 +} diff --git a/go/vt/vtgate/evalengine/expr_column_test.go b/go/vt/vtgate/evalengine/expr_column_test.go index 99f96ce8b3f..6a34a7b48bc 100644 --- a/go/vt/vtgate/evalengine/expr_column_test.go +++ b/go/vt/vtgate/evalengine/expr_column_test.go @@ -42,7 +42,9 @@ func TestTypeOf(t *testing.T) { } fields := []*querypb.Field{field1, field2} - c := &Column{} + c := &Column{ + Type: -1, + } env.Row = sqltypes.Row{sqltypes.NewInt64(10)} t.Run("Check when row value is not null", func(t *testing.T) { @@ -66,11 +68,10 @@ func TestTypeOf(t *testing.T) { c.Offset = 10 typ, flag := c.typeof(env, fields) if typ != -1 || flag != flagAmbiguousType { - t.Errorf("typeof() failed, expected sqltypes.Null and flagAmbiguousType, got %v and %v", typ, flag) + t.Errorf("typeof() failed, expected -1 and flagAmbiguousType, got %v and %v", typ, flag) } }) t.Run("Check when typed is true", func(t *testing.T) { - c.typed = true c.Type = querypb.Type_FLOAT32 typ, flag := c.typeof(env, fields) if typ != querypb.Type_FLOAT32 || flag != flagNullable { diff --git a/go/vt/vtgate/evalengine/translate.go b/go/vt/vtgate/evalengine/translate.go index be0538fc4b7..8089afde425 100644 --- a/go/vt/vtgate/evalengine/translate.go +++ b/go/vt/vtgate/evalengine/translate.go @@ -194,7 +194,7 @@ func defaultCoercionCollation(id collations.ID) collations.TypedCollation { func (ast *astCompiler) translateBindVar(arg *sqlparser.Argument) (Expr, error) { bvar := NewBindVar(arg.Name, arg.Type, ast.cfg.Collation) - if !bvar.typed { + if !bvar.typed() { ast.untyped++ } return bvar, nil @@ -211,7 +211,7 @@ func (ast *astCompiler) translateColOffset(col *sqlparser.Offset) (Expr, error) } column := NewColumn(col.V, typ, coll) - if !column.typed { + if !column.typed() { ast.untyped++ } return column, nil @@ -236,7 +236,7 @@ func (ast *astCompiler) translateColName(colname *sqlparser.ColName) (Expr, erro column := NewColumn(idx, typ, coll) - if !column.typed { + if !column.typed() { ast.untyped++ } return column, nil From cd022564ab3c4e56b701382cfd0d5d8b07d2dbf5 Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Mon, 17 Jul 2023 14:03:10 +0200 Subject: [PATCH 07/13] planbuilder: Pass through type information as well With the type information we can better decide if we need to print the collation change or not, since it's only needed for textual columns. This also fixes a number of cases where we used to use the weight_string but we no longer need to since we now pass through the correct type information in more cases. Signed-off-by: Dirkjan Bussink --- go/vt/vtgate/engine/aggregations.go | 4 +- go/vt/vtgate/engine/cached_size.go | 8 +-- go/vt/vtgate/engine/distinct.go | 7 +-- go/vt/vtgate/engine/distinct_test.go | 7 ++- go/vt/vtgate/engine/memory_sort_test.go | 2 + go/vt/vtgate/engine/merge_sort_test.go | 1 + go/vt/vtgate/engine/ordered_aggregate.go | 3 +- go/vt/vtgate/engine/projection.go | 2 +- go/vt/vtgate/engine/route.go | 8 ++- go/vt/vtgate/engine/route_test.go | 2 + go/vt/vtgate/evalengine/cached_size.go | 4 +- go/vt/vtgate/evalengine/eval_result.go | 26 +++++----- go/vt/vtgate/planbuilder/grouping.go | 4 +- go/vt/vtgate/planbuilder/horizon_planning.go | 28 +++++++--- go/vt/vtgate/planbuilder/memory_sort.go | 1 + .../planbuilder/operator_transformers.go | 38 +++++++------- .../vtgate/planbuilder/operators/distinct.go | 4 +- .../planbuilder/operators/queryprojection.go | 15 ++++-- go/vt/vtgate/planbuilder/ordering.go | 1 + go/vt/vtgate/planbuilder/route.go | 2 +- .../testdata/info_schema57_cases.json | 51 +++++++++---------- .../testdata/info_schema80_cases.json | 51 +++++++++---------- .../planbuilder/testdata/union_cases.json | 32 ++++++------ go/vt/vtgate/semantics/semantic_state.go | 9 ---- go/vt/vttablet/tabletmanager/vdiff/utils.go | 6 +-- go/vt/wrangler/vdiff.go | 6 +-- 26 files changed, 176 insertions(+), 146 deletions(-) diff --git a/go/vt/vtgate/engine/aggregations.go b/go/vt/vtgate/engine/aggregations.go index abe542c1e4c..a6f2f01fcfb 100644 --- a/go/vt/vtgate/engine/aggregations.go +++ b/go/vt/vtgate/engine/aggregations.go @@ -43,6 +43,7 @@ type AggregateParams struct { // These are used only for distinct opcodes. KeyCol int WCol int + Type sqltypes.Type CollationID collations.ID Alias string `json:",omitempty"` @@ -60,6 +61,7 @@ func NewAggregateParam(opcode AggregateOpcode, col int, alias string) *Aggregate Col: col, Alias: alias, WCol: -1, + Type: -1, } if opcode.NeedsComparableValues() { out.KeyCol = col @@ -76,7 +78,7 @@ func (ap *AggregateParams) String() string { if ap.WAssigned() { keyCol = fmt.Sprintf("%s|%d", keyCol, ap.WCol) } - if ap.CollationID != collations.Unknown { + if sqltypes.IsText(ap.Type) && ap.CollationID != collations.Unknown { keyCol += " COLLATE " + ap.CollationID.Get().Name() } dispOrigOp := "" diff --git a/go/vt/vtgate/engine/cached_size.go b/go/vt/vtgate/engine/cached_size.go index 8f7ca406d3d..6014ba7a154 100644 --- a/go/vt/vtgate/engine/cached_size.go +++ b/go/vt/vtgate/engine/cached_size.go @@ -192,7 +192,7 @@ func (cached *Distinct) CachedSize(alloc bool) int64 { } // field CheckCols []vitess.io/vitess/go/vt/vtgate/engine.CheckCol { - size += hack.RuntimeAllocSize(int64(cap(cached.CheckCols)) * int64(18)) + size += hack.RuntimeAllocSize(int64(cap(cached.CheckCols)) * int64(22)) for _, elem := range cached.CheckCols { size += elem.CachedSize(false) } @@ -533,7 +533,7 @@ func (cached *MemorySort) CachedSize(alloc bool) int64 { } // field OrderBy []vitess.io/vitess/go/vt/vtgate/engine.OrderByParams { - size += hack.RuntimeAllocSize(int64(cap(cached.OrderBy)) * int64(36)) + size += hack.RuntimeAllocSize(int64(cap(cached.OrderBy)) * int64(42)) } // field Input vitess.io/vitess/go/vt/vtgate/engine.Primitive if cc, ok := cached.Input.(cachedObject); ok { @@ -560,7 +560,7 @@ func (cached *MergeSort) CachedSize(alloc bool) int64 { } // field OrderBy []vitess.io/vitess/go/vt/vtgate/engine.OrderByParams { - size += hack.RuntimeAllocSize(int64(cap(cached.OrderBy)) * int64(36)) + size += hack.RuntimeAllocSize(int64(cap(cached.OrderBy)) * int64(42)) } return size } @@ -775,7 +775,7 @@ func (cached *Route) CachedSize(alloc bool) int64 { size += hack.RuntimeAllocSize(int64(len(cached.FieldQuery))) // field OrderBy []vitess.io/vitess/go/vt/vtgate/engine.OrderByParams { - size += hack.RuntimeAllocSize(int64(cap(cached.OrderBy)) * int64(36)) + size += hack.RuntimeAllocSize(int64(cap(cached.OrderBy)) * int64(42)) } // field RoutingParameters *vitess.io/vitess/go/vt/vtgate/engine.RoutingParameters size += cached.RoutingParameters.CachedSize(true) diff --git a/go/vt/vtgate/engine/distinct.go b/go/vt/vtgate/engine/distinct.go index fcd74ce5eeb..0998bfa0e62 100644 --- a/go/vt/vtgate/engine/distinct.go +++ b/go/vt/vtgate/engine/distinct.go @@ -40,6 +40,7 @@ type ( CheckCol struct { Col int WsCol *int + Type sqltypes.Type Collation collations.ID } probeTable struct { @@ -274,15 +275,15 @@ func (cc CheckCol) SwitchToWeightString() CheckCol { return CheckCol{ Col: *cc.WsCol, WsCol: nil, + Type: sqltypes.VarBinary, Collation: collations.CollationBinaryID, } } func (cc CheckCol) String() string { - coll := cc.Collation.Get() var collation string - if coll != nil { - collation = ": " + coll.Name() + if sqltypes.IsText(cc.Type) && cc.Collation != collations.Unknown { + collation = ": " + cc.Collation.Get().Name() } var column string diff --git a/go/vt/vtgate/engine/distinct_test.go b/go/vt/vtgate/engine/distinct_test.go index 2c379da40e9..3fbf00835ca 100644 --- a/go/vt/vtgate/engine/distinct_test.go +++ b/go/vt/vtgate/engine/distinct_test.go @@ -65,12 +65,12 @@ func TestDistinct(t *testing.T) { expectedError: "text type with an unknown/unsupported collation cannot be hashed", }, { testName: "varchar columns with collations", - collations: []collations.ID{collations.ID(0x21)}, + collations: []collations.ID{collations.CollationUtf8mb4ID}, inputs: r("myid", "varchar", "monkey", "horse", "Horse", "Monkey", "horses", "MONKEY"), expectedResult: r("myid", "varchar", "monkey", "horse", "horses"), }, { testName: "mixed columns", - collations: []collations.ID{collations.ID(0x21), collations.Unknown}, + collations: []collations.ID{collations.CollationUtf8mb4ID, collations.Unknown}, inputs: r("myid|id", "varchar|int64", "monkey|1", "horse|1", "Horse|1", "Monkey|1", "horses|1", "MONKEY|2"), expectedResult: r("myid|id", "varchar|int64", "monkey|1", "horse|1", "horses|1", "MONKEY|2"), }} @@ -88,6 +88,7 @@ func TestDistinct(t *testing.T) { } checkCols = append(checkCols, CheckCol{ Col: i, + Type: tc.inputs.Fields[i].Type, Collation: collID, }) } @@ -133,6 +134,7 @@ func TestWeightStringFallBack(t *testing.T) { checkCols := []CheckCol{{ Col: 0, WsCol: &offsetOne, + Type: -1, Collation: collations.Unknown, }} input := r("myid|weightstring(myid)", @@ -158,6 +160,7 @@ func TestWeightStringFallBack(t *testing.T) { utils.MustMatch(t, []CheckCol{{ Col: 0, WsCol: &offsetOne, + Type: -1, Collation: collations.Unknown, }}, distinct.CheckCols, "checkCols should not be updated") } diff --git a/go/vt/vtgate/engine/memory_sort_test.go b/go/vt/vtgate/engine/memory_sort_test.go index 9028f9188f9..0af0564dc7c 100644 --- a/go/vt/vtgate/engine/memory_sort_test.go +++ b/go/vt/vtgate/engine/memory_sort_test.go @@ -231,6 +231,7 @@ func TestMemorySortStreamExecuteCollation(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderByParams{{ Col: 0, + Type: sqltypes.VarChar, CollationID: collationID, }}, Input: fp, @@ -319,6 +320,7 @@ func TestMemorySortExecuteCollation(t *testing.T) { ms := &MemorySort{ OrderBy: []OrderByParams{{ Col: 0, + Type: sqltypes.VarChar, CollationID: collationID, }}, Input: fp, diff --git a/go/vt/vtgate/engine/merge_sort_test.go b/go/vt/vtgate/engine/merge_sort_test.go index c72f31c0993..74c8d320d2b 100644 --- a/go/vt/vtgate/engine/merge_sort_test.go +++ b/go/vt/vtgate/engine/merge_sort_test.go @@ -180,6 +180,7 @@ func TestMergeSortCollation(t *testing.T) { collationID, _ := collations.Local().LookupID("utf8mb4_hu_0900_ai_ci") orderBy := []OrderByParams{{ Col: 0, + Type: sqltypes.VarChar, CollationID: collationID, }} diff --git a/go/vt/vtgate/engine/ordered_aggregate.go b/go/vt/vtgate/engine/ordered_aggregate.go index 114e8f0e820..8427ebb10ce 100644 --- a/go/vt/vtgate/engine/ordered_aggregate.go +++ b/go/vt/vtgate/engine/ordered_aggregate.go @@ -66,6 +66,7 @@ type GroupByParams struct { WeightStringCol int Expr sqlparser.Expr FromGroupBy bool + Type sqltypes.Type CollationID collations.ID } @@ -78,7 +79,7 @@ func (gbp GroupByParams) String() string { out = fmt.Sprintf("(%d|%d)", gbp.KeyCol, gbp.WeightStringCol) } - if gbp.CollationID != collations.Unknown { + if sqltypes.IsText(gbp.Type) && gbp.CollationID != collations.Unknown { collation := gbp.CollationID.Get() out += " COLLATE " + collation.Name() } diff --git a/go/vt/vtgate/engine/projection.go b/go/vt/vtgate/engine/projection.go index 604030cb909..5438b8144a6 100644 --- a/go/vt/vtgate/engine/projection.go +++ b/go/vt/vtgate/engine/projection.go @@ -146,7 +146,7 @@ func (p *Projection) evalFields(env *evalengine.ExpressionEnv, infields []*query return nil, err } var cs collations.ID = collations.CollationBinaryID - if sqltypes.IsQuoted(q) && !sqltypes.IsBinary(q) { + if sqltypes.IsText(q) { cs = vcursor.ConnCollation() } diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index db129d91f34..25619ebfb82 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -122,6 +122,8 @@ type OrderByParams struct { StarColFixedIndex int // v3 specific boolean. Used to also add weight strings originating from GroupBys to the Group by clause FromGroupBy bool + // Type for knowing if the collation is relevant + Type querypb.Type // Collation ID for comparison using collation CollationID collations.ID } @@ -140,7 +142,11 @@ func (obp OrderByParams) String() string { } else { val += " ASC" } - if obp.CollationID != collations.Unknown { + + if obp.Type != -1 && obp.CollationID == collations.Unknown { + panic("OrderByParams: collationID is unknown but type is not unknown") + } + if sqltypes.IsText(obp.Type) && obp.CollationID != collations.Unknown { collation := obp.CollationID.Get() val += " COLLATE " + collation.Name() } diff --git a/go/vt/vtgate/engine/route_test.go b/go/vt/vtgate/engine/route_test.go index e7f3ade2e57..fcfed19c68e 100644 --- a/go/vt/vtgate/engine/route_test.go +++ b/go/vt/vtgate/engine/route_test.go @@ -1081,6 +1081,7 @@ func TestRouteSortCollation(t *testing.T) { sel.OrderBy = []OrderByParams{{ Col: 0, + Type: sqltypes.VarChar, CollationID: collationID, }} @@ -1147,6 +1148,7 @@ func TestRouteSortCollation(t *testing.T) { t.Run("Error when Unknown Collation", func(t *testing.T) { sel.OrderBy = []OrderByParams{{ Col: 0, + Type: -1, CollationID: collations.Unknown, }} diff --git a/go/vt/vtgate/evalengine/cached_size.go b/go/vt/vtgate/evalengine/cached_size.go index 9955fc85c09..e8680e6ea45 100644 --- a/go/vt/vtgate/evalengine/cached_size.go +++ b/go/vt/vtgate/evalengine/cached_size.go @@ -63,7 +63,7 @@ func (cached *BindVariable) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(32) + size += int64(24) } // field Key string size += hack.RuntimeAllocSize(int64(len(cached.Key))) @@ -157,7 +157,7 @@ func (cached *Column) CachedSize(alloc bool) int64 { } size := int64(0) if alloc { - size += int64(24) + size += int64(16) } return size } diff --git a/go/vt/vtgate/evalengine/eval_result.go b/go/vt/vtgate/evalengine/eval_result.go index 8603cc6eac4..fa48ce19a25 100644 --- a/go/vt/vtgate/evalengine/eval_result.go +++ b/go/vt/vtgate/evalengine/eval_result.go @@ -35,21 +35,19 @@ type EvalResult struct { // collation and what the client expects the result to be in. func (er EvalResult) Value(id collations.ID) sqltypes.Value { str, ok := er.v.(*evalBytes) - if ok && !str.isBinary() { - if str.col.Collation == id { - return sqltypes.MakeTrusted(str.SQLType(), str.bytes) - } - dst, err := charset.Convert(nil, id.Get().Charset(), str.bytes, str.col.Collation.Get().Charset()) - if err != nil { - // If we can't convert, we just return what we have, but it's going - // to be invalidly encoded. Should normally never happen as only utf8mb4 - // is really supported for the connection character set anyway and all - // other charsets can be converted to utf8mb4. - return sqltypes.MakeTrusted(str.SQLType(), str.bytes) - } - return sqltypes.MakeTrusted(str.SQLType(), dst) + if !ok || str.isBinary() || str.col.Collation == collations.Unknown || str.col.Collation == id { + return evalToSQLValue(er.v) + } + + dst, err := charset.Convert(nil, id.Get().Charset(), str.bytes, str.col.Collation.Get().Charset()) + if err != nil { + // If we can't convert, we just return what we have, but it's going + // to be invalidly encoded. Should normally never happen as only utf8mb4 + // is really supported for the connection character set anyway and all + // other charsets can be converted to utf8mb4. + return sqltypes.MakeTrusted(str.SQLType(), str.bytes) } - return evalToSQLValue(er.v) + return sqltypes.MakeTrusted(str.SQLType(), dst) } func (er EvalResult) Collation() collations.ID { diff --git a/go/vt/vtgate/planbuilder/grouping.go b/go/vt/vtgate/planbuilder/grouping.go index 0bd10666029..5af57b0a2e8 100644 --- a/go/vt/vtgate/planbuilder/grouping.go +++ b/go/vt/vtgate/planbuilder/grouping.go @@ -78,7 +78,7 @@ func planGroupBy(pb *primitiveBuilder, input logicalPlan, groupBy sqlparser.Grou default: return nil, vterrors.VT12001("in scatter query: only simple references are allowed") } - node.groupByKeys = append(node.groupByKeys, &engine.GroupByParams{KeyCol: colNumber, WeightStringCol: -1, FromGroupBy: true}) + node.groupByKeys = append(node.groupByKeys, &engine.GroupByParams{KeyCol: colNumber, WeightStringCol: -1, Type: -1, FromGroupBy: true}) } // Append the distinct aggregate if any. if node.extraDistinct != nil { @@ -111,7 +111,7 @@ func planDistinct(input logicalPlan) (logicalPlan, error) { if rc.column.Origin() == node { return newDistinctV3(node), nil } - node.groupByKeys = append(node.groupByKeys, &engine.GroupByParams{KeyCol: i, WeightStringCol: -1, FromGroupBy: false}) + node.groupByKeys = append(node.groupByKeys, &engine.GroupByParams{KeyCol: i, WeightStringCol: -1, Type: -1, FromGroupBy: false}) } newInput, err := planDistinct(node.input) if err != nil { diff --git a/go/vt/vtgate/planbuilder/horizon_planning.go b/go/vt/vtgate/planbuilder/horizon_planning.go index 8f78959d72d..598895d0b05 100644 --- a/go/vt/vtgate/planbuilder/horizon_planning.go +++ b/go/vt/vtgate/planbuilder/horizon_planning.go @@ -19,6 +19,7 @@ package planbuilder import ( "fmt" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" @@ -279,10 +280,12 @@ func (hp *horizonPlanning) planAggrUsingOA( // here we are building up the grouping keys for the OA, // but they are lacking the input offsets because we have yet to push the columns down for _, expr := range grouping { + typ, col, _ := ctx.SemTable.TypeForExpr(expr.Inner) oa.groupByKeys = append(oa.groupByKeys, &engine.GroupByParams{ Expr: expr.Inner, FromGroupBy: true, - CollationID: ctx.SemTable.CollationForExpr(expr.Inner), + Type: typ, + CollationID: col, }) } @@ -474,12 +477,14 @@ func addColumnsToOA( o := groupings[count] count++ a := aggregationExprs[offset] - collID := ctx.SemTable.CollationForExpr(a.Func.GetArg()) aggr := engine.NewAggregateParam(a.OpCode, o.col, a.Alias) aggr.KeyCol = o.col aggr.WCol = o.wsCol aggr.Original = a.Original - aggr.CollationID = collID + aggr.Type, aggr.CollationID, _ = ctx.SemTable.TypeForExpr(a.Func.GetArg()) + if aggr.Type != -1 && aggr.CollationID == collations.Unknown { + panic("unexpected: aggregate function without collation") + } oa.aggregates = append(oa.aggregates, aggr) } lastOffset := distinctOffsets[len(distinctOffsets)-1] @@ -678,11 +683,13 @@ func planOrderByForRoute(ctx *plancontext.PlanningContext, orderExprs []ops.Orde if err != nil { return nil, err } + typ, col, _ := ctx.SemTable.TypeForExpr(order.Inner.Expr) plan.eroute.OrderBy = append(plan.eroute.OrderBy, engine.OrderByParams{ Col: offset, WeightStringCol: weightStringOffset, Desc: order.Inner.Direction == sqlparser.DescOrder, - CollationID: ctx.SemTable.CollationForExpr(order.Inner.Expr), + Type: typ, + CollationID: col, }) } return plan, nil @@ -823,12 +830,13 @@ func createMemorySortPlanOnAggregation(ctx *plancontext.PlanningContext, plan *o return nil, vterrors.VT13001(fmt.Sprintf("expected to find ORDER BY expression (%s) in orderedAggregate", sqlparser.String(order.Inner))) } - collationID := ctx.SemTable.CollationForExpr(order.SimplifiedExpr) + typ, collationID, _ := ctx.SemTable.TypeForExpr(order.SimplifiedExpr) ms.eMemorySort.OrderBy = append(ms.eMemorySort.OrderBy, engine.OrderByParams{ Col: offset, WeightStringCol: woffset, Desc: order.Inner.Direction == sqlparser.DescOrder, StarColFixedIndex: offset, + Type: typ, CollationID: collationID, }) } @@ -871,12 +879,14 @@ func (hp *horizonPlanning) createMemorySortPlan(ctx *plancontext.PlanningContext if err != nil { return nil, err } + typ, col, _ := ctx.SemTable.TypeForExpr(order.Inner.Expr) ms.eMemorySort.OrderBy = append(ms.eMemorySort.OrderBy, engine.OrderByParams{ Col: offset, WeightStringCol: weightStringOffset, Desc: order.Inner.Direction == sqlparser.DescOrder, StarColFixedIndex: offset, - CollationID: ctx.SemTable.CollationForExpr(order.Inner.Expr), + Type: typ, + CollationID: col, }) } return ms, nil @@ -941,7 +951,8 @@ func (hp *horizonPlanning) planDistinctOA(semTable *semantics.SemTable, currPlan for _, aggrParam := range currPlan.aggregates { if semTable.EqualsExpr(expr, aggrParam.Expr) { found = true - oa.groupByKeys = append(oa.groupByKeys, &engine.GroupByParams{KeyCol: aggrParam.Col, WeightStringCol: -1, CollationID: semTable.CollationForExpr(expr)}) + typ, col, _ := semTable.TypeForExpr(expr) + oa.groupByKeys = append(oa.groupByKeys, &engine.GroupByParams{KeyCol: aggrParam.Col, WeightStringCol: -1, Type: typ, CollationID: col}) break } } @@ -973,7 +984,8 @@ func (hp *horizonPlanning) addDistinct(ctx *plancontext.PlanningContext, plan lo inner = sqlparser.NewColName(aliasExpr.As.String()) ctx.SemTable.CopyDependencies(aliasExpr.Expr, inner) } - grpParam := &engine.GroupByParams{KeyCol: index, WeightStringCol: -1, CollationID: ctx.SemTable.CollationForExpr(inner), Expr: inner} + typ, col, _ := ctx.SemTable.TypeForExpr(inner) + grpParam := &engine.GroupByParams{KeyCol: index, WeightStringCol: -1, Type: typ, CollationID: col, Expr: inner} _, wOffset, err := wrapAndPushExpr(ctx, aliasExpr.Expr, aliasExpr.Expr, plan) if err != nil { return nil, err diff --git a/go/vt/vtgate/planbuilder/memory_sort.go b/go/vt/vtgate/planbuilder/memory_sort.go index 20dd125ecd0..59c1b103382 100644 --- a/go/vt/vtgate/planbuilder/memory_sort.go +++ b/go/vt/vtgate/planbuilder/memory_sort.go @@ -95,6 +95,7 @@ func newMemorySort(plan logicalPlan, orderBy v3OrderBy) (*memorySort, error) { Desc: order.Direction == sqlparser.DescOrder, StarColFixedIndex: colNumber, FromGroupBy: order.fromGroupBy, + Type: -1, CollationID: collations.Unknown, } ms.eMemorySort.OrderBy = append(ms.eMemorySort.OrderBy, ob) diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 5d6a67ed167..7c163b034c0 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -92,15 +92,17 @@ func transformAggregator(ctx *plancontext.PlanningContext, op *operators.Aggrega aggrParam.Original = aggr.Original aggrParam.OrigOpcode = aggr.OriginalOpCode aggrParam.WCol = aggr.WSOffset - aggrParam.CollationID = aggr.GetCollation(ctx) + aggrParam.Type, aggrParam.CollationID = aggr.GetTypeCollation(ctx) oa.aggregates = append(oa.aggregates, aggrParam) } for _, groupBy := range op.Grouping { + typ, col, _ := ctx.SemTable.TypeForExpr(groupBy.SimplifiedExpr) oa.groupByKeys = append(oa.groupByKeys, &engine.GroupByParams{ KeyCol: groupBy.ColOffset, WeightStringCol: groupBy.WSOffset, Expr: groupBy.AsAliasedExpr().Expr, - CollationID: ctx.SemTable.CollationForExpr(groupBy.SimplifiedExpr), + Type: typ, + CollationID: col, }) } @@ -142,12 +144,13 @@ func createMemorySort(ctx *plancontext.PlanningContext, src logicalPlan, orderin } for idx, order := range ordering.Order { - collationID := ctx.SemTable.CollationForExpr(order.SimplifiedExpr) + typ, collationID, _ := ctx.SemTable.TypeForExpr(order.SimplifiedExpr) ms.eMemorySort.OrderBy = append(ms.eMemorySort.OrderBy, engine.OrderByParams{ Col: ordering.Offset[idx], WeightStringCol: ordering.WOffset[idx], Desc: order.Inner.Direction == sqlparser.DescOrder, StarColFixedIndex: ordering.Offset[idx], + Type: typ, CollationID: collationID, }) } @@ -179,7 +182,7 @@ func transformProjection(ctx *plancontext.PlanningContext, op *operators.Project case operators.Offset: t, found := ctx.SemTable.ExprTypes[e.Expr] if !found { - return evalengine.NewColumn(e.Offset, -1, collations.CollationBinaryID) + return evalengine.NewColumn(e.Offset, -1, collations.Unknown) } return evalengine.NewColumn(e.Offset, t.Type, t.Collation) default: @@ -383,11 +386,12 @@ func transformRoutePlan(ctx *plancontext.PlanningContext, op *operators.Route) ( replaceSubQuery(ctx, sel) eroute, err := routeToEngineRoute(ctx, op) for _, order := range op.Ordering { - collation := ctx.SemTable.CollationForExpr(order.AST) + typ, collation, _ := ctx.SemTable.TypeForExpr(order.AST) eroute.OrderBy = append(eroute.OrderBy, engine.OrderByParams{ Col: order.Offset, WeightStringCol: order.WOffset, Desc: order.Direction == sqlparser.DescOrder, + Type: typ, CollationID: collation, }) } @@ -697,11 +701,11 @@ func getWeightStringForSelectExpr(selectExpr sqlparser.SelectExpr) (*sqlparser.A return &sqlparser.AliasedExpr{Expr: weightStringFor(expr.Expr)}, nil } -func getCheckColsForUnion(ctx *plancontext.PlanningContext, result logicalPlan, colls []collations.ID) ([]engine.CheckCol, error) { +func getCheckColsForUnion(ctx *plancontext.PlanningContext, result logicalPlan, colls []collationInfo) ([]engine.CheckCol, error) { checkCols := make([]engine.CheckCol, 0, len(colls)) for i, coll := range colls { - checkCol := engine.CheckCol{Col: i, Collation: coll} - if coll != collations.Unknown { + checkCol := engine.CheckCol{Col: i, Type: coll.typ, Collation: coll.col} + if coll.typ >= 0 { checkCols = append(checkCols, checkCol) continue } @@ -839,9 +843,14 @@ func transformAndMergeInOrder(ctx *plancontext.PlanningContext, op *operators.Un return sources, nil } -func getCollationsFor(ctx *plancontext.PlanningContext, n *operators.Union) []collations.ID { +type collationInfo struct { + typ sqltypes.Type + col collations.ID +} + +func getCollationsFor(ctx *plancontext.PlanningContext, n *operators.Union) []collationInfo { // TODO: coerce selects' select expressions' collations - var colls []collations.ID + var colls []collationInfo sel, err := n.GetSelectFor(0) if err != nil { @@ -852,13 +861,8 @@ func getCollationsFor(ctx *plancontext.PlanningContext, n *operators.Union) []co if !ok { return nil } - typ := ctx.SemTable.CollationForExpr(aliasedE.Expr) - if typ == collations.Unknown { - if t, hasT := ctx.SemTable.ExprTypes[aliasedE.Expr]; hasT && sqltypes.IsNumber(t.Type) { - typ = collations.CollationBinaryID - } - } - colls = append(colls, typ) + typ, col, _ := ctx.SemTable.TypeForExpr(aliasedE.Expr) + colls = append(colls, collationInfo{typ: typ, col: col}) } return colls } diff --git a/go/vt/vtgate/planbuilder/operators/distinct.go b/go/vt/vtgate/planbuilder/operators/distinct.go index d5905f87adb..ae54672c353 100644 --- a/go/vt/vtgate/planbuilder/operators/distinct.go +++ b/go/vt/vtgate/planbuilder/operators/distinct.go @@ -53,9 +53,11 @@ func (d *Distinct) planOffsets(ctx *plancontext.PlanningContext) error { d.Source = newSrc e := d.QP.GetSimplifiedExpr(col.Expr) exprs = append(exprs, e) + typ, coll, _ := ctx.SemTable.TypeForExpr(e) d.Columns = append(d.Columns, engine.CheckCol{ Col: offset, - Collation: ctx.SemTable.CollationForExpr(e), + Type: typ, + Collation: coll, }) } for i, e := range exprs { diff --git a/go/vt/vtgate/planbuilder/operators/queryprojection.go b/go/vt/vtgate/planbuilder/operators/queryprojection.go index 8f70df5c52e..40c4f6293bc 100644 --- a/go/vt/vtgate/planbuilder/operators/queryprojection.go +++ b/go/vt/vtgate/planbuilder/operators/queryprojection.go @@ -24,6 +24,8 @@ import ( "golang.org/x/exp/slices" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" @@ -113,15 +115,20 @@ func (aggr Aggr) NeedsWeightString(ctx *plancontext.PlanningContext) bool { return aggr.OpCode.NeedsComparableValues() && ctx.SemTable.NeedsWeightString(aggr.Func.GetArg()) } -func (aggr Aggr) GetCollation(ctx *plancontext.PlanningContext) collations.ID { +func (aggr Aggr) GetTypeCollation(ctx *plancontext.PlanningContext) (sqltypes.Type, collations.ID) { if aggr.Func == nil { - return collations.Unknown + return -1, collations.Unknown } switch aggr.OpCode { case opcode.AggregateMin, opcode.AggregateMax, opcode.AggregateSumDistinct, opcode.AggregateCountDistinct: - return ctx.SemTable.CollationForExpr(aggr.Func.GetArg()) + typ, col, _ := ctx.SemTable.TypeForExpr(aggr.Func.GetArg()) + if typ != -1 && col == collations.Unknown { + panic("unexpected: aggregate function without collation") + } + return typ, col + } - return collations.Unknown + return -1, collations.Unknown } // NewGroupBy creates a new group by from the given fields. diff --git a/go/vt/vtgate/planbuilder/ordering.go b/go/vt/vtgate/planbuilder/ordering.go index 2a8613620e7..7fbe146d657 100644 --- a/go/vt/vtgate/planbuilder/ordering.go +++ b/go/vt/vtgate/planbuilder/ordering.go @@ -342,6 +342,7 @@ func planRouteOrdering(orderBy v3OrderBy, node *route) (logicalPlan, error) { Desc: order.Direction == sqlparser.DescOrder, StarColFixedIndex: starColFixedIndex, FromGroupBy: order.fromGroupBy, + Type: -1, CollationID: collations.Unknown, } node.eroute.OrderBy = append(node.eroute.OrderBy, ob) diff --git a/go/vt/vtgate/planbuilder/route.go b/go/vt/vtgate/planbuilder/route.go index d36cc60f071..cf904b0191f 100644 --- a/go/vt/vtgate/planbuilder/route.go +++ b/go/vt/vtgate/planbuilder/route.go @@ -250,7 +250,7 @@ func (rb *route) procureValues(plan logicalPlan, jt *jointab, val sqlparser.Expr return evalengine.NewTupleExpr(exprs...), nil case *sqlparser.ColName: joinVar := jt.Procure(plan, typedVal, rb.Order()) - return evalengine.NewBindVar(joinVar, -1, collations.CollationBinaryID), nil + return evalengine.NewBindVar(joinVar, -1, collations.Unknown), nil default: return evalengine.Translate(typedVal, nil) } diff --git a/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json b/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json index d96ac0d9caf..3723cc0834c 100644 --- a/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/info_schema57_cases.json @@ -191,29 +191,28 @@ "Instructions": { "OperatorType": "Distinct", "Collations": [ - "(0:21)", - "(1:22)", - "(2:23)", - "(3:24)", - "(4:25)", - "5: binary", - "(6:26)", - "7: binary", - "8: binary", - "9: binary", - "10: binary", - "11: binary", - "12: binary", - "13: binary", - "(14:27)", - "(15:28)", - "(16:29)", - "(17:30)", - "18: binary", - "(19:31)", - "(20:32)" + "0: utf8mb4_0900_ai_ci", + "1: utf8mb4_0900_ai_ci", + "2: utf8mb4_0900_ai_ci", + "3: utf8mb4_0900_ai_ci", + "4: utf8mb4_0900_ai_ci", + "5", + "6: utf8mb4_0900_ai_ci", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17: utf8mb4_0900_ai_ci", + "18", + "19: utf8mb4_0900_ai_ci", + "20: utf8mb4_0900_ai_ci" ], - "ResultColumns": 21, "Inputs": [ { "OperatorType": "Concatenate", @@ -225,8 +224,8 @@ "Name": "main", "Sharded": false }, - "FieldQuery": "select TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT, weight_string(TABLE_CATALOG), weight_string(TABLE_SCHEMA), weight_string(TABLE_NAME), weight_string(TABLE_TYPE), weight_string(`ENGINE`), weight_string(`ROW_FORMAT`), weight_string(CREATE_TIME), weight_string(UPDATE_TIME), weight_string(CHECK_TIME), weight_string(TABLE_COLLATION), weight_string(CREATE_OPTIONS), weight_string(TABLE_COMMENT) from information_schema.`tables` where 1 != 1", - "Query": "select distinct TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT, weight_string(TABLE_CATALOG), weight_string(TABLE_SCHEMA), weight_string(TABLE_NAME), weight_string(TABLE_TYPE), weight_string(`ENGINE`), weight_string(`ROW_FORMAT`), weight_string(CREATE_TIME), weight_string(UPDATE_TIME), weight_string(CHECK_TIME), weight_string(TABLE_COLLATION), weight_string(CREATE_OPTIONS), weight_string(TABLE_COMMENT) from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */", + "FieldQuery": "select TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT from information_schema.`tables` where 1 != 1", + "Query": "select distinct TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */", "SysTableTableSchema": "[VARCHAR(\"user\")]", "Table": "information_schema.`tables`" }, @@ -237,8 +236,8 @@ "Name": "main", "Sharded": false }, - "FieldQuery": "select TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT, weight_string(TABLE_CATALOG), weight_string(TABLE_SCHEMA), weight_string(TABLE_NAME), weight_string(TABLE_TYPE), weight_string(`ENGINE`), weight_string(`ROW_FORMAT`), weight_string(CREATE_TIME), weight_string(UPDATE_TIME), weight_string(CHECK_TIME), weight_string(TABLE_COLLATION), weight_string(CREATE_OPTIONS), weight_string(TABLE_COMMENT) from information_schema.`tables` where 1 != 1", - "Query": "select distinct TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT, weight_string(TABLE_CATALOG), weight_string(TABLE_SCHEMA), weight_string(TABLE_NAME), weight_string(TABLE_TYPE), weight_string(`ENGINE`), weight_string(`ROW_FORMAT`), weight_string(CREATE_TIME), weight_string(UPDATE_TIME), weight_string(CHECK_TIME), weight_string(TABLE_COLLATION), weight_string(CREATE_OPTIONS), weight_string(TABLE_COMMENT) from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */", + "FieldQuery": "select TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT from information_schema.`tables` where 1 != 1", + "Query": "select distinct TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */", "SysTableTableSchema": "[VARCHAR(\"main\")]", "Table": "information_schema.`tables`" } diff --git a/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json b/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json index 6363203b88a..51767530a3c 100644 --- a/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/info_schema80_cases.json @@ -191,29 +191,28 @@ "Instructions": { "OperatorType": "Distinct", "Collations": [ - "(0:21)", - "(1:22)", - "(2:23)", - "(3:24)", - "(4:25)", - "5: binary", - "(6:26)", - "7: binary", - "8: binary", - "9: binary", - "10: binary", - "11: binary", - "12: binary", - "13: binary", - "(14:27)", - "(15:28)", - "(16:29)", - "(17:30)", - "18: binary", - "(19:31)", - "(20:32)" + "0: utf8mb4_0900_ai_ci", + "1: utf8mb4_0900_ai_ci", + "2: utf8mb4_0900_ai_ci", + "3", + "4: utf8mb4_0900_ai_ci", + "5", + "6", + "7", + "8", + "9", + "10", + "11", + "12", + "13", + "14", + "15", + "16", + "17: utf8mb4_0900_ai_ci", + "18", + "19: utf8mb4_0900_ai_ci", + "20: utf8mb4_0900_ai_ci" ], - "ResultColumns": 21, "Inputs": [ { "OperatorType": "Concatenate", @@ -225,8 +224,8 @@ "Name": "main", "Sharded": false }, - "FieldQuery": "select TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT, weight_string(TABLE_CATALOG), weight_string(TABLE_SCHEMA), weight_string(TABLE_NAME), weight_string(TABLE_TYPE), weight_string(`ENGINE`), weight_string(`ROW_FORMAT`), weight_string(CREATE_TIME), weight_string(UPDATE_TIME), weight_string(CHECK_TIME), weight_string(TABLE_COLLATION), weight_string(CREATE_OPTIONS), weight_string(TABLE_COMMENT) from information_schema.`tables` where 1 != 1", - "Query": "select distinct TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT, weight_string(TABLE_CATALOG), weight_string(TABLE_SCHEMA), weight_string(TABLE_NAME), weight_string(TABLE_TYPE), weight_string(`ENGINE`), weight_string(`ROW_FORMAT`), weight_string(CREATE_TIME), weight_string(UPDATE_TIME), weight_string(CHECK_TIME), weight_string(TABLE_COLLATION), weight_string(CREATE_OPTIONS), weight_string(TABLE_COMMENT) from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */", + "FieldQuery": "select TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT from information_schema.`tables` where 1 != 1", + "Query": "select distinct TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */", "SysTableTableSchema": "[VARCHAR(\"user\")]", "Table": "information_schema.`tables`" }, @@ -237,8 +236,8 @@ "Name": "main", "Sharded": false }, - "FieldQuery": "select TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT, weight_string(TABLE_CATALOG), weight_string(TABLE_SCHEMA), weight_string(TABLE_NAME), weight_string(TABLE_TYPE), weight_string(`ENGINE`), weight_string(`ROW_FORMAT`), weight_string(CREATE_TIME), weight_string(UPDATE_TIME), weight_string(CHECK_TIME), weight_string(TABLE_COLLATION), weight_string(CREATE_OPTIONS), weight_string(TABLE_COMMENT) from information_schema.`tables` where 1 != 1", - "Query": "select distinct TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT, weight_string(TABLE_CATALOG), weight_string(TABLE_SCHEMA), weight_string(TABLE_NAME), weight_string(TABLE_TYPE), weight_string(`ENGINE`), weight_string(`ROW_FORMAT`), weight_string(CREATE_TIME), weight_string(UPDATE_TIME), weight_string(CHECK_TIME), weight_string(TABLE_COLLATION), weight_string(CREATE_OPTIONS), weight_string(TABLE_COMMENT) from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */", + "FieldQuery": "select TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT from information_schema.`tables` where 1 != 1", + "Query": "select distinct TABLE_CATALOG, TABLE_SCHEMA, TABLE_NAME, TABLE_TYPE, `ENGINE`, VERSION, `ROW_FORMAT`, TABLE_ROWS, `AVG_ROW_LENGTH`, DATA_LENGTH, MAX_DATA_LENGTH, INDEX_LENGTH, DATA_FREE, `AUTO_INCREMENT`, CREATE_TIME, UPDATE_TIME, CHECK_TIME, TABLE_COLLATION, `CHECKSUM`, CREATE_OPTIONS, TABLE_COMMENT from information_schema.`tables` where table_schema = :__vtschemaname /* VARCHAR */", "SysTableTableSchema": "[VARCHAR(\"main\")]", "Table": "information_schema.`tables`" } diff --git a/go/vt/vtgate/planbuilder/testdata/union_cases.json b/go/vt/vtgate/planbuilder/testdata/union_cases.json index ed836bf207b..880bc1caa8b 100644 --- a/go/vt/vtgate/planbuilder/testdata/union_cases.json +++ b/go/vt/vtgate/planbuilder/testdata/union_cases.json @@ -630,9 +630,8 @@ "Instructions": { "OperatorType": "Distinct", "Collations": [ - "(0:1)" + "0: utf8mb4_0900_ai_ci" ], - "ResultColumns": 1, "Inputs": [ { "OperatorType": "Concatenate", @@ -644,8 +643,8 @@ "Name": "main", "Sharded": false }, - "FieldQuery": "select CHARACTER_SET_NAME, weight_string(CHARACTER_SET_NAME) from information_schema.CHARACTER_SETS where 1 != 1", - "Query": "select distinct CHARACTER_SET_NAME, weight_string(CHARACTER_SET_NAME) from information_schema.CHARACTER_SETS", + "FieldQuery": "select CHARACTER_SET_NAME from information_schema.CHARACTER_SETS where 1 != 1", + "Query": "select distinct CHARACTER_SET_NAME from information_schema.CHARACTER_SETS", "Table": "information_schema.CHARACTER_SETS" }, { @@ -655,8 +654,8 @@ "Name": "main", "Sharded": false }, - "FieldQuery": "select user_name, weight_string(user_name) from unsharded where 1 != 1", - "Query": "select distinct user_name, weight_string(user_name) from unsharded", + "FieldQuery": "select user_name from unsharded where 1 != 1", + "Query": "select distinct user_name from unsharded", "Table": "unsharded" } ] @@ -1121,7 +1120,7 @@ "Instructions": { "OperatorType": "Distinct", "Collations": [ - "0: binary" + "0" ], "Inputs": [ { @@ -1284,7 +1283,7 @@ "Instructions": { "OperatorType": "Distinct", "Collations": [ - "0: binary" + "0" ], "Inputs": [ { @@ -1509,10 +1508,9 @@ "Instructions": { "OperatorType": "Distinct", "Collations": [ - "(0:2)", - "(1:3)" + "0: utf8mb4_0900_ai_ci", + "1: utf8mb4_0900_ai_ci" ], - "ResultColumns": 2, "Inputs": [ { "OperatorType": "Concatenate", @@ -1524,14 +1522,14 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select 'b', 'c', weight_string('b'), weight_string('c') from `user` where 1 != 1", - "Query": "select distinct 'b', 'c', weight_string('b'), weight_string('c') from `user`", + "FieldQuery": "select 'b', 'c' from `user` where 1 != 1", + "Query": "select distinct 'b', 'c' from `user`", "Table": "`user`" }, { "OperatorType": "Join", "Variant": "Join", - "JoinColumnIndexes": "L:0,L:1,L:2,L:3", + "JoinColumnIndexes": "L:0,L:1", "TableName": "`user`_user_extra", "Inputs": [ { @@ -1541,8 +1539,8 @@ "Name": "user", "Sharded": true }, - "FieldQuery": "select `user`.id, `user`.`name`, weight_string(`user`.id), weight_string(`user`.`name`) from `user` where 1 != 1", - "Query": "select `user`.id, `user`.`name`, weight_string(`user`.id), weight_string(`user`.`name`) from `user`", + "FieldQuery": "select `user`.id, `user`.`name` from `user` where 1 != 1", + "Query": "select `user`.id, `user`.`name` from `user`", "Table": "`user`" }, { @@ -1627,7 +1625,7 @@ "Instructions": { "OperatorType": "Distinct", "Collations": [ - "0: binary" + "0" ], "Inputs": [ { diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 8d84c03fd55..8cdff1c780e 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -263,15 +263,6 @@ func (st *SemTable) TypeForExpr(e sqlparser.Expr) (sqltypes.Type, collations.ID, return -1, collations.Unknown, false } -// CollationForExpr returns the collation name of expressions in the query -func (st *SemTable) CollationForExpr(e sqlparser.Expr) collations.ID { - typ, found := st.ExprTypes[e] - if found { - return typ.Collation - } - return collations.Unknown -} - // NeedsWeightString returns true if the given expression needs weight_string to do safe comparisons func (st *SemTable) NeedsWeightString(e sqlparser.Expr) bool { typ, found := st.ExprTypes[e] diff --git a/go/vt/vttablet/tabletmanager/vdiff/utils.go b/go/vt/vttablet/tabletmanager/vdiff/utils.go index a6c711bae79..7c48ade9775 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/utils.go +++ b/go/vt/vttablet/tabletmanager/vdiff/utils.go @@ -41,9 +41,9 @@ func newMergeSorter(participants map[string]*shardStreamer, comparePKs []compare weightStringCol := -1 // if the collation is nil or unknown, use binary collation to compare as bytes if cpk.collation == nil { - ob[i] = engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, CollationID: collations.CollationBinaryID} + ob[i] = engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: -1, CollationID: collations.CollationBinaryID} } else { - ob[i] = engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, CollationID: cpk.collation.ID()} + ob[i] = engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: -1, CollationID: cpk.collation.ID()} } } return &engine.MergeSort{ @@ -64,7 +64,7 @@ func encodeString(in string) string { func pkColsToGroupByParams(pkCols []int) []*engine.GroupByParams { var res []*engine.GroupByParams for _, col := range pkCols { - res = append(res, &engine.GroupByParams{KeyCol: col, WeightStringCol: -1}) + res = append(res, &engine.GroupByParams{KeyCol: col, WeightStringCol: -1, Type: -1}) } return res } diff --git a/go/vt/wrangler/vdiff.go b/go/vt/wrangler/vdiff.go index 761412f3726..4cd46eb9d9c 100644 --- a/go/vt/wrangler/vdiff.go +++ b/go/vt/wrangler/vdiff.go @@ -767,7 +767,7 @@ func (df *vdiff) buildTablePlan(table *tabletmanagerdatapb.TableDefinition, quer func pkColsToGroupByParams(pkCols []int) []*engine.GroupByParams { var res []*engine.GroupByParams for _, col := range pkCols { - res = append(res, &engine.GroupByParams{KeyCol: col, WeightStringCol: -1}) + res = append(res, &engine.GroupByParams{KeyCol: col, WeightStringCol: -1, Type: -1}) } return res } @@ -783,9 +783,9 @@ func newMergeSorter(participants map[string]*shardStreamer, comparePKs []compare weightStringCol := -1 // if the collation is nil or unknown, use binary collation to compare as bytes if cpk.collation == nil { - ob = append(ob, engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, CollationID: collations.CollationBinaryID}) + ob = append(ob, engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: -1, CollationID: collations.CollationBinaryID}) } else { - ob = append(ob, engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, CollationID: cpk.collation.ID()}) + ob = append(ob, engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: -1, CollationID: cpk.collation.ID()}) } } return &engine.MergeSort{ From 2aee93a9c4b37e2a6af9f68abb28843cf084495e Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Mon, 17 Jul 2023 14:36:24 +0200 Subject: [PATCH 08/13] vtexplain: Update test output for improved query support Signed-off-by: Dirkjan Bussink --- .../testdata/multi-output/selectsharded-output.txt | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt b/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt index c81298be981..7ae20ca1a7f 100644 --- a/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt +++ b/go/vt/vtexplain/testdata/multi-output/selectsharded-output.txt @@ -137,10 +137,10 @@ select name from user where exists (select id from t1) /* non-correlated subquer ---------------------------------------------------------------------- select * from name_info order by info /* select * and order by varchar column */ -1 ks_sharded/-40: select `name`, info, weight_string(info) from name_info order by info asc limit 10001 /* select * and order by varchar column */ -1 ks_sharded/40-80: select `name`, info, weight_string(info) from name_info order by info asc limit 10001 /* select * and order by varchar column */ -1 ks_sharded/80-c0: select `name`, info, weight_string(info) from name_info order by info asc limit 10001 /* select * and order by varchar column */ -1 ks_sharded/c0-: select `name`, info, weight_string(info) from name_info order by info asc limit 10001 /* select * and order by varchar column */ +1 ks_sharded/-40: select `name`, info from name_info order by info asc limit 10001 /* select * and order by varchar column */ +1 ks_sharded/40-80: select `name`, info from name_info order by info asc limit 10001 /* select * and order by varchar column */ +1 ks_sharded/80-c0: select `name`, info from name_info order by info asc limit 10001 /* select * and order by varchar column */ +1 ks_sharded/c0-: select `name`, info from name_info order by info asc limit 10001 /* select * and order by varchar column */ ---------------------------------------------------------------------- select distinct(name) from user where id = 1 /* select distinct */ From 12d3cc9afa5fbaddd6e2c4c258b1e55ced7eac1d Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Mon, 17 Jul 2023 15:00:42 +0200 Subject: [PATCH 09/13] Fix another test issue Signed-off-by: Dirkjan Bussink --- go/vt/wrangler/vdiff_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/go/vt/wrangler/vdiff_test.go b/go/vt/wrangler/vdiff_test.go index 500cc9a4080..23414fb7a38 100644 --- a/go/vt/wrangler/vdiff_test.go +++ b/go/vt/wrangler/vdiff_test.go @@ -404,7 +404,7 @@ func TestVDiffPlanSuccess(t *testing.T) { engine.NewAggregateParam(opcode.AggregateSum, 2, ""), engine.NewAggregateParam(opcode.AggregateSum, 3, ""), }, - GroupByKeys: []*engine.GroupByParams{{KeyCol: 0, WeightStringCol: -1}}, + GroupByKeys: []*engine.GroupByParams{{KeyCol: 0, WeightStringCol: -1, Type: -1}}, Input: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), }, targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), From e382a3f7e28cc03ffe710c243894e6ed96443805 Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Mon, 17 Jul 2023 17:03:44 +0200 Subject: [PATCH 10/13] Fix license and do cleanup Signed-off-by: Dirkjan Bussink --- go/mysql/hex/hex.go | 16 ++++++++++++++++ go/vt/vtgate/engine/route.go | 3 --- go/vt/vtgate/planbuilder/horizon_planning.go | 4 ---- .../planbuilder/operators/queryprojection.go | 3 --- 4 files changed, 16 insertions(+), 10 deletions(-) diff --git a/go/mysql/hex/hex.go b/go/mysql/hex/hex.go index 4e6cf103905..941cc08e5b5 100644 --- a/go/mysql/hex/hex.go +++ b/go/mysql/hex/hex.go @@ -1,3 +1,19 @@ +/* +Copyright 2023 The Vitess Authors. + +Licensed under the Apache License, Version 2.0 (the "License"); +you may not use this file except in compliance with the License. +You may obtain a copy of the License at + + http://www.apache.org/licenses/LICENSE-2.0 + +Unless required by applicable law or agreed to in writing, software +distributed under the License is distributed on an "AS IS" BASIS, +WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +See the License for the specific language governing permissions and +limitations under the License. +*/ + package hex import ( diff --git a/go/vt/vtgate/engine/route.go b/go/vt/vtgate/engine/route.go index 25619ebfb82..bcbd51609ef 100644 --- a/go/vt/vtgate/engine/route.go +++ b/go/vt/vtgate/engine/route.go @@ -143,9 +143,6 @@ func (obp OrderByParams) String() string { val += " ASC" } - if obp.Type != -1 && obp.CollationID == collations.Unknown { - panic("OrderByParams: collationID is unknown but type is not unknown") - } if sqltypes.IsText(obp.Type) && obp.CollationID != collations.Unknown { collation := obp.CollationID.Get() val += " COLLATE " + collation.Name() diff --git a/go/vt/vtgate/planbuilder/horizon_planning.go b/go/vt/vtgate/planbuilder/horizon_planning.go index 598895d0b05..e37ba94b8d4 100644 --- a/go/vt/vtgate/planbuilder/horizon_planning.go +++ b/go/vt/vtgate/planbuilder/horizon_planning.go @@ -19,7 +19,6 @@ package planbuilder import ( "fmt" - "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" @@ -482,9 +481,6 @@ func addColumnsToOA( aggr.WCol = o.wsCol aggr.Original = a.Original aggr.Type, aggr.CollationID, _ = ctx.SemTable.TypeForExpr(a.Func.GetArg()) - if aggr.Type != -1 && aggr.CollationID == collations.Unknown { - panic("unexpected: aggregate function without collation") - } oa.aggregates = append(oa.aggregates, aggr) } lastOffset := distinctOffsets[len(distinctOffsets)-1] diff --git a/go/vt/vtgate/planbuilder/operators/queryprojection.go b/go/vt/vtgate/planbuilder/operators/queryprojection.go index 40c4f6293bc..25006a1ef79 100644 --- a/go/vt/vtgate/planbuilder/operators/queryprojection.go +++ b/go/vt/vtgate/planbuilder/operators/queryprojection.go @@ -122,9 +122,6 @@ func (aggr Aggr) GetTypeCollation(ctx *plancontext.PlanningContext) (sqltypes.Ty switch aggr.OpCode { case opcode.AggregateMin, opcode.AggregateMax, opcode.AggregateSumDistinct, opcode.AggregateCountDistinct: typ, col, _ := ctx.SemTable.TypeForExpr(aggr.Func.GetArg()) - if typ != -1 && col == collations.Unknown { - panic("unexpected: aggregate function without collation") - } return typ, col } From f780e6090803fe1267322df1bd2dc7f0dc0082cd Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Tue, 18 Jul 2023 12:51:00 +0200 Subject: [PATCH 11/13] Add constant for sqltypes.Unknown Signed-off-by: Dirkjan Bussink --- go/sqltypes/type.go | 1 + go/vt/sqlparser/ast_funcs.go | 2 +- go/vt/vtgate/engine/aggregations.go | 2 +- go/vt/vtgate/engine/delete_test.go | 4 ++-- go/vt/vtgate/engine/distinct_test.go | 4 ++-- go/vt/vtgate/engine/route_test.go | 2 +- go/vt/vtgate/engine/set_test.go | 2 +- go/vt/vtgate/engine/update_test.go | 2 +- go/vt/vtgate/evalengine/expr_column.go | 2 +- go/vt/vtgate/evalengine/expr_column_test.go | 4 ++-- go/vt/vtgate/evalengine/translate.go | 6 +++--- go/vt/vtgate/planbuilder/expression_converter.go | 3 ++- go/vt/vtgate/planbuilder/expression_converter_test.go | 4 +++- go/vt/vtgate/planbuilder/grouping.go | 5 +++-- go/vt/vtgate/planbuilder/memory_sort.go | 2 +- go/vt/vtgate/planbuilder/operator_transformers.go | 2 +- go/vt/vtgate/planbuilder/operators/queryprojection.go | 4 ++-- go/vt/vtgate/planbuilder/ordering.go | 3 ++- go/vt/vtgate/planbuilder/route.go | 3 ++- go/vt/vtgate/semantics/semantic_state.go | 2 +- go/vt/vttablet/tabletmanager/vdiff/utils.go | 6 +++--- go/vt/wrangler/vdiff.go | 6 +++--- go/vt/wrangler/vdiff_test.go | 2 +- 23 files changed, 40 insertions(+), 33 deletions(-) diff --git a/go/sqltypes/type.go b/go/sqltypes/type.go index af0c8c42c5a..eeaa4e9ddf6 100644 --- a/go/sqltypes/type.go +++ b/go/sqltypes/type.go @@ -136,6 +136,7 @@ func IsNull(t querypb.Type) bool { // switch statements for those who want to cover types // by their category. const ( + Unknown = -1 Null = querypb.Type_NULL_TYPE Int8 = querypb.Type_INT8 Uint8 = querypb.Type_UINT8 diff --git a/go/vt/sqlparser/ast_funcs.go b/go/vt/sqlparser/ast_funcs.go index 96ebea71ec6..0b7a0ea4541 100644 --- a/go/vt/sqlparser/ast_funcs.go +++ b/go/vt/sqlparser/ast_funcs.go @@ -533,7 +533,7 @@ func NewTimestampLiteral(in string) *Literal { // NewArgument builds a new ValArg. func NewArgument(in string) *Argument { - return &Argument{Name: in, Type: -1} + return &Argument{Name: in, Type: sqltypes.Unknown} } func parseBindVariable(yylex yyLexer, bvar string) *Argument { diff --git a/go/vt/vtgate/engine/aggregations.go b/go/vt/vtgate/engine/aggregations.go index a6f2f01fcfb..725e1882757 100644 --- a/go/vt/vtgate/engine/aggregations.go +++ b/go/vt/vtgate/engine/aggregations.go @@ -61,7 +61,7 @@ func NewAggregateParam(opcode AggregateOpcode, col int, alias string) *Aggregate Col: col, Alias: alias, WCol: -1, - Type: -1, + Type: sqltypes.Unknown, } if opcode.NeedsComparableValues() { out.KeyCol = col diff --git a/go/vt/vtgate/engine/delete_test.go b/go/vt/vtgate/engine/delete_test.go index 10f5bcb3449..8ee00edb2de 100644 --- a/go/vt/vtgate/engine/delete_test.go +++ b/go/vt/vtgate/engine/delete_test.go @@ -90,7 +90,7 @@ func TestDeleteEqual(t *testing.T) { }) // Failure case - expr := evalengine.NewBindVar("aa", -1, collations.CollationBinaryID) + expr := evalengine.NewBindVar("aa", sqltypes.Unknown, collations.Unknown) del.Values = []evalengine.Expr{expr} _, err = del.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false) require.EqualError(t, err, "query arguments missing for aa") @@ -122,7 +122,7 @@ func TestDeleteEqualMultiCol(t *testing.T) { }) // Failure case - expr := evalengine.NewBindVar("aa", -1, collations.CollationBinaryID) + expr := evalengine.NewBindVar("aa", sqltypes.Unknown, collations.Unknown) del.Values = []evalengine.Expr{expr} _, err = del.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false) require.EqualError(t, err, "query arguments missing for aa") diff --git a/go/vt/vtgate/engine/distinct_test.go b/go/vt/vtgate/engine/distinct_test.go index 3fbf00835ca..e120c60bd3e 100644 --- a/go/vt/vtgate/engine/distinct_test.go +++ b/go/vt/vtgate/engine/distinct_test.go @@ -134,7 +134,7 @@ func TestWeightStringFallBack(t *testing.T) { checkCols := []CheckCol{{ Col: 0, WsCol: &offsetOne, - Type: -1, + Type: sqltypes.Unknown, Collation: collations.Unknown, }} input := r("myid|weightstring(myid)", @@ -160,7 +160,7 @@ func TestWeightStringFallBack(t *testing.T) { utils.MustMatch(t, []CheckCol{{ Col: 0, WsCol: &offsetOne, - Type: -1, + Type: sqltypes.Unknown, Collation: collations.Unknown, }}, distinct.CheckCols, "checkCols should not be updated") } diff --git a/go/vt/vtgate/engine/route_test.go b/go/vt/vtgate/engine/route_test.go index fcfed19c68e..7317ac913fb 100644 --- a/go/vt/vtgate/engine/route_test.go +++ b/go/vt/vtgate/engine/route_test.go @@ -1148,7 +1148,7 @@ func TestRouteSortCollation(t *testing.T) { t.Run("Error when Unknown Collation", func(t *testing.T) { sel.OrderBy = []OrderByParams{{ Col: 0, - Type: -1, + Type: sqltypes.Unknown, CollationID: collations.Unknown, }} diff --git a/go/vt/vtgate/engine/set_test.go b/go/vt/vtgate/engine/set_test.go index a8e8aba2043..62ffa42b8d6 100644 --- a/go/vt/vtgate/engine/set_test.go +++ b/go/vt/vtgate/engine/set_test.go @@ -108,7 +108,7 @@ func TestSetTable(t *testing.T) { setOps: []SetOp{ &UserDefinedVariable{ Name: "x", - Expr: evalengine.NewColumn(0, -1, collations.CollationBinaryID), + Expr: evalengine.NewColumn(0, sqltypes.Unknown, collations.Unknown), }, }, qr: []*sqltypes.Result{sqltypes.MakeTestResult( diff --git a/go/vt/vtgate/engine/update_test.go b/go/vt/vtgate/engine/update_test.go index 59adebfa1e6..f8d091879fb 100644 --- a/go/vt/vtgate/engine/update_test.go +++ b/go/vt/vtgate/engine/update_test.go @@ -94,7 +94,7 @@ func TestUpdateEqual(t *testing.T) { }) // Failure case - upd.Values = []evalengine.Expr{evalengine.NewBindVar("aa", -1, collations.CollationBinaryID)} + upd.Values = []evalengine.Expr{evalengine.NewBindVar("aa", sqltypes.Unknown, collations.Unknown)} _, err = upd.TryExecute(context.Background(), vc, map[string]*querypb.BindVariable{}, false) require.EqualError(t, err, `query arguments missing for aa`) } diff --git a/go/vt/vtgate/evalengine/expr_column.go b/go/vt/vtgate/evalengine/expr_column.go index e0212bb5b28..bf3129bbb0a 100644 --- a/go/vt/vtgate/evalengine/expr_column.go +++ b/go/vt/vtgate/evalengine/expr_column.go @@ -55,7 +55,7 @@ func (c *Column) typeof(env *ExpressionEnv, fields []*querypb.Field) (sqltypes.T if c.typed() { return c.Type, flagNullable } - return -1, flagAmbiguousType + return sqltypes.Unknown, flagAmbiguousType } func (column *Column) compile(c *compiler) (ctype, error) { diff --git a/go/vt/vtgate/evalengine/expr_column_test.go b/go/vt/vtgate/evalengine/expr_column_test.go index 6a34a7b48bc..fbe45d6027c 100644 --- a/go/vt/vtgate/evalengine/expr_column_test.go +++ b/go/vt/vtgate/evalengine/expr_column_test.go @@ -43,7 +43,7 @@ func TestTypeOf(t *testing.T) { fields := []*querypb.Field{field1, field2} c := &Column{ - Type: -1, + Type: sqltypes.Unknown, } env.Row = sqltypes.Row{sqltypes.NewInt64(10)} @@ -67,7 +67,7 @@ func TestTypeOf(t *testing.T) { t.Run("Check when offset is out of bounds", func(t *testing.T) { c.Offset = 10 typ, flag := c.typeof(env, fields) - if typ != -1 || flag != flagAmbiguousType { + if typ != sqltypes.Unknown || flag != flagAmbiguousType { t.Errorf("typeof() failed, expected -1 and flagAmbiguousType, got %v and %v", typ, flag) } }) diff --git a/go/vt/vtgate/evalengine/translate.go b/go/vt/vtgate/evalengine/translate.go index 8089afde425..998827db095 100644 --- a/go/vt/vtgate/evalengine/translate.go +++ b/go/vt/vtgate/evalengine/translate.go @@ -201,7 +201,7 @@ func (ast *astCompiler) translateBindVar(arg *sqlparser.Argument) (Expr, error) } func (ast *astCompiler) translateColOffset(col *sqlparser.Offset) (Expr, error) { - var typ querypb.Type = -1 + var typ sqltypes.Type = sqltypes.Unknown var coll collations.ID if ast.cfg.ResolveType != nil { typ, coll, _ = ast.cfg.ResolveType(col.Original) @@ -225,7 +225,7 @@ func (ast *astCompiler) translateColName(colname *sqlparser.ColName) (Expr, erro if err != nil { return nil, err } - var typ querypb.Type = -1 + var typ sqltypes.Type = sqltypes.Unknown var coll collations.ID if ast.cfg.ResolveType != nil { typ, coll, _ = ast.cfg.ResolveType(colname) @@ -640,5 +640,5 @@ func (fields FieldResolver) Type(expr sqlparser.Expr) (sqltypes.Type, collations } } } - return -1, collations.Unknown, false + return sqltypes.Unknown, collations.Unknown, false } diff --git a/go/vt/vtgate/planbuilder/expression_converter.go b/go/vt/vtgate/planbuilder/expression_converter.go index e6972f5acd4..f100d0d93e0 100644 --- a/go/vt/vtgate/planbuilder/expression_converter.go +++ b/go/vt/vtgate/planbuilder/expression_converter.go @@ -20,6 +20,7 @@ import ( "fmt" "strings" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/vtgate/planbuilder/plancontext" "vitess.io/vitess/go/mysql/collations" @@ -86,7 +87,7 @@ func (ec *expressionConverter) convert(astExpr sqlparser.Expr, boolean, identifi if !strings.Contains(err.Error(), evalengine.ErrTranslateExprNotSupported) { return nil, err } - evalExpr = evalengine.NewColumn(len(ec.tabletExpressions), -1, collations.Default()) + evalExpr = evalengine.NewColumn(len(ec.tabletExpressions), sqltypes.Unknown, collations.Unknown) ec.tabletExpressions = append(ec.tabletExpressions, astExpr) } return evalExpr, nil diff --git a/go/vt/vtgate/planbuilder/expression_converter_test.go b/go/vt/vtgate/planbuilder/expression_converter_test.go index 8bc8100ebc6..e59df3c7fd1 100644 --- a/go/vt/vtgate/planbuilder/expression_converter_test.go +++ b/go/vt/vtgate/planbuilder/expression_converter_test.go @@ -21,6 +21,8 @@ import ( "github.com/stretchr/testify/require" + "vitess.io/vitess/go/sqltypes" + "vitess.io/vitess/go/mysql/collations" "vitess.io/vitess/go/vt/sqlparser" @@ -42,7 +44,7 @@ func TestConversion(t *testing.T) { expressionsOut: e(evalengine.NewLiteralInt(1)), }, { expressionsIn: "@@foo", - expressionsOut: e(evalengine.NewColumn(0, -1, collations.Default())), + expressionsOut: e(evalengine.NewColumn(0, sqltypes.Unknown, collations.Unknown)), }} for _, tc := range queries { diff --git a/go/vt/vtgate/planbuilder/grouping.go b/go/vt/vtgate/planbuilder/grouping.go index 5af57b0a2e8..a442d6b23fa 100644 --- a/go/vt/vtgate/planbuilder/grouping.go +++ b/go/vt/vtgate/planbuilder/grouping.go @@ -19,6 +19,7 @@ package planbuilder import ( "fmt" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine" @@ -78,7 +79,7 @@ func planGroupBy(pb *primitiveBuilder, input logicalPlan, groupBy sqlparser.Grou default: return nil, vterrors.VT12001("in scatter query: only simple references are allowed") } - node.groupByKeys = append(node.groupByKeys, &engine.GroupByParams{KeyCol: colNumber, WeightStringCol: -1, Type: -1, FromGroupBy: true}) + node.groupByKeys = append(node.groupByKeys, &engine.GroupByParams{KeyCol: colNumber, WeightStringCol: -1, Type: sqltypes.Unknown, FromGroupBy: true}) } // Append the distinct aggregate if any. if node.extraDistinct != nil { @@ -111,7 +112,7 @@ func planDistinct(input logicalPlan) (logicalPlan, error) { if rc.column.Origin() == node { return newDistinctV3(node), nil } - node.groupByKeys = append(node.groupByKeys, &engine.GroupByParams{KeyCol: i, WeightStringCol: -1, Type: -1, FromGroupBy: false}) + node.groupByKeys = append(node.groupByKeys, &engine.GroupByParams{KeyCol: i, WeightStringCol: -1, Type: sqltypes.Unknown, FromGroupBy: false}) } newInput, err := planDistinct(node.input) if err != nil { diff --git a/go/vt/vtgate/planbuilder/memory_sort.go b/go/vt/vtgate/planbuilder/memory_sort.go index 59c1b103382..81abb6f3a53 100644 --- a/go/vt/vtgate/planbuilder/memory_sort.go +++ b/go/vt/vtgate/planbuilder/memory_sort.go @@ -95,7 +95,7 @@ func newMemorySort(plan logicalPlan, orderBy v3OrderBy) (*memorySort, error) { Desc: order.Direction == sqlparser.DescOrder, StarColFixedIndex: colNumber, FromGroupBy: order.fromGroupBy, - Type: -1, + Type: sqltypes.Unknown, CollationID: collations.Unknown, } ms.eMemorySort.OrderBy = append(ms.eMemorySort.OrderBy, ob) diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 7c163b034c0..66171ae5794 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -182,7 +182,7 @@ func transformProjection(ctx *plancontext.PlanningContext, op *operators.Project case operators.Offset: t, found := ctx.SemTable.ExprTypes[e.Expr] if !found { - return evalengine.NewColumn(e.Offset, -1, collations.Unknown) + return evalengine.NewColumn(e.Offset, sqltypes.Unknown, collations.Unknown) } return evalengine.NewColumn(e.Offset, t.Type, t.Collation) default: diff --git a/go/vt/vtgate/planbuilder/operators/queryprojection.go b/go/vt/vtgate/planbuilder/operators/queryprojection.go index 25006a1ef79..58f447901f7 100644 --- a/go/vt/vtgate/planbuilder/operators/queryprojection.go +++ b/go/vt/vtgate/planbuilder/operators/queryprojection.go @@ -117,7 +117,7 @@ func (aggr Aggr) NeedsWeightString(ctx *plancontext.PlanningContext) bool { func (aggr Aggr) GetTypeCollation(ctx *plancontext.PlanningContext) (sqltypes.Type, collations.ID) { if aggr.Func == nil { - return -1, collations.Unknown + return sqltypes.Unknown, collations.Unknown } switch aggr.OpCode { case opcode.AggregateMin, opcode.AggregateMax, opcode.AggregateSumDistinct, opcode.AggregateCountDistinct: @@ -125,7 +125,7 @@ func (aggr Aggr) GetTypeCollation(ctx *plancontext.PlanningContext) (sqltypes.Ty return typ, col } - return -1, collations.Unknown + return sqltypes.Unknown, collations.Unknown } // NewGroupBy creates a new group by from the given fields. diff --git a/go/vt/vtgate/planbuilder/ordering.go b/go/vt/vtgate/planbuilder/ordering.go index 7fbe146d657..01764ed9018 100644 --- a/go/vt/vtgate/planbuilder/ordering.go +++ b/go/vt/vtgate/planbuilder/ordering.go @@ -17,6 +17,7 @@ import ( "fmt" "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine" @@ -342,7 +343,7 @@ func planRouteOrdering(orderBy v3OrderBy, node *route) (logicalPlan, error) { Desc: order.Direction == sqlparser.DescOrder, StarColFixedIndex: starColFixedIndex, FromGroupBy: order.fromGroupBy, - Type: -1, + Type: sqltypes.Unknown, CollationID: collations.Unknown, } node.eroute.OrderBy = append(node.eroute.OrderBy, ob) diff --git a/go/vt/vtgate/planbuilder/route.go b/go/vt/vtgate/planbuilder/route.go index cf904b0191f..ad48c9c4c31 100644 --- a/go/vt/vtgate/planbuilder/route.go +++ b/go/vt/vtgate/planbuilder/route.go @@ -21,6 +21,7 @@ import ( "strconv" "vitess.io/vitess/go/mysql/collations" + "vitess.io/vitess/go/sqltypes" "vitess.io/vitess/go/vt/sqlparser" "vitess.io/vitess/go/vt/vterrors" "vitess.io/vitess/go/vt/vtgate/engine" @@ -250,7 +251,7 @@ func (rb *route) procureValues(plan logicalPlan, jt *jointab, val sqlparser.Expr return evalengine.NewTupleExpr(exprs...), nil case *sqlparser.ColName: joinVar := jt.Procure(plan, typedVal, rb.Order()) - return evalengine.NewBindVar(joinVar, -1, collations.Unknown), nil + return evalengine.NewBindVar(joinVar, sqltypes.Unknown, collations.Unknown), nil default: return evalengine.Translate(typedVal, nil) } diff --git a/go/vt/vtgate/semantics/semantic_state.go b/go/vt/vtgate/semantics/semantic_state.go index 8cdff1c780e..3058f0608cc 100644 --- a/go/vt/vtgate/semantics/semantic_state.go +++ b/go/vt/vtgate/semantics/semantic_state.go @@ -260,7 +260,7 @@ func (st *SemTable) TypeForExpr(e sqlparser.Expr) (sqltypes.Type, collations.ID, if typ, found := st.ExprTypes[e]; found { return typ.Type, typ.Collation, true } - return -1, collations.Unknown, false + return sqltypes.Unknown, collations.Unknown, false } // NeedsWeightString returns true if the given expression needs weight_string to do safe comparisons diff --git a/go/vt/vttablet/tabletmanager/vdiff/utils.go b/go/vt/vttablet/tabletmanager/vdiff/utils.go index 7c48ade9775..34a9b3f164b 100644 --- a/go/vt/vttablet/tabletmanager/vdiff/utils.go +++ b/go/vt/vttablet/tabletmanager/vdiff/utils.go @@ -41,9 +41,9 @@ func newMergeSorter(participants map[string]*shardStreamer, comparePKs []compare weightStringCol := -1 // if the collation is nil or unknown, use binary collation to compare as bytes if cpk.collation == nil { - ob[i] = engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: -1, CollationID: collations.CollationBinaryID} + ob[i] = engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: sqltypes.Unknown, CollationID: collations.CollationBinaryID} } else { - ob[i] = engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: -1, CollationID: cpk.collation.ID()} + ob[i] = engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: sqltypes.Unknown, CollationID: cpk.collation.ID()} } } return &engine.MergeSort{ @@ -64,7 +64,7 @@ func encodeString(in string) string { func pkColsToGroupByParams(pkCols []int) []*engine.GroupByParams { var res []*engine.GroupByParams for _, col := range pkCols { - res = append(res, &engine.GroupByParams{KeyCol: col, WeightStringCol: -1, Type: -1}) + res = append(res, &engine.GroupByParams{KeyCol: col, WeightStringCol: -1, Type: sqltypes.Unknown}) } return res } diff --git a/go/vt/wrangler/vdiff.go b/go/vt/wrangler/vdiff.go index 4cd46eb9d9c..119a1bc5367 100644 --- a/go/vt/wrangler/vdiff.go +++ b/go/vt/wrangler/vdiff.go @@ -767,7 +767,7 @@ func (df *vdiff) buildTablePlan(table *tabletmanagerdatapb.TableDefinition, quer func pkColsToGroupByParams(pkCols []int) []*engine.GroupByParams { var res []*engine.GroupByParams for _, col := range pkCols { - res = append(res, &engine.GroupByParams{KeyCol: col, WeightStringCol: -1, Type: -1}) + res = append(res, &engine.GroupByParams{KeyCol: col, WeightStringCol: -1, Type: sqltypes.Unknown}) } return res } @@ -783,9 +783,9 @@ func newMergeSorter(participants map[string]*shardStreamer, comparePKs []compare weightStringCol := -1 // if the collation is nil or unknown, use binary collation to compare as bytes if cpk.collation == nil { - ob = append(ob, engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: -1, CollationID: collations.CollationBinaryID}) + ob = append(ob, engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: sqltypes.Unknown, CollationID: collations.CollationBinaryID}) } else { - ob = append(ob, engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: -1, CollationID: cpk.collation.ID()}) + ob = append(ob, engine.OrderByParams{Col: cpk.colIndex, WeightStringCol: weightStringCol, Type: sqltypes.Unknown, CollationID: cpk.collation.ID()}) } } return &engine.MergeSort{ diff --git a/go/vt/wrangler/vdiff_test.go b/go/vt/wrangler/vdiff_test.go index 23414fb7a38..b9ca2d0dbf2 100644 --- a/go/vt/wrangler/vdiff_test.go +++ b/go/vt/wrangler/vdiff_test.go @@ -404,7 +404,7 @@ func TestVDiffPlanSuccess(t *testing.T) { engine.NewAggregateParam(opcode.AggregateSum, 2, ""), engine.NewAggregateParam(opcode.AggregateSum, 3, ""), }, - GroupByKeys: []*engine.GroupByParams{{KeyCol: 0, WeightStringCol: -1, Type: -1}}, + GroupByKeys: []*engine.GroupByParams{{KeyCol: 0, WeightStringCol: -1, Type: sqltypes.Unknown}}, Input: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), }, targetPrimitive: newMergeSorter(nil, []compareColInfo{{0, collations.Collation(nil), true}}), From 7afedbf52ee5efd257ab1d4d466d554d7c23104e Mon Sep 17 00:00:00 2001 From: Dirkjan Bussink Date: Tue, 18 Jul 2023 17:04:18 +0200 Subject: [PATCH 12/13] Small code cleanup Signed-off-by: Dirkjan Bussink --- go/vt/vtgate/planbuilder/operator_transformers.go | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) diff --git a/go/vt/vtgate/planbuilder/operator_transformers.go b/go/vt/vtgate/planbuilder/operator_transformers.go index 66171ae5794..fbc7983f1f4 100644 --- a/go/vt/vtgate/planbuilder/operator_transformers.go +++ b/go/vt/vtgate/planbuilder/operator_transformers.go @@ -180,11 +180,8 @@ func transformProjection(ctx *plancontext.PlanningContext, op *operators.Project case operators.Eval: return e.EExpr case operators.Offset: - t, found := ctx.SemTable.ExprTypes[e.Expr] - if !found { - return evalengine.NewColumn(e.Offset, sqltypes.Unknown, collations.Unknown) - } - return evalengine.NewColumn(e.Offset, t.Type, t.Collation) + typ, col, _ := ctx.SemTable.TypeForExpr(e.Expr) + return evalengine.NewColumn(e.Offset, typ, col) default: failed = true return nil From afcebb2c2a53c5e24c99c6422f5a68b783ce5116 Mon Sep 17 00:00:00 2001 From: Harshit Gangal Date: Wed, 19 Jul 2023 12:23:22 +0530 Subject: [PATCH 13/13] fix fake test result to not have weight_string as it is not present in the query send down Signed-off-by: Harshit Gangal --- go/vt/vtgate/executor_select_test.go | 2 -- 1 file changed, 2 deletions(-) diff --git a/go/vt/vtgate/executor_select_test.go b/go/vt/vtgate/executor_select_test.go index 91c9e394d21..91e102b6a49 100644 --- a/go/vt/vtgate/executor_select_test.go +++ b/go/vt/vtgate/executor_select_test.go @@ -1822,7 +1822,6 @@ func TestSelectScatterOrderByVarChar(t *testing.T) { // This will allow us to test that cross-shard ordering // still works correctly. sqltypes.NewVarChar(fmt.Sprintf("%d", i%4)), - sqltypes.NewVarBinary(fmt.Sprintf("%d", i%4)), }}, }}) conns = append(conns, sbc) @@ -1941,7 +1940,6 @@ func TestStreamSelectScatterOrderByVarChar(t *testing.T) { Rows: [][]sqltypes.Value{{ sqltypes.NewInt32(1), sqltypes.NewVarChar(fmt.Sprintf("%d", i%4)), - sqltypes.NewVarBinary(fmt.Sprintf("%d", i%4)), }}, }}) conns = append(conns, sbc)