From a53dd9f1cfcd5a8c0dd729f99205a3b1f84bd8f9 Mon Sep 17 00:00:00 2001 From: tangenta Date: Fri, 12 Nov 2021 19:55:21 +0800 Subject: [PATCH 01/15] expression: convert charset by wrapping implicit builtin function --- expression/builtin.go | 1 + expression/builtin_cast.go | 23 ++++++++ expression/builtin_charset.go | 91 ++++++++++++++++++++++++++++++++ expression/builtin_string.go | 13 +---- expression/builtin_string_vec.go | 9 +--- expression/distsql_builtin.go | 7 +-- 6 files changed, 119 insertions(+), 25 deletions(-) create mode 100644 expression/builtin_charset.go diff --git a/expression/builtin.go b/expression/builtin.go index 2ec8672c5cce1..9860489da6ea3 100644 --- a/expression/builtin.go +++ b/expression/builtin.go @@ -140,6 +140,7 @@ func newBaseBuiltinFuncWithTp(ctx sessionctx.Context, funcName string, args []Ex args[i] = WrapWithCastAsDecimal(ctx, args[i]) case types.ETString: args[i] = WrapWithCastAsString(ctx, args[i]) + args[i] = WrapWithCharsetConvert(ctx, args[i], funcName) case types.ETDatetime: args[i] = WrapWithCastAsTime(ctx, args[i], types.NewFieldType(mysql.TypeDatetime)) case types.ETTimestamp: diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go index 407982af11e31..71ef316d4d035 100644 --- a/expression/builtin_cast.go +++ b/expression/builtin_cast.go @@ -29,6 +29,7 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/parser/ast" + "github.com/pingcap/tidb/parser/charset" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/parser/terror" @@ -1907,6 +1908,28 @@ func WrapWithCastAsDecimal(ctx sessionctx.Context, expr Expression) Expression { return BuildCastFunction(ctx, expr, tp) } +// WrapWithCastAsStringWithTp wraps `expr` with converting charset. +func WrapWithCharsetConvert(ctx sessionctx.Context, expr Expression, funcName string) Expression { + retTp := expr.GetType() + if _, err := charset.GetDefaultCollationLegacy(retTp.Charset); err != nil { + const charsetConv = "charset_convert" + switch funcName { + case ast.Hex: + bf, err := newBaseBuiltinFunc(ctx, charsetConv, []Expression{expr}, retTp.EvalType()) + if err != nil { + return expr + } + chsSig := &builtinCharsetConvSig{bf} + return &ScalarFunction{ + FuncName: model.NewCIStr(charsetConv), + RetType: retTp, + Function: chsSig, + } + } + } + return expr +} + // WrapWithCastAsString wraps `expr` with `cast` if the return type of expr is // not type string, otherwise, returns `expr` directly. func WrapWithCastAsString(ctx sessionctx.Context, expr Expression) Expression { diff --git a/expression/builtin_charset.go b/expression/builtin_charset.go new file mode 100644 index 0000000000000..0c994e604e998 --- /dev/null +++ b/expression/builtin_charset.go @@ -0,0 +1,91 @@ +// Copyright 2021 PingCAP, Inc. +// +// Licensed under the Apache License, Version 2.0 (the "License"); +// you may not use this file except in compliance with the License. +// You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, software +// distributed under the License is distributed on an "AS IS" BASIS, +// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +// See the License for the specific language governing permissions and +// limitations under the License. + +package expression + +import ( + "github.com/pingcap/tidb/parser/charset" + "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/util/chunk" +) + +var _ builtinFunc = &builtinCharsetConvSig{} + +type builtinCharsetConvSig struct { + baseBuiltinFunc +} + +func (b *builtinCharsetConvSig) Clone() builtinFunc { + newSig := &builtinCharsetConvSig{} + newSig.cloneFrom(&b.baseBuiltinFunc) + return newSig +} + +func (b *builtinCharsetConvSig) evalString(row chunk.Row) (res string, isNull bool, err error) { + val, isNull, err := b.args[0].EvalString(b.ctx, row) + if isNull || err != nil { + return res, isNull, err + } + tp := b.args[0].GetType() + enc := charset.NewEncoding(tp.Charset) + res, err = enc.EncodeString(val) + if err != nil { + return res, false, err + } + res, err = types.ProduceStrWithSpecifiedTp(res, b.tp, b.ctx.GetSessionVars().StmtCtx, false) + if err != nil { + return res, false, err + } + return padZeroForBinaryType(res, b.tp, b.ctx) +} + +func (b *builtinCharsetConvSig) vectorized() bool { + return true +} + +func (b *builtinCharsetConvSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { + n := input.NumRows() + buf, err := b.bufAllocator.get() + if err != nil { + return err + } + defer b.bufAllocator.put(buf) + if err := b.args[0].VecEvalString(b.ctx, input, buf); err != nil { + return err + } + result.ReserveString(n) + for i := 0; i < n; i++ { + var str string + if buf.IsNull(i) { + result.AppendNull() + continue + } + str = buf.GetString(i) + str, err = types.ProduceStrWithSpecifiedTp(str, b.tp, b.ctx.GetSessionVars().StmtCtx, false) + if err != nil { + return err + } + var d bool + str, d, err = padZeroForBinaryType(str, b.tp, b.ctx) + if err != nil { + return err + } + if d { + result.AppendNull() + } else { + result.AppendString(str) + } + } + return nil +} diff --git a/expression/builtin_string.go b/expression/builtin_string.go index 13733abd4cc10..517f07927ed27 100644 --- a/expression/builtin_string.go +++ b/expression/builtin_string.go @@ -1664,7 +1664,7 @@ func (c *hexFunctionClass) getFunction(ctx sessionctx.Context, args []Expression argFieldTp := args[0].GetType() // Use UTF8MB4 as default. bf.tp.Flen = argFieldTp.Flen * 4 * 2 - sig := &builtinHexStrArgSig{bf, charset.NewEncoding(argFieldTp.Charset)} + sig := &builtinHexStrArgSig{bf} sig.setPbCode(tipb.ScalarFuncSig_HexStrArg) return sig, nil case types.ETInt, types.ETReal, types.ETDecimal: @@ -1684,15 +1684,11 @@ func (c *hexFunctionClass) getFunction(ctx sessionctx.Context, args []Expression type builtinHexStrArgSig struct { baseBuiltinFunc - encoding *charset.Encoding } func (b *builtinHexStrArgSig) Clone() builtinFunc { newSig := &builtinHexStrArgSig{} newSig.cloneFrom(&b.baseBuiltinFunc) - if b.encoding != nil { - newSig.encoding = charset.NewEncoding(b.encoding.Name()) - } return newSig } @@ -1703,12 +1699,7 @@ func (b *builtinHexStrArgSig) evalString(row chunk.Row) (string, bool, error) { if isNull || err != nil { return d, isNull, err } - dBytes := hack.Slice(d) - dBytes, err = b.encoding.Encode(nil, dBytes) - if err != nil { - return d, false, err - } - return strings.ToUpper(hex.EncodeToString(dBytes)), false, nil + return strings.ToUpper(hex.EncodeToString(hack.Slice(d))), false, nil } type builtinHexIntArgSig struct { diff --git a/expression/builtin_string_vec.go b/expression/builtin_string_vec.go index ca21e724f0dfa..6d93c17849eef 100644 --- a/expression/builtin_string_vec.go +++ b/expression/builtin_string_vec.go @@ -447,7 +447,6 @@ func (b *builtinHexStrArgSig) vecEvalString(input *chunk.Chunk, result *chunk.Co return err } defer b.bufAllocator.put(buf0) - var encodedBuf []byte if err := b.args[0].VecEvalString(b.ctx, input, buf0); err != nil { return err } @@ -457,13 +456,7 @@ func (b *builtinHexStrArgSig) vecEvalString(input *chunk.Chunk, result *chunk.Co result.AppendNull() continue } - buf0Bytes := buf0.GetBytes(i) - encodedBuf, err = b.encoding.Encode(encodedBuf, buf0Bytes) - if err != nil { - return err - } - buf0Bytes = encodedBuf - result.AppendString(strings.ToUpper(hex.EncodeToString(buf0Bytes))) + result.AppendString(strings.ToUpper(hex.EncodeToString(buf0.GetBytes(i)))) } return nil } diff --git a/expression/distsql_builtin.go b/expression/distsql_builtin.go index d4f0d5eb84fd9..db9b39a2db010 100644 --- a/expression/distsql_builtin.go +++ b/expression/distsql_builtin.go @@ -21,7 +21,6 @@ import ( "time" "github.com/pingcap/errors" - "github.com/pingcap/tidb/parser/charset" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/sessionctx" @@ -965,11 +964,7 @@ func getSignatureByPB(ctx sessionctx.Context, sigCode tipb.ScalarFuncSig, tp *ti case tipb.ScalarFuncSig_HexIntArg: f = &builtinHexIntArgSig{base} case tipb.ScalarFuncSig_HexStrArg: - chs, args := "utf-8", base.getArgs() - if len(args) == 1 { - chs, _ = args[0].CharsetAndCollation() - } - f = &builtinHexStrArgSig{base, charset.NewEncoding(chs)} + f = &builtinHexStrArgSig{base} case tipb.ScalarFuncSig_InsertUTF8: f = &builtinInsertUTF8Sig{base, maxAllowedPacket} case tipb.ScalarFuncSig_Insert: From 65c57f60c3458d2f0e81694db0c1dd132c21c9be Mon Sep 17 00:00:00 2001 From: tangenta Date: Fri, 12 Nov 2021 20:02:12 +0800 Subject: [PATCH 02/15] expression/builtin_cast.go: fix comment of exported function --- expression/builtin_cast.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go index 71ef316d4d035..cd2c69c5d09d6 100644 --- a/expression/builtin_cast.go +++ b/expression/builtin_cast.go @@ -1908,7 +1908,7 @@ func WrapWithCastAsDecimal(ctx sessionctx.Context, expr Expression) Expression { return BuildCastFunction(ctx, expr, tp) } -// WrapWithCastAsStringWithTp wraps `expr` with converting charset. +// WrapWithCharsetConvert wraps `expr` with converting charset. func WrapWithCharsetConvert(ctx sessionctx.Context, expr Expression, funcName string) Expression { retTp := expr.GetType() if _, err := charset.GetDefaultCollationLegacy(retTp.Charset); err != nil { From 79fc364e80cdaf3ce71ced4131050118024016a8 Mon Sep 17 00:00:00 2001 From: tangenta Date: Fri, 12 Nov 2021 20:18:36 +0800 Subject: [PATCH 03/15] expression/builtin_charset.go: fix vectored builtinCharsetConvSig for explaintest --- expression/builtin_charset.go | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/expression/builtin_charset.go b/expression/builtin_charset.go index 0c994e604e998..0ea8b2db9c46c 100644 --- a/expression/builtin_charset.go +++ b/expression/builtin_charset.go @@ -64,6 +64,7 @@ func (b *builtinCharsetConvSig) vecEvalString(input *chunk.Chunk, result *chunk. if err := b.args[0].VecEvalString(b.ctx, input, buf); err != nil { return err } + enc := charset.NewEncoding(b.args[0].GetType().Charset) result.ReserveString(n) for i := 0; i < n; i++ { var str string @@ -72,6 +73,10 @@ func (b *builtinCharsetConvSig) vecEvalString(input *chunk.Chunk, result *chunk. continue } str = buf.GetString(i) + str, err = enc.EncodeString(str) + if err != nil { + return err + } str, err = types.ProduceStrWithSpecifiedTp(str, b.tp, b.ctx.GetSessionVars().StmtCtx, false) if err != nil { return err From 782fc3151139e14ff6be3b5ceb5f41d46a26b1e8 Mon Sep 17 00:00:00 2001 From: tangenta Date: Mon, 15 Nov 2021 08:31:53 +0800 Subject: [PATCH 04/15] expression: use wrapping method for length, ascii and to_base64 --- expression/builtin.go | 2 +- expression/builtin_cast.go | 21 +++++++++++-------- expression/builtin_charset.go | 14 ++++++------- expression/builtin_string.go | 21 ------------------- expression/builtin_string_vec.go | 35 +------------------------------- 5 files changed, 22 insertions(+), 71 deletions(-) diff --git a/expression/builtin.go b/expression/builtin.go index 9860489da6ea3..a21ae1afb324b 100644 --- a/expression/builtin.go +++ b/expression/builtin.go @@ -140,7 +140,7 @@ func newBaseBuiltinFuncWithTp(ctx sessionctx.Context, funcName string, args []Ex args[i] = WrapWithCastAsDecimal(ctx, args[i]) case types.ETString: args[i] = WrapWithCastAsString(ctx, args[i]) - args[i] = WrapWithCharsetConvert(ctx, args[i], funcName) + args[i] = WrapWithConvertCharset(ctx, args[i], funcName) case types.ETDatetime: args[i] = WrapWithCastAsTime(ctx, args[i], types.NewFieldType(mysql.TypeDatetime)) case types.ETTimestamp: diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go index cd2c69c5d09d6..4c9e0537e9d73 100644 --- a/expression/builtin_cast.go +++ b/expression/builtin_cast.go @@ -1908,20 +1908,25 @@ func WrapWithCastAsDecimal(ctx sessionctx.Context, expr Expression) Expression { return BuildCastFunction(ctx, expr, tp) } -// WrapWithCharsetConvert wraps `expr` with converting charset. -func WrapWithCharsetConvert(ctx sessionctx.Context, expr Expression, funcName string) Expression { +// charsetConvertMap contains the builtin functions which arguments need to be converted to the correct charset. +var charsetConvertMap = map[string]struct{}{ + ast.Hex: {}, ast.Length: {}, ast.OctetLength: {}, ast.ASCII: {}, + ast.ToBase64: {}, +} + +// WrapWithConvertCharset wraps `expr` with converting charset sig. +func WrapWithConvertCharset(ctx sessionctx.Context, expr Expression, funcName string) Expression { retTp := expr.GetType() if _, err := charset.GetDefaultCollationLegacy(retTp.Charset); err != nil { - const charsetConv = "charset_convert" - switch funcName { - case ast.Hex: - bf, err := newBaseBuiltinFunc(ctx, charsetConv, []Expression{expr}, retTp.EvalType()) + const convChs = "convert_charset" + if _, ok := charsetConvertMap[funcName]; ok { + bf, err := newBaseBuiltinFunc(ctx, convChs, []Expression{expr}, retTp.EvalType()) if err != nil { return expr } - chsSig := &builtinCharsetConvSig{bf} + chsSig := &builtinConvertCharsetSig{bf} return &ScalarFunction{ - FuncName: model.NewCIStr(charsetConv), + FuncName: model.NewCIStr(convChs), RetType: retTp, Function: chsSig, } diff --git a/expression/builtin_charset.go b/expression/builtin_charset.go index 0ea8b2db9c46c..bdc7f53f4fe25 100644 --- a/expression/builtin_charset.go +++ b/expression/builtin_charset.go @@ -20,19 +20,19 @@ import ( "github.com/pingcap/tidb/util/chunk" ) -var _ builtinFunc = &builtinCharsetConvSig{} +var _ builtinFunc = &builtinConvertCharsetSig{} -type builtinCharsetConvSig struct { +type builtinConvertCharsetSig struct { baseBuiltinFunc } -func (b *builtinCharsetConvSig) Clone() builtinFunc { - newSig := &builtinCharsetConvSig{} +func (b *builtinConvertCharsetSig) Clone() builtinFunc { + newSig := &builtinConvertCharsetSig{} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } -func (b *builtinCharsetConvSig) evalString(row chunk.Row) (res string, isNull bool, err error) { +func (b *builtinConvertCharsetSig) evalString(row chunk.Row) (res string, isNull bool, err error) { val, isNull, err := b.args[0].EvalString(b.ctx, row) if isNull || err != nil { return res, isNull, err @@ -50,11 +50,11 @@ func (b *builtinCharsetConvSig) evalString(row chunk.Row) (res string, isNull bo return padZeroForBinaryType(res, b.tp, b.ctx) } -func (b *builtinCharsetConvSig) vectorized() bool { +func (b *builtinConvertCharsetSig) vectorized() bool { return true } -func (b *builtinCharsetConvSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { +func (b *builtinConvertCharsetSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { n := input.NumRows() buf, err := b.bufAllocator.get() if err != nil { diff --git a/expression/builtin_string.go b/expression/builtin_string.go index 517f07927ed27..bf948605045b2 100644 --- a/expression/builtin_string.go +++ b/expression/builtin_string.go @@ -222,15 +222,6 @@ func (b *builtinLengthSig) evalInt(row chunk.Row) (int64, bool, error) { if isNull || err != nil { return 0, isNull, err } - - argTp := b.args[0].GetType() - if !types.IsBinaryStr(argTp) { - dBytes, err := charset.NewEncoding(argTp.Charset).EncodeString(val) - if err == nil { - return int64(len(dBytes)), false, nil - } - } - return int64(len([]byte(val))), false, nil } @@ -272,13 +263,6 @@ func (b *builtinASCIISig) evalInt(row chunk.Row) (int64, bool, error) { if len(val) == 0 { return 0, false, nil } - argTp := b.args[0].GetType() - if !types.IsBinaryStr(argTp) { - dBytes, err := charset.NewEncoding(argTp.Charset).EncodeString(val) - if err == nil { - return int64(dBytes[0]), false, nil - } - } return int64(val[0]), false, nil } @@ -3625,11 +3609,6 @@ func (b *builtinToBase64Sig) evalString(row chunk.Row) (d string, isNull bool, e if isNull || err != nil { return "", isNull, err } - argTp := b.args[0].GetType() - str, err = charset.NewEncoding(argTp.Charset).EncodeString(str) - if err != nil { - return "", false, err - } needEncodeLen := base64NeededEncodedLength(len(str)) if needEncodeLen == -1 { return "", true, nil diff --git a/expression/builtin_string_vec.go b/expression/builtin_string_vec.go index 6d93c17849eef..11933f305fe58 100644 --- a/expression/builtin_string_vec.go +++ b/expression/builtin_string_vec.go @@ -905,11 +905,6 @@ func (b *builtinASCIISig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) e if err = b.args[0].VecEvalString(b.ctx, input, buf); err != nil { return err } - - argTp := b.args[0].GetType() - enc := charset.NewEncoding(argTp.Charset) - isBinaryStr := types.IsBinaryStr(argTp) - result.ResizeInt64(n, false) result.MergeNulls(buf) i64s := result.Int64s() @@ -922,14 +917,6 @@ func (b *builtinASCIISig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) e i64s[i] = 0 continue } - if !isBinaryStr { - dBytes, err := enc.EncodeString(str) - if err != nil { - return err - } - i64s[i] = int64(dBytes[0]) - continue - } i64s[i] = int64(str[0]) } return nil @@ -2155,27 +2142,14 @@ func (b *builtinLengthSig) vecEvalInt(input *chunk.Chunk, result *chunk.Column) return err } - argTp := b.args[0].GetType() - enc := charset.NewEncoding(argTp.Charset) - isBinaryStr := types.IsBinaryStr(argTp) - result.ResizeInt64(n, false) result.MergeNulls(buf) i64s := result.Int64s() - var encodeBuf []byte for i := 0; i < n; i++ { if result.IsNull(i) { continue } str := buf.GetBytes(i) - if !isBinaryStr { - dBytes, err := enc.Encode(encodeBuf, str) - if err != nil { - return err - } - i64s[i] = int64(len(dBytes)) - continue - } i64s[i] = int64(len(str)) } return nil @@ -2463,20 +2437,13 @@ func (b *builtinToBase64Sig) vecEvalString(input *chunk.Chunk, result *chunk.Col if err := b.args[0].VecEvalString(b.ctx, input, buf); err != nil { return err } - - argTp := b.args[0].GetType() - enc := charset.NewEncoding(argTp.Charset) - result.ReserveString(n) for i := 0; i < n; i++ { if buf.IsNull(i) { result.AppendNull() continue } - str, err := enc.EncodeString(buf.GetString(i)) - if err != nil { - return err - } + str := buf.GetString(i) needEncodeLen := base64NeededEncodedLength(len(str)) if needEncodeLen == -1 { result.AppendNull() From 69cfec5216c77bd2dc150d9141abb3c656bdeca9 Mon Sep 17 00:00:00 2001 From: tangenta Date: Mon, 15 Nov 2021 14:16:58 +0800 Subject: [PATCH 05/15] expression: wrap with convert_charset() in utf8 strings --- expression/builtin_cast.go | 28 ----------- ..._charset.go => builtin_convert_charset.go} | 49 +++++++++++-------- 2 files changed, 28 insertions(+), 49 deletions(-) rename expression/{builtin_charset.go => builtin_convert_charset.go} (67%) diff --git a/expression/builtin_cast.go b/expression/builtin_cast.go index 4c9e0537e9d73..407982af11e31 100644 --- a/expression/builtin_cast.go +++ b/expression/builtin_cast.go @@ -29,7 +29,6 @@ import ( "github.com/pingcap/errors" "github.com/pingcap/tidb/parser/ast" - "github.com/pingcap/tidb/parser/charset" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/parser/mysql" "github.com/pingcap/tidb/parser/terror" @@ -1908,33 +1907,6 @@ func WrapWithCastAsDecimal(ctx sessionctx.Context, expr Expression) Expression { return BuildCastFunction(ctx, expr, tp) } -// charsetConvertMap contains the builtin functions which arguments need to be converted to the correct charset. -var charsetConvertMap = map[string]struct{}{ - ast.Hex: {}, ast.Length: {}, ast.OctetLength: {}, ast.ASCII: {}, - ast.ToBase64: {}, -} - -// WrapWithConvertCharset wraps `expr` with converting charset sig. -func WrapWithConvertCharset(ctx sessionctx.Context, expr Expression, funcName string) Expression { - retTp := expr.GetType() - if _, err := charset.GetDefaultCollationLegacy(retTp.Charset); err != nil { - const convChs = "convert_charset" - if _, ok := charsetConvertMap[funcName]; ok { - bf, err := newBaseBuiltinFunc(ctx, convChs, []Expression{expr}, retTp.EvalType()) - if err != nil { - return expr - } - chsSig := &builtinConvertCharsetSig{bf} - return &ScalarFunction{ - FuncName: model.NewCIStr(convChs), - RetType: retTp, - Function: chsSig, - } - } - } - return expr -} - // WrapWithCastAsString wraps `expr` with `cast` if the return type of expr is // not type string, otherwise, returns `expr` directly. func WrapWithCastAsString(ctx sessionctx.Context, expr Expression) Expression { diff --git a/expression/builtin_charset.go b/expression/builtin_convert_charset.go similarity index 67% rename from expression/builtin_charset.go rename to expression/builtin_convert_charset.go index bdc7f53f4fe25..8c1013ae97bc7 100644 --- a/expression/builtin_charset.go +++ b/expression/builtin_convert_charset.go @@ -15,8 +15,10 @@ package expression import ( + "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/charset" - "github.com/pingcap/tidb/types" + "github.com/pingcap/tidb/parser/model" + "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/util/chunk" ) @@ -40,14 +42,7 @@ func (b *builtinConvertCharsetSig) evalString(row chunk.Row) (res string, isNull tp := b.args[0].GetType() enc := charset.NewEncoding(tp.Charset) res, err = enc.EncodeString(val) - if err != nil { - return res, false, err - } - res, err = types.ProduceStrWithSpecifiedTp(res, b.tp, b.ctx.GetSessionVars().StmtCtx, false) - if err != nil { - return res, false, err - } - return padZeroForBinaryType(res, b.tp, b.ctx) + return res, false, err } func (b *builtinConvertCharsetSig) vectorized() bool { @@ -77,20 +72,32 @@ func (b *builtinConvertCharsetSig) vecEvalString(input *chunk.Chunk, result *chu if err != nil { return err } - str, err = types.ProduceStrWithSpecifiedTp(str, b.tp, b.ctx.GetSessionVars().StmtCtx, false) - if err != nil { - return err - } - var d bool - str, d, err = padZeroForBinaryType(str, b.tp, b.ctx) + result.AppendString(str) + } + return nil +} + +// charsetConvertMap contains the builtin functions which arguments need to be converted to the correct charset. +var charsetConvertMap = map[string]struct{}{ + ast.Hex: {}, ast.Length: {}, ast.OctetLength: {}, ast.ASCII: {}, + ast.ToBase64: {}, +} + +// WrapWithConvertCharset wraps `expr` with converting charset sig. +func WrapWithConvertCharset(ctx sessionctx.Context, expr Expression, funcName string) Expression { + retTp := expr.GetType() + const convChs = "convert_charset" + if _, ok := charsetConvertMap[funcName]; ok { + bf, err := newBaseBuiltinFunc(ctx, convChs, []Expression{expr}, retTp.EvalType()) if err != nil { - return err + return expr } - if d { - result.AppendNull() - } else { - result.AppendString(str) + chsSig := &builtinConvertCharsetSig{bf} + return &ScalarFunction{ + FuncName: model.NewCIStr(convChs), + RetType: retTp, + Function: chsSig, } } - return nil + return expr } From f71aeb7b2eefd4e817a0d74c9df1d7587a5bcdaf Mon Sep 17 00:00:00 2001 From: tangenta Date: Mon, 15 Nov 2021 15:00:02 +0800 Subject: [PATCH 06/15] expression: fold constant recursively in convert_charset() --- expression/constant_fold.go | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/expression/constant_fold.go b/expression/constant_fold.go index d7cbaf5d8edd6..4b0a99e2750a0 100644 --- a/expression/constant_fold.go +++ b/expression/constant_fold.go @@ -165,7 +165,11 @@ func foldConstant(expr Expression) (Expression, bool) { allConstArg := true isDeferredConst := false for i := 0; i < len(args); i++ { - switch x := args[i].(type) { + arg := args[i] + if sf, ok := arg.(*ScalarFunction); ok && sf.FuncName.L == "convert_charset" { + arg = FoldConstant(sf) + } + switch x := arg.(type) { case *Constant: isDeferredConst = isDeferredConst || x.DeferredExpr != nil || x.ParamMarker != nil argIsConst[i] = true From 5f420777c72798f9bea87b7537e4f60a9d94da6b Mon Sep 17 00:00:00 2001 From: tangenta Date: Mon, 15 Nov 2021 17:29:50 +0800 Subject: [PATCH 07/15] expression: rename convert_charset to to_binary and support fold constant --- expression/builtin_convert_charset.go | 52 +++++++++++++++------------ expression/constant_fold.go | 2 +- expression/scalar_function.go | 2 ++ 3 files changed, 33 insertions(+), 23 deletions(-) diff --git a/expression/builtin_convert_charset.go b/expression/builtin_convert_charset.go index 8c1013ae97bc7..3150965ac19fd 100644 --- a/expression/builtin_convert_charset.go +++ b/expression/builtin_convert_charset.go @@ -19,22 +19,26 @@ import ( "github.com/pingcap/tidb/parser/charset" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" + "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" ) -var _ builtinFunc = &builtinConvertCharsetSig{} +// InternalFuncToBinary accepts a string and returns another string encoded in a given charset. +const InternalFuncToBinary = "to_binary" -type builtinConvertCharsetSig struct { +var _ builtinFunc = &builtinInternalToBinarySig{} + +type builtinInternalToBinarySig struct { baseBuiltinFunc } -func (b *builtinConvertCharsetSig) Clone() builtinFunc { - newSig := &builtinConvertCharsetSig{} +func (b *builtinInternalToBinarySig) Clone() builtinFunc { + newSig := &builtinInternalToBinarySig{} newSig.cloneFrom(&b.baseBuiltinFunc) return newSig } -func (b *builtinConvertCharsetSig) evalString(row chunk.Row) (res string, isNull bool, err error) { +func (b *builtinInternalToBinarySig) evalString(row chunk.Row) (res string, isNull bool, err error) { val, isNull, err := b.args[0].EvalString(b.ctx, row) if isNull || err != nil { return res, isNull, err @@ -45,11 +49,11 @@ func (b *builtinConvertCharsetSig) evalString(row chunk.Row) (res string, isNull return res, false, err } -func (b *builtinConvertCharsetSig) vectorized() bool { +func (b *builtinInternalToBinarySig) vectorized() bool { return true } -func (b *builtinConvertCharsetSig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { +func (b *builtinInternalToBinarySig) vecEvalString(input *chunk.Chunk, result *chunk.Column) error { n := input.NumRows() buf, err := b.bufAllocator.get() if err != nil { @@ -77,27 +81,31 @@ func (b *builtinConvertCharsetSig) vecEvalString(input *chunk.Chunk, result *chu return nil } -// charsetConvertMap contains the builtin functions which arguments need to be converted to the correct charset. -var charsetConvertMap = map[string]struct{}{ +// toBinaryMap contains the builtin functions which arguments need to be converted to the correct charset. +var toBinaryMap = map[string]struct{}{ ast.Hex: {}, ast.Length: {}, ast.OctetLength: {}, ast.ASCII: {}, ast.ToBase64: {}, } -// WrapWithConvertCharset wraps `expr` with converting charset sig. +// BuildToBinaryFunction builds a to_binary ScalarFunction from the Expression. +func BuildToBinaryFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldType) Expression { + bf, err := newBaseBuiltinFunc(ctx, InternalFuncToBinary, []Expression{expr}, tp.EvalType()) + if err != nil { + return expr + } + chsSig := &builtinInternalToBinarySig{bf} + return &ScalarFunction{ + FuncName: model.NewCIStr(InternalFuncToBinary), + RetType: tp, + Function: chsSig, + } +} + +// WrapWithConvertCharset wraps `expr` with to_binary sig. func WrapWithConvertCharset(ctx sessionctx.Context, expr Expression, funcName string) Expression { retTp := expr.GetType() - const convChs = "convert_charset" - if _, ok := charsetConvertMap[funcName]; ok { - bf, err := newBaseBuiltinFunc(ctx, convChs, []Expression{expr}, retTp.EvalType()) - if err != nil { - return expr - } - chsSig := &builtinConvertCharsetSig{bf} - return &ScalarFunction{ - FuncName: model.NewCIStr(convChs), - RetType: retTp, - Function: chsSig, - } + if _, ok := toBinaryMap[funcName]; ok { + return BuildToBinaryFunction(ctx, expr, retTp) } return expr } diff --git a/expression/constant_fold.go b/expression/constant_fold.go index 4b0a99e2750a0..a1c0dce24f764 100644 --- a/expression/constant_fold.go +++ b/expression/constant_fold.go @@ -166,7 +166,7 @@ func foldConstant(expr Expression) (Expression, bool) { isDeferredConst := false for i := 0; i < len(args); i++ { arg := args[i] - if sf, ok := arg.(*ScalarFunction); ok && sf.FuncName.L == "convert_charset" { + if sf, ok := arg.(*ScalarFunction); ok && sf.FuncName.L == InternalFuncToBinary { arg = FoldConstant(sf) } switch x := arg.(type) { diff --git a/expression/scalar_function.go b/expression/scalar_function.go index 220eeb090a5b1..4091b54a7550f 100644 --- a/expression/scalar_function.go +++ b/expression/scalar_function.go @@ -183,6 +183,8 @@ func newFunctionImpl(ctx sessionctx.Context, fold int, funcName string, retType return BuildCastFunction(ctx, args[0], retType), nil case ast.GetVar: return BuildGetVarFunction(ctx, args[0], retType) + case InternalFuncToBinary: + return BuildToBinaryFunction(ctx, args[0], retType), nil } fc, ok := funcs[funcName] if !ok { From d271cecc69626e5d1c63ea37c503860d4c4ea325 Mon Sep 17 00:00:00 2001 From: tangenta Date: Tue, 16 Nov 2021 11:53:46 +0800 Subject: [PATCH 08/15] expression: enable wrapping to_binary() for new charset only --- expression/builtin.go | 2 +- expression/builtin_convert_charset.go | 10 ++++++---- 2 files changed, 7 insertions(+), 5 deletions(-) diff --git a/expression/builtin.go b/expression/builtin.go index a21ae1afb324b..c53471b5f0132 100644 --- a/expression/builtin.go +++ b/expression/builtin.go @@ -140,7 +140,7 @@ func newBaseBuiltinFuncWithTp(ctx sessionctx.Context, funcName string, args []Ex args[i] = WrapWithCastAsDecimal(ctx, args[i]) case types.ETString: args[i] = WrapWithCastAsString(ctx, args[i]) - args[i] = WrapWithConvertCharset(ctx, args[i], funcName) + args[i] = WrapWithToBinary(ctx, args[i], funcName) case types.ETDatetime: args[i] = WrapWithCastAsTime(ctx, args[i], types.NewFieldType(mysql.TypeDatetime)) case types.ETTimestamp: diff --git a/expression/builtin_convert_charset.go b/expression/builtin_convert_charset.go index 3150965ac19fd..ca5fe50597daf 100644 --- a/expression/builtin_convert_charset.go +++ b/expression/builtin_convert_charset.go @@ -101,11 +101,13 @@ func BuildToBinaryFunction(ctx sessionctx.Context, expr Expression, tp *types.Fi } } -// WrapWithConvertCharset wraps `expr` with to_binary sig. -func WrapWithConvertCharset(ctx sessionctx.Context, expr Expression, funcName string) Expression { +// WrapWithToBinary wraps `expr` with to_binary sig. +func WrapWithToBinary(ctx sessionctx.Context, expr Expression, funcName string) Expression { retTp := expr.GetType() - if _, ok := toBinaryMap[funcName]; ok { - return BuildToBinaryFunction(ctx, expr, retTp) + if _, err := charset.GetDefaultCollationLegacy(retTp.Charset); err == nil { + if _, ok := toBinaryMap[funcName]; ok { + return BuildToBinaryFunction(ctx, expr, retTp) + } } return expr } From bfe99e30c6e41fccff7b14d875c9b4dba61801a8 Mon Sep 17 00:00:00 2001 From: tangenta Date: Tue, 16 Nov 2021 13:42:15 +0800 Subject: [PATCH 09/15] fix WrapWithToBinary --- expression/builtin_convert_charset.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expression/builtin_convert_charset.go b/expression/builtin_convert_charset.go index ca5fe50597daf..5de8fdb21a842 100644 --- a/expression/builtin_convert_charset.go +++ b/expression/builtin_convert_charset.go @@ -104,7 +104,7 @@ func BuildToBinaryFunction(ctx sessionctx.Context, expr Expression, tp *types.Fi // WrapWithToBinary wraps `expr` with to_binary sig. func WrapWithToBinary(ctx sessionctx.Context, expr Expression, funcName string) Expression { retTp := expr.GetType() - if _, err := charset.GetDefaultCollationLegacy(retTp.Charset); err == nil { + if _, err := charset.GetDefaultCollationLegacy(retTp.Charset); err != nil { if _, ok := toBinaryMap[funcName]; ok { return BuildToBinaryFunction(ctx, expr, retTp) } From a62efa9451420522f0e556b0787599a2a0fdc1da Mon Sep 17 00:00:00 2001 From: tangenta Date: Tue, 16 Nov 2021 16:06:09 +0800 Subject: [PATCH 10/15] expression: add function class and add to_binary() to it for constantant folding --- expression/builtin.go | 3 ++ expression/builtin_convert_charset.go | 56 ++++++++++++++++++++------- expression/constant_fold.go | 6 +-- expression/scalar_function.go | 2 - 4 files changed, 45 insertions(+), 22 deletions(-) diff --git a/expression/builtin.go b/expression/builtin.go index c53471b5f0132..59bc9f36737c1 100644 --- a/expression/builtin.go +++ b/expression/builtin.go @@ -880,6 +880,9 @@ var funcs = map[string]functionClass{ ast.NextVal: &nextValFunctionClass{baseFunctionClass{ast.NextVal, 1, 1}}, ast.LastVal: &lastValFunctionClass{baseFunctionClass{ast.LastVal, 1, 1}}, ast.SetVal: &setValFunctionClass{baseFunctionClass{ast.SetVal, 2, 2}}, + + // TiDB implicit internal functions. + InternalFuncToBinary: &tidbConvertCharsetFunctionClass{baseFunctionClass{InternalFuncToBinary, 1, 1}}, } // IsFunctionSupported check if given function name is a builtin sql function. diff --git a/expression/builtin_convert_charset.go b/expression/builtin_convert_charset.go index 5de8fdb21a842..793b710f95c69 100644 --- a/expression/builtin_convert_charset.go +++ b/expression/builtin_convert_charset.go @@ -15,17 +15,47 @@ package expression import ( + "fmt" + "github.com/pingcap/tidb/parser/ast" "github.com/pingcap/tidb/parser/charset" "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" + "github.com/pingcap/tipb/go-tipb" ) // InternalFuncToBinary accepts a string and returns another string encoded in a given charset. const InternalFuncToBinary = "to_binary" +type tidbConvertCharsetFunctionClass struct { + baseFunctionClass +} + +func (c *tidbConvertCharsetFunctionClass) getFunction(ctx sessionctx.Context, args []Expression) (builtinFunc, error) { + if err := c.verifyArgs(args); err != nil { + return nil, c.verifyArgs(args) + } + argFieldTp := args[0].GetType() + argTp := argFieldTp.EvalType() + bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, argTp, argTp) + if err != nil { + return nil, err + } + var sig builtinFunc + switch argTp { + case types.ETString: + sig = &builtinInternalToBinarySig{bf} + // TODO(tangenta): set pbcode for to_binary() sig. + sig.setPbCode(tipb.ScalarFuncSig_Unspecified) + default: + msg := fmt.Sprintf("unexpected argTp: %d", argTp) + panic(msg) + } + return sig, nil +} + var _ builtinFunc = &builtinInternalToBinarySig{} type builtinInternalToBinarySig struct { @@ -87,26 +117,22 @@ var toBinaryMap = map[string]struct{}{ ast.ToBase64: {}, } -// BuildToBinaryFunction builds a to_binary ScalarFunction from the Expression. -func BuildToBinaryFunction(ctx sessionctx.Context, expr Expression, tp *types.FieldType) Expression { - bf, err := newBaseBuiltinFunc(ctx, InternalFuncToBinary, []Expression{expr}, tp.EvalType()) - if err != nil { - return expr - } - chsSig := &builtinInternalToBinarySig{bf} - return &ScalarFunction{ - FuncName: model.NewCIStr(InternalFuncToBinary), - RetType: tp, - Function: chsSig, - } -} - // WrapWithToBinary wraps `expr` with to_binary sig. func WrapWithToBinary(ctx sessionctx.Context, expr Expression, funcName string) Expression { retTp := expr.GetType() if _, err := charset.GetDefaultCollationLegacy(retTp.Charset); err != nil { if _, ok := toBinaryMap[funcName]; ok { - return BuildToBinaryFunction(ctx, expr, retTp) + bf, err := newBaseBuiltinFunc(ctx, InternalFuncToBinary, []Expression{expr}, retTp.EvalType()) + if err != nil { + return expr + } + chsSig := &builtinInternalToBinarySig{bf} + sf := &ScalarFunction{ + FuncName: model.NewCIStr(InternalFuncToBinary), + RetType: retTp, + Function: chsSig, + } + return FoldConstant(sf) } } return expr diff --git a/expression/constant_fold.go b/expression/constant_fold.go index a1c0dce24f764..d7cbaf5d8edd6 100644 --- a/expression/constant_fold.go +++ b/expression/constant_fold.go @@ -165,11 +165,7 @@ func foldConstant(expr Expression) (Expression, bool) { allConstArg := true isDeferredConst := false for i := 0; i < len(args); i++ { - arg := args[i] - if sf, ok := arg.(*ScalarFunction); ok && sf.FuncName.L == InternalFuncToBinary { - arg = FoldConstant(sf) - } - switch x := arg.(type) { + switch x := args[i].(type) { case *Constant: isDeferredConst = isDeferredConst || x.DeferredExpr != nil || x.ParamMarker != nil argIsConst[i] = true diff --git a/expression/scalar_function.go b/expression/scalar_function.go index 4091b54a7550f..220eeb090a5b1 100644 --- a/expression/scalar_function.go +++ b/expression/scalar_function.go @@ -183,8 +183,6 @@ func newFunctionImpl(ctx sessionctx.Context, fold int, funcName string, retType return BuildCastFunction(ctx, args[0], retType), nil case ast.GetVar: return BuildGetVarFunction(ctx, args[0], retType) - case InternalFuncToBinary: - return BuildToBinaryFunction(ctx, args[0], retType), nil } fc, ok := funcs[funcName] if !ok { From 8a620989f2bd8f08bcae9bfdc5eaaf9c5b71023a Mon Sep 17 00:00:00 2001 From: tangenta Date: Tue, 16 Nov 2021 16:24:18 +0800 Subject: [PATCH 11/15] test: add a test for to_binary constant folding --- expression/constant_test.go | 40 +++++++++++++++++++++++++++++++++++-- 1 file changed, 38 insertions(+), 2 deletions(-) diff --git a/expression/constant_test.go b/expression/constant_test.go index 08d201ffd6b38..7061e1a842b50 100644 --- a/expression/constant_test.go +++ b/expression/constant_test.go @@ -49,9 +49,20 @@ func newLonglong(value int64) *Constant { } } +func newString(value string, collation string) *Constant { + return &Constant{ + Value: types.NewStringDatum(value), + RetType: types.NewFieldTypeWithCollation(mysql.TypeVarchar, collation, 255), + } +} + func newFunction(funcName string, args ...Expression) Expression { - typeLong := types.NewFieldType(mysql.TypeLonglong) - return NewFunctionInternal(mock.NewContext(), funcName, typeLong, args...) + return newFunctionWithType(funcName, mysql.TypeLonglong, args...) +} + +func newFunctionWithType(funcName string, tp byte, args ...Expression) Expression { + ft := types.NewFieldType(tp) + return NewFunctionInternal(mock.NewContext(), funcName, ft, args...) } func TestConstantPropagation(t *testing.T) { @@ -220,6 +231,31 @@ func TestConstantFolding(t *testing.T) { } } +func TestConstantFoldingCharsetConvert(t *testing.T) { + t.Parallel() + tests := []struct { + condition Expression + result string + } { + { + condition: newFunction(ast.Length, newFunctionWithType( + InternalFuncToBinary, mysql.TypeVarchar, + newString("中文", "gbk_bin"))), + result: "4", + }, + { + condition: newFunction(ast.Length, newFunctionWithType( + InternalFuncToBinary, mysql.TypeVarchar, + newString("中文", "utf8mb4_bin"))), + result: "6", + }, + } + for _, tt := range tests { + newConds := FoldConstant(tt.condition) + require.Equalf(t, tt.result, newConds.String(), "different for expr %s", tt.condition) + } +} + func TestDeferredParamNotNull(t *testing.T) { t.Parallel() From df296cef004006a596b13904196f41b9d588be42 Mon Sep 17 00:00:00 2001 From: tangenta Date: Tue, 16 Nov 2021 16:32:23 +0800 Subject: [PATCH 12/15] gofmt: format code --- expression/constant_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/expression/constant_test.go b/expression/constant_test.go index 7061e1a842b50..2158b1e4f5d66 100644 --- a/expression/constant_test.go +++ b/expression/constant_test.go @@ -51,7 +51,7 @@ func newLonglong(value int64) *Constant { func newString(value string, collation string) *Constant { return &Constant{ - Value: types.NewStringDatum(value), + Value: types.NewStringDatum(value), RetType: types.NewFieldTypeWithCollation(mysql.TypeVarchar, collation, 255), } } @@ -236,7 +236,7 @@ func TestConstantFoldingCharsetConvert(t *testing.T) { tests := []struct { condition Expression result string - } { + }{ { condition: newFunction(ast.Length, newFunctionWithType( InternalFuncToBinary, mysql.TypeVarchar, From c811e5170875b5f50189e6049b2eb917e2aa0206 Mon Sep 17 00:00:00 2001 From: tangenta Date: Tue, 16 Nov 2021 17:12:57 +0800 Subject: [PATCH 13/15] expression: use getFunction in WrapWithToBinary --- expression/builtin_convert_charset.go | 15 +++++++-------- 1 file changed, 7 insertions(+), 8 deletions(-) diff --git a/expression/builtin_convert_charset.go b/expression/builtin_convert_charset.go index 793b710f95c69..f581e1cd98dd3 100644 --- a/expression/builtin_convert_charset.go +++ b/expression/builtin_convert_charset.go @@ -50,8 +50,7 @@ func (c *tidbConvertCharsetFunctionClass) getFunction(ctx sessionctx.Context, ar // TODO(tangenta): set pbcode for to_binary() sig. sig.setPbCode(tipb.ScalarFuncSig_Unspecified) default: - msg := fmt.Sprintf("unexpected argTp: %d", argTp) - panic(msg) + return nil, fmt.Errorf("unexpected argTp: %d", argTp) } return sig, nil } @@ -119,18 +118,18 @@ var toBinaryMap = map[string]struct{}{ // WrapWithToBinary wraps `expr` with to_binary sig. func WrapWithToBinary(ctx sessionctx.Context, expr Expression, funcName string) Expression { - retTp := expr.GetType() - if _, err := charset.GetDefaultCollationLegacy(retTp.Charset); err != nil { + exprTp := expr.GetType() + if _, err := charset.GetDefaultCollationLegacy(exprTp.Charset); err != nil { if _, ok := toBinaryMap[funcName]; ok { - bf, err := newBaseBuiltinFunc(ctx, InternalFuncToBinary, []Expression{expr}, retTp.EvalType()) + fc := funcs[InternalFuncToBinary] + sig, err := fc.getFunction(ctx, []Expression{expr}) if err != nil { return expr } - chsSig := &builtinInternalToBinarySig{bf} sf := &ScalarFunction{ FuncName: model.NewCIStr(InternalFuncToBinary), - RetType: retTp, - Function: chsSig, + RetType: exprTp, + Function: sig, } return FoldConstant(sf) } From 945143430228a3d39e8bf27718f8ba2cadc62077 Mon Sep 17 00:00:00 2001 From: tangenta Date: Tue, 16 Nov 2021 17:19:52 +0800 Subject: [PATCH 14/15] expression: skip show to_binary builtin function --- expression/builtin.go | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/expression/builtin.go b/expression/builtin.go index 59bc9f36737c1..a7e9537e84a52 100644 --- a/expression/builtin.go +++ b/expression/builtin.go @@ -906,6 +906,7 @@ func GetDisplayName(name string) string { func GetBuiltinList() []string { res := make([]string, 0, len(funcs)) notImplementedFunctions := []string{ast.RowFunc, ast.IsTruthWithNull} + implicitFunctions := []string{InternalFuncToBinary} for funcName := range funcs { skipFunc := false // Skip not implemented functions @@ -914,6 +915,11 @@ func GetBuiltinList() []string { skipFunc = true } } + for _, implicitFunc := range implicitFunctions { + if funcName == implicitFunc { + skipFunc = true + } + } // Skip literal functions // (their names are not readable: 'tidb`.(dateliteral, for example) // See: https://github.com/pingcap/parser/pull/591 From 41263b9897dc86155fb9bc80dda4dfc190d11a21 Mon Sep 17 00:00:00 2001 From: tangenta Date: Wed, 17 Nov 2021 14:39:41 +0800 Subject: [PATCH 15/15] expression: update tipb to set pbcode for to_binary --- expression/builtin_convert_charset.go | 14 ++++++-------- go.mod | 2 +- go.sum | 4 ++-- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/expression/builtin_convert_charset.go b/expression/builtin_convert_charset.go index f581e1cd98dd3..243a50e5d0a4e 100644 --- a/expression/builtin_convert_charset.go +++ b/expression/builtin_convert_charset.go @@ -37,18 +37,16 @@ func (c *tidbConvertCharsetFunctionClass) getFunction(ctx sessionctx.Context, ar if err := c.verifyArgs(args); err != nil { return nil, c.verifyArgs(args) } - argFieldTp := args[0].GetType() - argTp := argFieldTp.EvalType() - bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, argTp, argTp) - if err != nil { - return nil, err - } + argTp := args[0].GetType().EvalType() var sig builtinFunc switch argTp { case types.ETString: + bf, err := newBaseBuiltinFuncWithTp(ctx, c.funcName, args, types.ETString, types.ETString) + if err != nil { + return nil, err + } sig = &builtinInternalToBinarySig{bf} - // TODO(tangenta): set pbcode for to_binary() sig. - sig.setPbCode(tipb.ScalarFuncSig_Unspecified) + sig.setPbCode(tipb.ScalarFuncSig_ToBinary) default: return nil, fmt.Errorf("unexpected argTp: %d", argTp) } diff --git a/go.mod b/go.mod index aa4dc6c700ff1..165ed4ad2f538 100644 --- a/go.mod +++ b/go.mod @@ -53,7 +53,7 @@ require ( github.com/pingcap/sysutil v0.0.0-20210730114356-fcd8a63f68c5 github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible github.com/pingcap/tidb/parser v0.0.0-20211011031125-9b13dc409c5e - github.com/pingcap/tipb v0.0.0-20211105090418-71142a4d40e3 + github.com/pingcap/tipb v0.0.0-20211116093845-e9b045a0bdf8 github.com/prometheus/client_golang v1.5.1 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.9.1 diff --git a/go.sum b/go.sum index 1237e7f202a53..804792f146573 100644 --- a/go.sum +++ b/go.sum @@ -600,8 +600,8 @@ github.com/pingcap/tidb-dashboard v0.0.0-20211008050453-a25c25809529/go.mod h1:O github.com/pingcap/tidb-dashboard v0.0.0-20211031170437-08e58c069a2a/go.mod h1:OCXbZTBTIMRcIt0jFsuCakZP+goYRv6IjawKbwLS2TQ= github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible h1:c7+izmker91NkjkZ6FgTlmD4k1A5FLOAq+li6Ki2/GY= github.com/pingcap/tidb-tools v5.2.2-0.20211019062242-37a8bef2fa17+incompatible/go.mod h1:XGdcy9+yqlDSEMTpOXnwf3hiTeqrV6MN/u1se9N8yIM= -github.com/pingcap/tipb v0.0.0-20211105090418-71142a4d40e3 h1:xnp/Qkk5gELlB8TaY6oro0JNXMBXTafNVxU/vbrNU8I= -github.com/pingcap/tipb v0.0.0-20211105090418-71142a4d40e3/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= +github.com/pingcap/tipb v0.0.0-20211116093845-e9b045a0bdf8 h1:Vu/6oq8EFNWgyXRHiclNzTKIu+YKHPCSI/Ba5oVrLtM= +github.com/pingcap/tipb v0.0.0-20211116093845-e9b045a0bdf8/go.mod h1:A7mrd7WHBl1o63LE2bIBGEJMTNWXqhgmYiOvMLxozfs= github.com/pkg/errors v0.8.0/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.8.1/go.mod h1:bwawxfHBFNV+L2hUp1rHADufV3IMtnDRdf1r5NINEl0= github.com/pkg/errors v0.9.1 h1:FEBLx1zS214owpjy7qsBeixbURkuhQAwrK5UwLGTwt4=