Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

expression: cast charset according to the function's resulting charset #29905

Merged
merged 6 commits into from
Nov 24, 2021
Merged

expression: cast charset according to the function's resulting charset #29905

Show file tree
Hide file tree
Changes from 5 commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
92aac1c
add from binary
xiongjiwei Nov 17, 2021
6149aa8
add from binary
xiongjiwei Nov 19, 2021
e7dd3f4
add more test
xiongjiwei Nov 22, 2021
2ba2711
rebase and add more test
xiongjiwei Nov 24, 2021
691c843
rebase and add more test
xiongjiwei Nov 24, 2021
f2568eb
Merge branch 'master' into from_binary
zimulala Nov 24, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
204 changes: 204 additions & 0 deletions cmd/explaintest/r/new_character_set_builtin.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -286,3 +286,207 @@ select ord(a), ord(b), ord(c) from t;
ord(a) ord(b) ord(c)
14989485 54992 228
set @@tidb_enable_vectorized_expression = false;
drop table if exists t;
create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20));
insert into t values ('一', '一', 0xe4b880);
insert into t values ('一', '一', 0xd2bb);
insert into t values ('一', '一', 0xe4ba8c);
insert into t values ('一', '一', 0xb6fe);
set @@tidb_enable_vectorized_expression = true;
select hex(concat(a, c)), hex(concat(b, c)) from t;
hex(concat(a, c)) hex(concat(b, c))
E4B880E4B8800000000000000000000000000000000000 D2BBE4B8800000000000000000000000000000000000
E4B880D2BB000000000000000000000000000000000000 D2BBD2BB000000000000000000000000000000000000
E4B880E4BA8C0000000000000000000000000000000000 D2BBE4BA8C0000000000000000000000000000000000
E4B880B6FE000000000000000000000000000000000000 D2BBB6FE000000000000000000000000000000000000
select hex(concat(a, 0xe4b880)), hex(concat(b, 0xd2bb)) from t;
hex(concat(a, 0xe4b880)) hex(concat(b, 0xd2bb))
E4B880E4B880 D2BBD2BB
E4B880E4B880 D2BBD2BB
E4B880E4B880 D2BBD2BB
E4B880E4B880 D2BBD2BB
select a = 0xe4b880, b = 0xd2bb from t;
a = 0xe4b880 b = 0xd2bb
1 1
1 1
1 1
1 1
select a = c, b = c from t;
a = c b = c
0 0
0 0
0 0
0 0
select hex(insert(a, 1, 2, 0xe4ba8c)), hex(insert(b, 1, 2, 0xb6fe)) from t;
hex(insert(a, 1, 2, 0xe4ba8c)) hex(insert(b, 1, 2, 0xb6fe))
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
select hex(insert(a, 1, 2, c)), hex(insert(b, 1, 2, c)) from t;
hex(insert(a, 1, 2, c)) hex(insert(b, 1, 2, c))
E4B880000000000000000000000000000000000080 E4B8800000000000000000000000000000000000
D2BB00000000000000000000000000000000000080 D2BB000000000000000000000000000000000000
E4BA8C000000000000000000000000000000000080 E4BA8C0000000000000000000000000000000000
B6FE00000000000000000000000000000000000080 B6FE000000000000000000000000000000000000
select hex(lpad(a, 5, 0xe4ba8c)), hex(lpad(b, 5, 0xb6fe)) from t;
hex(lpad(a, 5, 0xe4ba8c)) hex(lpad(b, 5, 0xb6fe))
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
select hex(lpad(a, 5, c)), hex(lpad(b, 5, c)) from t;
hex(lpad(a, 5, c)) hex(lpad(b, 5, c))
E4B8E4B880 E4B880D2BB
D2BBE4B880 D2BB00D2BB
E4BAE4B880 E4BA8CD2BB
B6FEE4B880 B6FE00D2BB
select hex(rpad(a, 5, 0xe4ba8c)), hex(rpad(b, 5, 0xb6fe)) from t;
Copy link
Contributor Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

mysql has different results here and they confirmed it is bug https://bugs.mysql.com/bug.php?id=105668&thanks=4

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Please make a note of it and put it in our discussion document.

hex(rpad(a, 5, 0xe4ba8c)) hex(rpad(b, 5, 0xb6fe))
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
select hex(rpad(a, 5, c)), hex(rpad(b, 5, c)) from t;
hex(rpad(a, 5, c)) hex(rpad(b, 5, c))
E4B880E4B8 D2BBE4B880
E4B880D2BB D2BBD2BB00
E4B880E4BA D2BBE4BA8C
E4B880B6FE D2BBB6FE00
select hex(elt(2, a, 0xe4ba8c)), hex(elt(2, b, 0xb6fe)) from t;
hex(elt(2, a, 0xe4ba8c)) hex(elt(2, b, 0xb6fe))
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
select hex(elt(2, a, c)), hex(elt(2, b, c)) from t;
hex(elt(2, a, c)) hex(elt(2, b, c))
E4B8800000000000000000000000000000000000 E4B8800000000000000000000000000000000000
D2BB000000000000000000000000000000000000 D2BB000000000000000000000000000000000000
E4BA8C0000000000000000000000000000000000 E4BA8C0000000000000000000000000000000000
B6FE000000000000000000000000000000000000 B6FE000000000000000000000000000000000000
select hex(instr(a, 0xe4b880)), hex(instr(b, 0xd2bb)) from t;
hex(instr(a, 0xe4b880)) hex(instr(b, 0xd2bb))
1 1
1 1
1 1
1 1
select hex(position(a in 0xe4b880)), hex(position(b in 0xd2bb)) from t;
hex(position(a in 0xe4b880)) hex(position(b in 0xd2bb))
1 1
1 1
1 1
1 1
select a like 0xe4b880, b like 0xd2bb from t;
a like 0xe4b880 b like 0xd2bb
1 1
1 1
1 1
1 1
select a = 0xb6fe from t;
Error 3854: Cannot convert string 'B6FE' from binary to utf8mb4
select b = 0xe4ba8c from t;
Error 3854: Cannot convert string 'E4BA8C' from binary to gbk
select concat(a, 0xb6fe) from t;
Error 3854: Cannot convert string 'B6FE' from binary to utf8mb4
select concat(b, 0xe4ba8c) from t;
Error 3854: Cannot convert string 'E4BA8C' from binary to gbk
set @@tidb_enable_vectorized_expression = false;
select hex(concat(a, c)), hex(concat(b, c)) from t;
hex(concat(a, c)) hex(concat(b, c))
E4B880E4B8800000000000000000000000000000000000 D2BBE4B8800000000000000000000000000000000000
E4B880D2BB000000000000000000000000000000000000 D2BBD2BB000000000000000000000000000000000000
E4B880E4BA8C0000000000000000000000000000000000 D2BBE4BA8C0000000000000000000000000000000000
E4B880B6FE000000000000000000000000000000000000 D2BBB6FE000000000000000000000000000000000000
select hex(concat(a, 0xe4b880)), hex(concat(b, 0xd2bb)) from t;
hex(concat(a, 0xe4b880)) hex(concat(b, 0xd2bb))
E4B880E4B880 D2BBD2BB
E4B880E4B880 D2BBD2BB
E4B880E4B880 D2BBD2BB
E4B880E4B880 D2BBD2BB
select a = 0xe4b880, b = 0xd2bb from t;
a = 0xe4b880 b = 0xd2bb
1 1
1 1
1 1
1 1
select a = c, b = c from t;
a = c b = c
0 0
0 0
0 0
0 0
select hex(insert(a, 1, 2, 0xe4ba8c)), hex(insert(b, 1, 2, 0xb6fe)) from t;
hex(insert(a, 1, 2, 0xe4ba8c)) hex(insert(b, 1, 2, 0xb6fe))
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
select hex(insert(a, 1, 2, c)), hex(insert(b, 1, 2, c)) from t;
hex(insert(a, 1, 2, c)) hex(insert(b, 1, 2, c))
E4B880000000000000000000000000000000000080 E4B8800000000000000000000000000000000000
D2BB00000000000000000000000000000000000080 D2BB000000000000000000000000000000000000
E4BA8C000000000000000000000000000000000080 E4BA8C0000000000000000000000000000000000
B6FE00000000000000000000000000000000000080 B6FE000000000000000000000000000000000000
select hex(lpad(a, 5, 0xe4ba8c)), hex(lpad(b, 5, 0xb6fe)) from t;
hex(lpad(a, 5, 0xe4ba8c)) hex(lpad(b, 5, 0xb6fe))
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
select hex(lpad(a, 5, c)), hex(lpad(b, 5, c)) from t;
hex(lpad(a, 5, c)) hex(lpad(b, 5, c))
E4B8E4B880 E4B880D2BB
D2BBE4B880 D2BB00D2BB
E4BAE4B880 E4BA8CD2BB
B6FEE4B880 B6FE00D2BB
select hex(rpad(a, 5, 0xe4ba8c)), hex(rpad(b, 5, 0xb6fe)) from t;
hex(rpad(a, 5, 0xe4ba8c)) hex(rpad(b, 5, 0xb6fe))
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
select hex(rpad(a, 5, c)), hex(rpad(b, 5, c)) from t;
hex(rpad(a, 5, c)) hex(rpad(b, 5, c))
E4B880E4B8 D2BBE4B880
E4B880D2BB D2BBD2BB00
E4B880E4BA D2BBE4BA8C
E4B880B6FE D2BBB6FE00
select hex(elt(2, a, 0xe4ba8c)), hex(elt(2, b, 0xb6fe)) from t;
hex(elt(2, a, 0xe4ba8c)) hex(elt(2, b, 0xb6fe))
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
select hex(elt(2, a, c)), hex(elt(2, b, c)) from t;
hex(elt(2, a, c)) hex(elt(2, b, c))
E4B8800000000000000000000000000000000000 E4B8800000000000000000000000000000000000
D2BB000000000000000000000000000000000000 D2BB000000000000000000000000000000000000
E4BA8C0000000000000000000000000000000000 E4BA8C0000000000000000000000000000000000
B6FE000000000000000000000000000000000000 B6FE000000000000000000000000000000000000
select hex(instr(a, 0xe4b880)), hex(instr(b, 0xd2bb)) from t;
hex(instr(a, 0xe4b880)) hex(instr(b, 0xd2bb))
1 1
1 1
1 1
1 1
select hex(position(a in 0xe4b880)), hex(position(b in 0xd2bb)) from t;
hex(position(a in 0xe4b880)) hex(position(b in 0xd2bb))
1 1
1 1
1 1
1 1
select a like 0xe4b880, b like 0xd2bb from t;
a like 0xe4b880 b like 0xd2bb
1 1
1 1
1 1
1 1
select a = 0xb6fe from t;
Error 3854: Cannot convert string 'B6FE' from binary to utf8mb4
select b = 0xe4ba8c from t;
Error 3854: Cannot convert string 'E4BA8C' from binary to gbk
select concat(a, 0xb6fe) from t;
Error 3854: Cannot convert string 'B6FE' from binary to utf8mb4
select concat(b, 0xe4ba8c) from t;
Error 3854: Cannot convert string 'E4BA8C' from binary to gbk
4 changes: 4 additions & 0 deletions cmd/explaintest/r/select.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -495,3 +495,7 @@ insert into precise_types values (
SELECT a, b, c, d FROM precise_types;
a b c d
18446744073709551614 -9223372036854775806 99999999999999999999.0 1.8446744073709552e19
create table t3(a char(10), primary key (a));
insert into t3 values ('a');
select * from t3 where a > 0x80;
Error 1105: Cannot convert string '80' from binary to utf8mb4
59 changes: 59 additions & 0 deletions cmd/explaintest/t/new_character_set_builtin.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -161,3 +161,62 @@ select ord(a), ord(b), ord(c) from t;
set @@tidb_enable_vectorized_expression = true;
select ord(a), ord(b), ord(c) from t;
set @@tidb_enable_vectorized_expression = false;

drop table if exists t;
create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20));
insert into t values ('一', '一', 0xe4b880);
insert into t values ('一', '一', 0xd2bb);
insert into t values ('一', '一', 0xe4ba8c);
insert into t values ('一', '一', 0xb6fe);

set @@tidb_enable_vectorized_expression = true;
select hex(concat(a, c)), hex(concat(b, c)) from t;
select hex(concat(a, 0xe4b880)), hex(concat(b, 0xd2bb)) from t;
select a = 0xe4b880, b = 0xd2bb from t;
select a = c, b = c from t;
select hex(insert(a, 1, 2, 0xe4ba8c)), hex(insert(b, 1, 2, 0xb6fe)) from t;
select hex(insert(a, 1, 2, c)), hex(insert(b, 1, 2, c)) from t;
select hex(lpad(a, 5, 0xe4ba8c)), hex(lpad(b, 5, 0xb6fe)) from t;
select hex(lpad(a, 5, c)), hex(lpad(b, 5, c)) from t;
select hex(rpad(a, 5, 0xe4ba8c)), hex(rpad(b, 5, 0xb6fe)) from t;
select hex(rpad(a, 5, c)), hex(rpad(b, 5, c)) from t;
select hex(elt(2, a, 0xe4ba8c)), hex(elt(2, b, 0xb6fe)) from t;
select hex(elt(2, a, c)), hex(elt(2, b, c)) from t;
select hex(instr(a, 0xe4b880)), hex(instr(b, 0xd2bb)) from t;
select hex(position(a in 0xe4b880)), hex(position(b in 0xd2bb)) from t;
select a like 0xe4b880, b like 0xd2bb from t;

--error ER_CANNOT_CONVERT_STRING
select a = 0xb6fe from t;
--error ER_CANNOT_CONVERT_STRING
select b = 0xe4ba8c from t;
--error ER_CANNOT_CONVERT_STRING
select concat(a, 0xb6fe) from t;
--error ER_CANNOT_CONVERT_STRING
select concat(b, 0xe4ba8c) from t;

set @@tidb_enable_vectorized_expression = false;
select hex(concat(a, c)), hex(concat(b, c)) from t;
select hex(concat(a, 0xe4b880)), hex(concat(b, 0xd2bb)) from t;
select a = 0xe4b880, b = 0xd2bb from t;
select a = c, b = c from t;
select hex(insert(a, 1, 2, 0xe4ba8c)), hex(insert(b, 1, 2, 0xb6fe)) from t;
select hex(insert(a, 1, 2, c)), hex(insert(b, 1, 2, c)) from t;
select hex(lpad(a, 5, 0xe4ba8c)), hex(lpad(b, 5, 0xb6fe)) from t;
select hex(lpad(a, 5, c)), hex(lpad(b, 5, c)) from t;
select hex(rpad(a, 5, 0xe4ba8c)), hex(rpad(b, 5, 0xb6fe)) from t;
select hex(rpad(a, 5, c)), hex(rpad(b, 5, c)) from t;
select hex(elt(2, a, 0xe4ba8c)), hex(elt(2, b, 0xb6fe)) from t;
select hex(elt(2, a, c)), hex(elt(2, b, c)) from t;
select hex(instr(a, 0xe4b880)), hex(instr(b, 0xd2bb)) from t;
select hex(position(a in 0xe4b880)), hex(position(b in 0xd2bb)) from t;
select a like 0xe4b880, b like 0xd2bb from t;

--error ER_CANNOT_CONVERT_STRING
select a = 0xb6fe from t;
--error ER_CANNOT_CONVERT_STRING
select b = 0xe4ba8c from t;
--error ER_CANNOT_CONVERT_STRING
select concat(a, 0xb6fe) from t;
--error ER_CANNOT_CONVERT_STRING
select concat(b, 0xe4ba8c) from t;
5 changes: 5 additions & 0 deletions cmd/explaintest/t/select.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -244,3 +244,8 @@ insert into precise_types values (
18446744073709551614
);
SELECT a, b, c, d FROM precise_types;

create table t3(a char(10), primary key (a));
insert into t3 values ('a');
--error ER_CANNOT_CONVERT_STRING
select * from t3 where a > 0x80;
1 change: 1 addition & 0 deletions errno/errcode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -901,6 +901,7 @@ const (
ErrFKIncompatibleColumns = 3780
ErrFunctionalIndexRowValueIsNotAllowed = 3800
ErrDependentByFunctionalIndex = 3837
ErrCannotConvertString = 3854
ErrInvalidJSONValueForFuncIndex = 3903
ErrJSONValueOutOfRangeForFuncIndex = 3904
ErrFunctionalIndexDataIsTooLong = 3907
Expand Down
1 change: 1 addition & 0 deletions errno/errname.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -896,6 +896,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrFKIncompatibleColumns: mysql.Message("Referencing column '%s' in foreign key constraint '%s' are incompatible", nil),
ErrFunctionalIndexRowValueIsNotAllowed: mysql.Message("Expression of expression index '%s' cannot refer to a row value", nil),
ErrDependentByFunctionalIndex: mysql.Message("Column '%s' has an expression index dependency and cannot be dropped or renamed", nil),
ErrCannotConvertString: mysql.Message("Cannot convert string '%.64s' from %s to %s", nil),
ErrInvalidJSONValueForFuncIndex: mysql.Message("Invalid JSON value for CAST for expression index '%s'", nil),
ErrJSONValueOutOfRangeForFuncIndex: mysql.Message("Out of range JSON value for CAST for expression index '%s'", nil),
ErrFunctionalIndexDataIsTooLong: mysql.Message("Data too long for expression index '%s'", nil),
Expand Down
11 changes: 1 addition & 10 deletions expression/builtin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -140,7 +140,7 @@ func newBaseBuiltinFuncWithTp(ctx sessionctx.Context, funcName string, args []Ex
args[i] = WrapWithCastAsDecimal(ctx, args[i])
case types.ETString:
args[i] = WrapWithCastAsString(ctx, args[i])
args[i] = WrapWithToBinary(ctx, args[i], funcName)
args[i] = HandleBinaryLiteral(ctx, args[i], ec, funcName)
case types.ETDatetime:
args[i] = WrapWithCastAsTime(ctx, args[i], types.NewFieldType(mysql.TypeDatetime))
case types.ETTimestamp:
Expand Down Expand Up @@ -880,9 +880,6 @@ var funcs = map[string]functionClass{
ast.NextVal: &nextValFunctionClass{baseFunctionClass{ast.NextVal, 1, 1}},
ast.LastVal: &lastValFunctionClass{baseFunctionClass{ast.LastVal, 1, 1}},
ast.SetVal: &setValFunctionClass{baseFunctionClass{ast.SetVal, 2, 2}},

// TiDB implicit internal functions.
InternalFuncToBinary: &tidbConvertCharsetFunctionClass{baseFunctionClass{InternalFuncToBinary, 1, 1}},
}

// IsFunctionSupported check if given function name is a builtin sql function.
Expand All @@ -906,7 +903,6 @@ func GetDisplayName(name string) string {
func GetBuiltinList() []string {
res := make([]string, 0, len(funcs))
notImplementedFunctions := []string{ast.RowFunc, ast.IsTruthWithNull}
implicitFunctions := []string{InternalFuncToBinary}
for funcName := range funcs {
skipFunc := false
// Skip not implemented functions
Expand All @@ -915,11 +911,6 @@ func GetBuiltinList() []string {
skipFunc = true
}
}
for _, implicitFunc := range implicitFunctions {
if funcName == implicitFunc {
skipFunc = true
}
}
// Skip literal functions
// (their names are not readable: 'tidb`.(dateliteral, for example)
// See: https://github.com/pingcap/parser/pull/591
Expand Down
Loading