Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

collation: cast charset according to the function's resulting charset #29029

Closed
wants to merge 5 commits into from
Closed

collation: cast charset according to the function's resulting charset #29029

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
5 commits
Select commit Hold shift + click to select a range
46269e0
convert binary to non-binary
xiongjiwei Oct 21, 2021
f10b6d6
add test
xiongjiwei Oct 27, 2021
9152dd6
add comments
xiongjiwei Oct 27, 2021
f3e9e04
change test to readable character
xiongjiwei Oct 27, 2021
370da72
resolve conflict
xiongjiwei Nov 4, 2021
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
205 changes: 204 additions & 1 deletion cmd/explaintest/r/new_character_set_builtin.result
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -54,7 +54,6 @@ set @@tidb_enable_vectorized_expression = true;
select to_base64(a), to_base64(b), to_base64(c) from t;
to_base64(a) to_base64(b) to_base64(c)
5LiA5LqM5LiJ 0ru2/sj9 5LiA5LqM5LiJAAAAAAAAAAAAAAA=
set @@tidb_enable_vectorized_expression = false;
drop table if exists t;
create table t(a char(10));
insert into t values ('中文'), ('啊'), ('a'), ('1'), ('ㅂ');
Expand Down Expand Up @@ -152,3 +151,207 @@ select decode(encode(a,"monty"),"monty") = a, md5(decode(encode(b,"monty"),"mont
decode(encode(a,"monty"),"monty") = a md5(decode(encode(b,"monty"),"monty")) = md5(b) decode(encode(c,"monty"),"monty") = c
1 1 1
set @@tidb_enable_vectorized_expression = false;
drop table if exists t;
create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20));
insert into t values ('一', '一', 0xe4b880);
insert into t values ('一', '一', 0xd2bb);
insert into t values ('一', '一', 0xe4ba8c);
insert into t values ('一', '一', 0xb6fe);
set @@tidb_enable_vectorized_expression = true;
select hex(concat(a, c)), hex(concat(b, c)) from t;
hex(concat(a, c)) hex(concat(b, c))
E4B880E4B8800000000000000000000000000000000000 D2BBE4B8800000000000000000000000000000000000
E4B880D2BB000000000000000000000000000000000000 D2BBD2BB000000000000000000000000000000000000
E4B880E4BA8C0000000000000000000000000000000000 D2BBE4BA8C0000000000000000000000000000000000
E4B880B6FE000000000000000000000000000000000000 D2BBB6FE000000000000000000000000000000000000
select hex(concat(a, 0xe4b880)), hex(concat(b, 0xd2bb)) from t;
hex(concat(a, 0xe4b880)) hex(concat(b, 0xd2bb))
E4B880E4B880 D2BBD2BB
E4B880E4B880 D2BBD2BB
E4B880E4B880 D2BBD2BB
E4B880E4B880 D2BBD2BB
select a = 0xe4b880, b = 0xd2bb from t;
a = 0xe4b880 b = 0xd2bb
1 1
1 1
1 1
1 1
select a = c, b = c from t;
a = c b = c
0 0
0 0
0 0
0 0
select hex(insert(a, 1, 2, 0xe4ba8c)), hex(insert(b, 1, 2, 0xb6fe)) from t;
hex(insert(a, 1, 2, 0xe4ba8c)) hex(insert(b, 1, 2, 0xb6fe))
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
select hex(insert(a, 1, 2, c)), hex(insert(b, 1, 2, c)) from t;
hex(insert(a, 1, 2, c)) hex(insert(b, 1, 2, c))
E4B880000000000000000000000000000000000080 E4B8800000000000000000000000000000000000
D2BB00000000000000000000000000000000000080 D2BB000000000000000000000000000000000000
E4BA8C000000000000000000000000000000000080 E4BA8C0000000000000000000000000000000000
B6FE00000000000000000000000000000000000080 B6FE000000000000000000000000000000000000
select hex(lpad(a, 5, 0xe4ba8c)), hex(lpad(b, 5, 0xb6fe)) from t;
hex(lpad(a, 5, 0xe4ba8c)) hex(lpad(b, 5, 0xb6fe))
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
select hex(lpad(a, 5, c)), hex(lpad(b, 5, c)) from t;
hex(lpad(a, 5, c)) hex(lpad(b, 5, c))
E4B8E4B880 E4B880D2BB
D2BBE4B880 D2BB00D2BB
E4BAE4B880 E4BA8CD2BB
B6FEE4B880 B6FE00D2BB
select hex(rpad(a, 5, 0xe4ba8c)), hex(rpad(b, 5, 0xb6fe)) from t;
hex(rpad(a, 5, 0xe4ba8c)) hex(rpad(b, 5, 0xb6fe))
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
select hex(rpad(a, 5, c)), hex(rpad(b, 5, c)) from t;
hex(rpad(a, 5, c)) hex(rpad(b, 5, c))
E4B880E4B8 D2BBE4B880
E4B880D2BB D2BBD2BB00
E4B880E4BA D2BBE4BA8C
E4B880B6FE D2BBB6FE00
select hex(elt(2, a, 0xe4ba8c)), hex(elt(2, b, 0xb6fe)) from t;
hex(elt(2, a, 0xe4ba8c)) hex(elt(2, b, 0xb6fe))
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
select hex(elt(2, a, c)), hex(elt(2, b, c)) from t;
hex(elt(2, a, c)) hex(elt(2, b, c))
E4B8800000000000000000000000000000000000 E4B8800000000000000000000000000000000000
D2BB000000000000000000000000000000000000 D2BB000000000000000000000000000000000000
E4BA8C0000000000000000000000000000000000 E4BA8C0000000000000000000000000000000000
B6FE000000000000000000000000000000000000 B6FE000000000000000000000000000000000000
select hex(instr(a, 0xe4b880)), hex(instr(b, 0xd2bb)) from t;
hex(instr(a, 0xe4b880)) hex(instr(b, 0xd2bb))
1 1
1 1
1 1
1 1
select hex(position(a in 0xe4b880)), hex(position(b in 0xd2bb)) from t;
hex(position(a in 0xe4b880)) hex(position(b in 0xd2bb))
1 1
1 1
1 1
1 1
select a like 0xe4b880, b like 0xd2bb from t;
a like 0xe4b880 b like 0xd2bb
1 1
1 1
1 1
1 1
select a = 0xb6fe from t;
Error 3854: Cannot convert string 'B6FE' from binary to utf8mb4
select b = 0xe4ba8c from t;
Error 3854: Cannot convert string 'E4BA8C' from binary to gbk
select concat(a, 0xb6fe) from t;
Error 3854: Cannot convert string 'B6FE' from binary to utf8mb4
select concat(b, 0xe4ba8c) from t;
Error 3854: Cannot convert string 'E4BA8C' from binary to gbk
set @@tidb_enable_vectorized_expression = false;
select hex(concat(a, c)), hex(concat(b, c)) from t;
hex(concat(a, c)) hex(concat(b, c))
E4B880E4B8800000000000000000000000000000000000 D2BBE4B8800000000000000000000000000000000000
E4B880D2BB000000000000000000000000000000000000 D2BBD2BB000000000000000000000000000000000000
E4B880E4BA8C0000000000000000000000000000000000 D2BBE4BA8C0000000000000000000000000000000000
E4B880B6FE000000000000000000000000000000000000 D2BBB6FE000000000000000000000000000000000000
select hex(concat(a, 0xe4b880)), hex(concat(b, 0xd2bb)) from t;
hex(concat(a, 0xe4b880)) hex(concat(b, 0xd2bb))
E4B880E4B880 D2BBD2BB
E4B880E4B880 D2BBD2BB
E4B880E4B880 D2BBD2BB
E4B880E4B880 D2BBD2BB
select a = 0xe4b880, b = 0xd2bb from t;
a = 0xe4b880 b = 0xd2bb
1 1
1 1
1 1
1 1
select a = c, b = c from t;
a = c b = c
0 0
0 0
0 0
0 0
select hex(insert(a, 1, 2, 0xe4ba8c)), hex(insert(b, 1, 2, 0xb6fe)) from t;
hex(insert(a, 1, 2, 0xe4ba8c)) hex(insert(b, 1, 2, 0xb6fe))
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
select hex(insert(a, 1, 2, c)), hex(insert(b, 1, 2, c)) from t;
hex(insert(a, 1, 2, c)) hex(insert(b, 1, 2, c))
E4B880000000000000000000000000000000000080 E4B8800000000000000000000000000000000000
D2BB00000000000000000000000000000000000080 D2BB000000000000000000000000000000000000
E4BA8C000000000000000000000000000000000080 E4BA8C0000000000000000000000000000000000
B6FE00000000000000000000000000000000000080 B6FE000000000000000000000000000000000000
select hex(lpad(a, 5, 0xe4ba8c)), hex(lpad(b, 5, 0xb6fe)) from t;
hex(lpad(a, 5, 0xe4ba8c)) hex(lpad(b, 5, 0xb6fe))
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
E4BA8CE4BA8CE4BA8CE4BA8CE4B880 B6FEB6FEB6FEB6FED2BB
select hex(lpad(a, 5, c)), hex(lpad(b, 5, c)) from t;
hex(lpad(a, 5, c)) hex(lpad(b, 5, c))
E4B8E4B880 E4B880D2BB
D2BBE4B880 D2BB00D2BB
E4BAE4B880 E4BA8CD2BB
B6FEE4B880 B6FE00D2BB
select hex(rpad(a, 5, 0xe4ba8c)), hex(rpad(b, 5, 0xb6fe)) from t;
hex(rpad(a, 5, 0xe4ba8c)) hex(rpad(b, 5, 0xb6fe))
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
E4B880E4BA8CE4BA8CE4BA8CE4BA8C D2BBB6FEB6FEB6FEB6FE
select hex(rpad(a, 5, c)), hex(rpad(b, 5, c)) from t;
hex(rpad(a, 5, c)) hex(rpad(b, 5, c))
E4B880E4B8 D2BBE4B880
E4B880D2BB D2BBD2BB00
E4B880E4BA D2BBE4BA8C
E4B880B6FE D2BBB6FE00
select hex(elt(2, a, 0xe4ba8c)), hex(elt(2, b, 0xb6fe)) from t;
hex(elt(2, a, 0xe4ba8c)) hex(elt(2, b, 0xb6fe))
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
E4BA8C B6FE
select hex(elt(2, a, c)), hex(elt(2, b, c)) from t;
hex(elt(2, a, c)) hex(elt(2, b, c))
E4B8800000000000000000000000000000000000 E4B8800000000000000000000000000000000000
D2BB000000000000000000000000000000000000 D2BB000000000000000000000000000000000000
E4BA8C0000000000000000000000000000000000 E4BA8C0000000000000000000000000000000000
B6FE000000000000000000000000000000000000 B6FE000000000000000000000000000000000000
select hex(instr(a, 0xe4b880)), hex(instr(b, 0xd2bb)) from t;
hex(instr(a, 0xe4b880)) hex(instr(b, 0xd2bb))
1 1
1 1
1 1
1 1
select hex(position(a in 0xe4b880)), hex(position(b in 0xd2bb)) from t;
hex(position(a in 0xe4b880)) hex(position(b in 0xd2bb))
1 1
1 1
1 1
1 1
select a like 0xe4b880, b like 0xd2bb from t;
a like 0xe4b880 b like 0xd2bb
1 1
1 1
1 1
1 1
select a = 0xb6fe from t;
Error 3854: Cannot convert string 'B6FE' from binary to utf8mb4
select b = 0xe4ba8c from t;
Error 3854: Cannot convert string 'E4BA8C' from binary to gbk
select concat(a, 0xb6fe) from t;
Error 3854: Cannot convert string 'B6FE' from binary to utf8mb4
select concat(b, 0xe4ba8c) from t;
Error 3854: Cannot convert string 'E4BA8C' from binary to gbk
60 changes: 59 additions & 1 deletion cmd/explaintest/t/new_character_set_builtin.test
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -31,7 +31,6 @@ insert into t values ('一二三', '一二三', '一二三');
select to_base64(a), to_base64(b), to_base64(c) from t;
set @@tidb_enable_vectorized_expression = true;
select to_base64(a), to_base64(b), to_base64(c) from t;
set @@tidb_enable_vectorized_expression = false;

-- test for builtin function convert()
drop table if exists t;
Expand Down Expand Up @@ -75,3 +74,62 @@ select decode(encode(a,"monty"),"monty") = a, md5(decode(encode(b,"monty"),"mont
set @@tidb_enable_vectorized_expression = true;
select decode(encode(a,"monty"),"monty") = a, md5(decode(encode(b,"monty"),"monty")) = md5(b), decode(encode(c,"monty"),"monty") = c from t;
set @@tidb_enable_vectorized_expression = false;

drop table if exists t;
create table t (a char(20) charset utf8mb4, b char(20) charset gbk, c binary(20));
insert into t values ('一', '一', 0xe4b880);
insert into t values ('一', '一', 0xd2bb);
insert into t values ('一', '一', 0xe4ba8c);
insert into t values ('一', '一', 0xb6fe);

set @@tidb_enable_vectorized_expression = true;
select hex(concat(a, c)), hex(concat(b, c)) from t;
select hex(concat(a, 0xe4b880)), hex(concat(b, 0xd2bb)) from t;
select a = 0xe4b880, b = 0xd2bb from t;
select a = c, b = c from t;
select hex(insert(a, 1, 2, 0xe4ba8c)), hex(insert(b, 1, 2, 0xb6fe)) from t;
select hex(insert(a, 1, 2, c)), hex(insert(b, 1, 2, c)) from t;
select hex(lpad(a, 5, 0xe4ba8c)), hex(lpad(b, 5, 0xb6fe)) from t;
select hex(lpad(a, 5, c)), hex(lpad(b, 5, c)) from t;
select hex(rpad(a, 5, 0xe4ba8c)), hex(rpad(b, 5, 0xb6fe)) from t;
select hex(rpad(a, 5, c)), hex(rpad(b, 5, c)) from t;
select hex(elt(2, a, 0xe4ba8c)), hex(elt(2, b, 0xb6fe)) from t;
select hex(elt(2, a, c)), hex(elt(2, b, c)) from t;
select hex(instr(a, 0xe4b880)), hex(instr(b, 0xd2bb)) from t;
select hex(position(a in 0xe4b880)), hex(position(b in 0xd2bb)) from t;
select a like 0xe4b880, b like 0xd2bb from t;

--error ER_CANNOT_CONVERT_STRING
select a = 0xb6fe from t;
--error ER_CANNOT_CONVERT_STRING
select b = 0xe4ba8c from t;
--error ER_CANNOT_CONVERT_STRING
select concat(a, 0xb6fe) from t;
--error ER_CANNOT_CONVERT_STRING
select concat(b, 0xe4ba8c) from t;

set @@tidb_enable_vectorized_expression = false;
select hex(concat(a, c)), hex(concat(b, c)) from t;
select hex(concat(a, 0xe4b880)), hex(concat(b, 0xd2bb)) from t;
select a = 0xe4b880, b = 0xd2bb from t;
select a = c, b = c from t;
select hex(insert(a, 1, 2, 0xe4ba8c)), hex(insert(b, 1, 2, 0xb6fe)) from t;
select hex(insert(a, 1, 2, c)), hex(insert(b, 1, 2, c)) from t;
select hex(lpad(a, 5, 0xe4ba8c)), hex(lpad(b, 5, 0xb6fe)) from t;
select hex(lpad(a, 5, c)), hex(lpad(b, 5, c)) from t;
select hex(rpad(a, 5, 0xe4ba8c)), hex(rpad(b, 5, 0xb6fe)) from t;
select hex(rpad(a, 5, c)), hex(rpad(b, 5, c)) from t;
select hex(elt(2, a, 0xe4ba8c)), hex(elt(2, b, 0xb6fe)) from t;
select hex(elt(2, a, c)), hex(elt(2, b, c)) from t;
select hex(instr(a, 0xe4b880)), hex(instr(b, 0xd2bb)) from t;
select hex(position(a in 0xe4b880)), hex(position(b in 0xd2bb)) from t;
select a like 0xe4b880, b like 0xd2bb from t;

--error ER_CANNOT_CONVERT_STRING
select a = 0xb6fe from t;
--error ER_CANNOT_CONVERT_STRING
select b = 0xe4ba8c from t;
--error ER_CANNOT_CONVERT_STRING
select concat(a, 0xb6fe) from t;
--error ER_CANNOT_CONVERT_STRING
select concat(b, 0xe4ba8c) from t;
1 change: 1 addition & 0 deletions errno/errcode.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -901,6 +901,7 @@ const (
ErrFKIncompatibleColumns = 3780
ErrFunctionalIndexRowValueIsNotAllowed = 3800
ErrDependentByFunctionalIndex = 3837
ErrCannotConvertString = 3854
ErrInvalidJSONValueForFuncIndex = 3903
ErrJSONValueOutOfRangeForFuncIndex = 3904
ErrFunctionalIndexDataIsTooLong = 3907
Expand Down
1 change: 1 addition & 0 deletions errno/errname.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -896,6 +896,7 @@ var MySQLErrName = map[uint16]*mysql.ErrMessage{
ErrFKIncompatibleColumns: mysql.Message("Referencing column '%s' in foreign key constraint '%s' are incompatible", nil),
ErrFunctionalIndexRowValueIsNotAllowed: mysql.Message("Expression of expression index '%s' cannot refer to a row value", nil),
ErrDependentByFunctionalIndex: mysql.Message("Column '%s' has an expression index dependency and cannot be dropped or renamed", nil),
ErrCannotConvertString: mysql.Message("Cannot convert string '%.64s' from %s to %s", nil),
ErrInvalidJSONValueForFuncIndex: mysql.Message("Invalid JSON value for CAST for expression index '%s'", nil),
ErrJSONValueOutOfRangeForFuncIndex: mysql.Message("Out of range JSON value for CAST for expression index '%s'", nil),
ErrFunctionalIndexDataIsTooLong: mysql.Message("Data too long for expression index '%s'", nil),
Expand Down
6 changes: 3 additions & 3 deletions expression/builtin.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -91,7 +91,7 @@ func newBaseBuiltinFunc(ctx sessionctx.Context, funcName string, args []Expressi
if ctx == nil {
return baseBuiltinFunc{}, errors.New("unexpected nil session ctx")
}
ec, err := deriveCollation(ctx, funcName, args, retType, retType)
ec, _, err := deriveCollation(ctx, funcName, args, retType, retType)
if err != nil {
return baseBuiltinFunc{}, err
}
Expand Down Expand Up @@ -125,7 +125,7 @@ func newBaseBuiltinFuncWithTp(ctx sessionctx.Context, funcName string, args []Ex

// derive collation information for string function, and we must do it
// before doing implicit cast.
ec, err := deriveCollation(ctx, funcName, args, retType, argTps...)
ec, retTp, err := deriveCollation(ctx, funcName, args, retType, argTps...)
if err != nil {
return
}
Expand All @@ -139,7 +139,7 @@ func newBaseBuiltinFuncWithTp(ctx sessionctx.Context, funcName string, args []Ex
case types.ETDecimal:
args[i] = WrapWithCastAsDecimal(ctx, args[i])
case types.ETString:
args[i] = WrapWithCastAsString(ctx, args[i])
args[i] = WrapWithCastAsStringWithTp(ctx, args[i], retTp)
case types.ETDatetime:
args[i] = WrapWithCastAsTime(ctx, args[i], types.NewFieldType(mysql.TypeDatetime))
case types.ETTimestamp:
Expand Down
Loading