From 20d46c2c5332f82161b84789e5993876ecfef8d5 Mon Sep 17 00:00:00 2001 From: xiongjiwei Date: Fri, 16 Sep 2022 12:50:59 +0800 Subject: [PATCH] planner: make unsafe substitute can be used in generated column (#37779) close pingcap/tidb#35490 --- .../explain_generate_column_substitute.result | 84 +++++++++++++++++++ .../t/explain_generate_column_substitute.test | 36 ++++++++ parser/types/field_type.go | 20 +++++ .../core/rule_generate_column_substitute.go | 2 +- sessionctx/variable/session.go | 3 + sessionctx/variable/sysvar.go | 4 + sessionctx/variable/tidb_vars.go | 3 + 7 files changed, 151 insertions(+), 1 deletion(-) diff --git a/cmd/explaintest/r/explain_generate_column_substitute.result b/cmd/explaintest/r/explain_generate_column_substitute.result index 4673c0a143e92..83422a318e619 100644 --- a/cmd/explaintest/r/explain_generate_column_substitute.result +++ b/cmd/explaintest/r/explain_generate_column_substitute.result @@ -642,3 +642,87 @@ HashJoin 9.99 root inner join, equal:[eq(test.t2.c_decimal, test.t1.c_decimal)] └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo drop table t1; drop table t2; +set @@tidb_enable_unsafe_substitute=1; +CREATE TABLE person (id INT PRIMARY KEY,address_info JSON,city VARCHAR(2) AS (JSON_UNQUOTE(address_info->"$.city")),KEY (city)); +INSERT INTO `person` (`id`, `address_info`) VALUES('1','{\"city\": \"Beijing\"}'); +SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +id +1 +desc format = 'brief' SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +id estRows task access object operator info +Projection 8000.00 root test.person.id +└─TableReader 8000.00 root data:Selection + └─Selection 8000.00 cop[tikv] eq(json_unquote(cast(json_extract(test.person.address_info, "$.city"), var_string(16777216))), "Beijing") + └─TableFullScan 10000.00 cop[tikv] table:person keep order:false, stats:pseudo +SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +id +desc format = 'brief' SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +id estRows task access object operator info +Projection 10.00 root test.person.id +└─IndexReader 10.00 root index:IndexRangeScan + └─IndexRangeScan 10.00 cop[tikv] table:person, index:city(city) range:["Beijing","Beijing"], keep order:false, stats:pseudo +drop table person; +create table t(a char(5), b char(6) as (concat(a, a)), index bx(b)); +insert into t(a) values ('aaaaa'); +select * from t; +a b +aaaaa aaaaaa +select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa'; +a b +aaaaa aaaaaa +desc format = 'brief' select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa'; +id estRows task access object operator info +TableReader 8000.00 root data:Selection +└─Selection 8000.00 cop[tikv] eq(concat(test.t.a, test.t.a), "aaaaaaaaaa") + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa'; +a b +desc format = 'brief' select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa'; +id estRows task access object operator info +IndexLookUp 10.00 root +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:bx(b) range:["aaaaaaaaaa","aaaaaaaaaa"], keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo +drop table t; +CREATE TABLE person (id INT PRIMARY KEY,address_info JSON,city VARCHAR(64) AS (JSON_UNQUOTE(address_info->"$.city")),KEY (city)); +INSERT INTO `person` (`id`, `address_info`) VALUES('1','{\"city\": \"Beijing\"}'); +SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +id +1 +desc format = 'brief' SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +id estRows task access object operator info +Projection 8000.00 root test.person.id +└─TableReader 8000.00 root data:Selection + └─Selection 8000.00 cop[tikv] eq(json_unquote(cast(json_extract(test.person.address_info, "$.city"), var_string(16777216))), "Beijing") + └─TableFullScan 10000.00 cop[tikv] table:person keep order:false, stats:pseudo +SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +id +1 +desc format = 'brief' SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +id estRows task access object operator info +Projection 10.00 root test.person.id +└─IndexReader 10.00 root index:IndexRangeScan + └─IndexRangeScan 10.00 cop[tikv] table:person, index:city(city) range:["Beijing","Beijing"], keep order:false, stats:pseudo +drop table person; +create table t(a char(5), b char(10) as (concat(a, a)), index bx(b)); +insert into t(a) values ('aaaaa'); +select * from t; +a b +aaaaa aaaaaaaaaa +select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa'; +a b +aaaaa aaaaaaaaaa +desc format = 'brief' select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa'; +id estRows task access object operator info +TableReader 8000.00 root data:Selection +└─Selection 8000.00 cop[tikv] eq(concat(test.t.a, test.t.a), "aaaaaaaaaa") + └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo +select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa'; +a b +aaaaa aaaaaaaaaa +desc format = 'brief' select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa'; +id estRows task access object operator info +IndexLookUp 10.00 root +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t, index:bx(b) range:["aaaaaaaaaa","aaaaaaaaaa"], keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 10.00 cop[tikv] table:t keep order:false, stats:pseudo +drop table t; +set @@tidb_enable_unsafe_substitute=0; diff --git a/cmd/explaintest/t/explain_generate_column_substitute.test b/cmd/explaintest/t/explain_generate_column_substitute.test index 1acde78a709e7..527e404c9d17b 100644 --- a/cmd/explaintest/t/explain_generate_column_substitute.test +++ b/cmd/explaintest/t/explain_generate_column_substitute.test @@ -302,3 +302,39 @@ explain format = 'brief' select /*+ agg_to_cop() */ * from t1 where c_decimal in drop table t1; drop table t2; +set @@tidb_enable_unsafe_substitute=1; +CREATE TABLE person (id INT PRIMARY KEY,address_info JSON,city VARCHAR(2) AS (JSON_UNQUOTE(address_info->"$.city")),KEY (city)); +INSERT INTO `person` (`id`, `address_info`) VALUES('1','{\"city\": \"Beijing\"}'); +SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +desc format = 'brief' SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +desc format = 'brief' SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +drop table person; + +create table t(a char(5), b char(6) as (concat(a, a)), index bx(b)); +insert into t(a) values ('aaaaa'); +select * from t; +select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa'; +desc format = 'brief' select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa'; +select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa'; +desc format = 'brief' select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa'; +drop table t; + +CREATE TABLE person (id INT PRIMARY KEY,address_info JSON,city VARCHAR(64) AS (JSON_UNQUOTE(address_info->"$.city")),KEY (city)); +INSERT INTO `person` (`id`, `address_info`) VALUES('1','{\"city\": \"Beijing\"}'); +SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +desc format = 'brief' SELECT id FROM person ignore index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +desc format = 'brief' SELECT id FROM person force index(`city`) WHERE address_info->>"$.city" = 'Beijing'; +drop table person; + +create table t(a char(5), b char(10) as (concat(a, a)), index bx(b)); +insert into t(a) values ('aaaaa'); +select * from t; +select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa'; +desc format = 'brief' select * from t ignore index(bx) where concat(a, a) = 'aaaaaaaaaa'; +select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa'; +desc format = 'brief' select * from t force index(bx) where concat(a, a) = 'aaaaaaaaaa'; +drop table t; +set @@tidb_enable_unsafe_substitute=0; + diff --git a/parser/types/field_type.go b/parser/types/field_type.go index 481939cbae8d4..69484af6ed43a 100644 --- a/parser/types/field_type.go +++ b/parser/types/field_type.go @@ -273,6 +273,26 @@ func (ft *FieldType) Equal(other *FieldType) bool { return true } +// PartialEqual checks whether two FieldType objects are equal. +// If unsafe is true and the objects is string type, PartialEqual will ignore flen. +// See https://github.com/pingcap/tidb/issues/35490#issuecomment-1211658886 for more detail. +func (ft *FieldType) PartialEqual(other *FieldType, unsafe bool) bool { + if !unsafe || ft.EvalType() != ETString || other.EvalType() != ETString { + return ft.Equal(other) + } + + partialEqual := ft.charset == other.charset && ft.collate == other.collate && mysql.HasUnsignedFlag(ft.flag) == mysql.HasUnsignedFlag(other.flag) + if !partialEqual || len(ft.elems) != len(other.elems) { + return false + } + for i := range ft.elems { + if ft.elems[i] != other.elems[i] { + return false + } + } + return true +} + // EvalType gets the type in evaluation. func (ft *FieldType) EvalType() EvalType { switch ft.tp { diff --git a/planner/core/rule_generate_column_substitute.go b/planner/core/rule_generate_column_substitute.go index 0e53ae1be2377..c796f99af62c5 100644 --- a/planner/core/rule_generate_column_substitute.go +++ b/planner/core/rule_generate_column_substitute.go @@ -66,7 +66,7 @@ func collectGenerateColumn(lp LogicalPlan, exprToColumn ExprColumnMap) { if colInfo.IsGenerated() && !colInfo.GeneratedStored { s := ds.schema.Columns col := expression.ColInfo2Col(s, colInfo) - if col != nil && col.GetType().Equal(col.VirtualExpr.GetType()) { + if col != nil && col.GetType().PartialEqual(col.VirtualExpr.GetType(), lp.SCtx().GetSessionVars().EnableUnsafeSubstitute) { exprToColumn[col.VirtualExpr] = col } } diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index e7a2a96bd68b0..b145d7c77f9d2 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -1263,6 +1263,9 @@ type SessionVars struct { // EnableTiFlashReadForWriteStmt indicates whether to enable TiFlash to read for write statements. EnableTiFlashReadForWriteStmt bool + // EnableUnsafeSubstitute indicates whether to enable generate column takes unsafe substitute. + EnableUnsafeSubstitute bool + // ForeignKeyChecks indicates whether to enable foreign key constraint check. ForeignKeyChecks bool diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 3f27851e85bc2..f4452f1cb0874 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -1823,6 +1823,10 @@ var defaultSysVars = []*SysVar{ s.EnableTiFlashReadForWriteStmt = TiDBOptOn(val) return nil }}, + {Scope: ScopeGlobal | ScopeSession, Name: TiDBEnableUnsafeSubstitute, Value: BoolToOnOff(false), Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + s.EnableUnsafeSubstitute = TiDBOptOn(val) + return nil + }}, {Scope: ScopeGlobal | ScopeSession, Name: TiDBOptRangeMaxSize, Value: strconv.FormatInt(DefTiDBOptRangeMaxSize, 10), Type: TypeInt, MinValue: 0, MaxValue: math.MaxInt64, SetSession: func(s *SessionVars, val string) error { s.RangeMaxSize = TidbOptInt64(val, DefTiDBOptRangeMaxSize) return nil diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index bb8c3c4ab8bf2..92c76ed12cd9f 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -238,6 +238,9 @@ const ( // TiFlashFastScan indicates whether use fast scan in tiflash. TiFlashFastScan = "tiflash_fastscan" + // TiDBEnableUnsafeSubstitute indicates whether to enable generate column takes unsafe substitute. + TiDBEnableUnsafeSubstitute = "tidb_enable_unsafe_substitute" + // TiDBEnableTiFlashReadForWriteStmt indicates whether to enable TiFlash to read for write statements. TiDBEnableTiFlashReadForWriteStmt = "tidb_enable_tiflash_read_for_write_stmt" )