diff --git a/config/config.go b/config/config.go index c4b1580569095..e023d4a9c01c6 100644 --- a/config/config.go +++ b/config/config.go @@ -484,8 +484,10 @@ type Performance struct { CrossJoin bool `toml:"cross-join" json:"cross-join"` RunAutoAnalyze bool `toml:"run-auto-analyze" json:"run-auto-analyze"` DistinctAggPushDown bool `toml:"distinct-agg-push-down" json:"distinct-agg-push-down"` - CommitterConcurrency int `toml:"committer-concurrency" json:"committer-concurrency"` - MaxTxnTTL uint64 `toml:"max-txn-ttl" json:"max-txn-ttl"` + // Whether enable projection push down for coprocessors (both tikv & tiflash), default false. + ProjectionPushDown bool `toml:"projection-push-down" json:"projection-push-down"` + CommitterConcurrency int `toml:"committer-concurrency" json:"committer-concurrency"` + MaxTxnTTL uint64 `toml:"max-txn-ttl" json:"max-txn-ttl"` // Deprecated MemProfileInterval string `toml:"-" json:"-"` IndexUsageSyncLease string `toml:"index-usage-sync-lease" json:"index-usage-sync-lease"` @@ -693,6 +695,7 @@ var defaultConf = Config{ TxnEntrySizeLimit: DefTxnEntrySizeLimit, TxnTotalSizeLimit: DefTxnTotalSizeLimit, DistinctAggPushDown: false, + ProjectionPushDown: false, CommitterConcurrency: defTiKVCfg.CommitterConcurrency, MaxTxnTTL: defTiKVCfg.MaxTxnTTL, // 1hour // TODO: set indexUsageSyncLease to 60s. diff --git a/planner/core/exhaust_physical_plans.go b/planner/core/exhaust_physical_plans.go index f8ebbbb2ab63f..b15a1aee3d549 100644 --- a/planner/core/exhaust_physical_plans.go +++ b/planner/core/exhaust_physical_plans.go @@ -2050,6 +2050,13 @@ func (p *LogicalProjection) exhaustPhysicalPlans(prop *property.PhysicalProperty mppProp.TaskTp = property.MppTaskType newProps = append(newProps, mppProp) } + if newProp.TaskTp != property.CopSingleReadTaskType && p.SCtx().GetSessionVars().AllowProjectionPushDown && p.canPushToCop(kv.TiKV) && + expression.CanExprsPushDown(p.SCtx().GetSessionVars().StmtCtx, p.Exprs, p.SCtx().GetClient(), kv.TiKV) && !expression.ContainVirtualColumn(p.Exprs) { + copProp := newProp.CloneEssentialFields() + copProp.TaskTp = property.CopSingleReadTaskType + newProps = append(newProps, copProp) + } + ret := make([]PhysicalPlan, 0, len(newProps)) for _, newProp := range newProps { proj := PhysicalProjection{ diff --git a/planner/core/integration_test.go b/planner/core/integration_test.go index 765c16ee5e572..5afe5f3f01e19 100644 --- a/planner/core/integration_test.go +++ b/planner/core/integration_test.go @@ -4078,6 +4078,74 @@ func TestIssue32428(t *testing.T) { tk.MustQuery(`execute stmt using @a`).Check(testkit.Rows()) // empty result } +func TestPushDownProjectionForTiKV(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int, b real, i int, id int, value decimal(6,3), name char(128), d decimal(6,3), s char(128), t datetime, c bigint as ((a+1)) virtual, e real as ((b+a)))") + tk.MustExec("analyze table t") + tk.MustExec("set session tidb_opt_projection_push_down=1") + + var input []string + var output []struct { + SQL string + Plan []string + } + integrationSuiteData := core.GetIntegrationSuiteData() + integrationSuiteData.GetTestCases(t, &input, &output) + for i, tt := range input { + testdata.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + }) + res := tk.MustQuery(tt) + res.Check(testkit.Rows(output[i].Plan...)) + } +} + +func TestPushDownProjectionForTiFlashCoprocessor(t *testing.T) { + store, clean := testkit.CreateMockStore(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int, b real, i int, id int, value decimal(6,3), name char(128), d decimal(6,3), s char(128), t datetime, c bigint as ((a+1)) virtual, e real as ((b+a)))") + tk.MustExec("analyze table t") + tk.MustExec("set session tidb_opt_projection_push_down=1") + + // Create virtual tiflash replica info. + dom := domain.GetDomain(tk.Session()) + is := dom.InfoSchema() + db, exists := is.SchemaByName(model.NewCIStr("test")) + require.True(t, exists) + for _, tblInfo := range db.Tables { + if tblInfo.Name.L == "t" { + tblInfo.TiFlashReplica = &model.TiFlashReplicaInfo{ + Count: 1, + Available: true, + } + } + } + + var input []string + var output []struct { + SQL string + Plan []string + } + integrationSuiteData := core.GetIntegrationSuiteData() + integrationSuiteData.GetTestCases(t, &input, &output) + for i, tt := range input { + testdata.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = testdata.ConvertRowsToStrings(tk.MustQuery(tt).Rows()) + }) + res := tk.MustQuery(tt) + res.Check(testkit.Rows(output[i].Plan...)) + } +} + func TestPushDownProjectionForTiFlash(t *testing.T) { store, clean := testkit.CreateMockStore(t) defer clean() diff --git a/planner/core/physical_plans.go b/planner/core/physical_plans.go index 17e9f4537a12f..926e2d50d6d80 100644 --- a/planner/core/physical_plans.go +++ b/planner/core/physical_plans.go @@ -261,7 +261,7 @@ func (p *PhysicalIndexReader) SetSchema(_ *expression.Schema) { if p.indexPlan != nil { p.IndexPlans = flattenPushDownPlan(p.indexPlan) switch p.indexPlan.(type) { - case *PhysicalHashAgg, *PhysicalStreamAgg: + case *PhysicalHashAgg, *PhysicalStreamAgg, *PhysicalProjection: p.schema = p.indexPlan.Schema() default: is := p.IndexPlans[0].(*PhysicalIndexScan) diff --git a/planner/core/plan_to_pb.go b/planner/core/plan_to_pb.go index d4a5441045e69..6bf13553a958e 100644 --- a/planner/core/plan_to_pb.go +++ b/planner/core/plan_to_pb.go @@ -122,7 +122,7 @@ func (p *PhysicalProjection) ToPB(ctx sessionctx.Context, storeType kv.StoreType Exprs: exprs, } executorID := "" - if storeType == kv.TiFlash { + if storeType == kv.TiFlash || storeType == kv.TiKV { var err error projExec.Child, err = p.children[0].ToPB(ctx, storeType) if err != nil { @@ -130,7 +130,7 @@ func (p *PhysicalProjection) ToPB(ctx sessionctx.Context, storeType kv.StoreType } executorID = p.ExplainID().String() } else { - return nil, errors.Errorf("The projection can only be pushed down to TiFlash now, not %s", storeType.Name()) + return nil, errors.Errorf("the projection can only be pushed down to TiFlash or TiKV now, not %s", storeType.Name()) } return &tipb.Executor{Tp: tipb.ExecType_TypeProjection, Projection: projExec, ExecutorId: &executorID}, nil } diff --git a/planner/core/task.go b/planner/core/task.go index 522853c94c340..5d4793356572f 100644 --- a/planner/core/task.go +++ b/planner/core/task.go @@ -1460,7 +1460,7 @@ func (p *PhysicalProjection) GetCost(count float64) float64 { func (p *PhysicalProjection) attach2Task(tasks ...task) task { t := tasks[0].copy() if cop, ok := t.(*copTask); ok { - if len(cop.rootTaskConds) == 0 && cop.getStoreType() == kv.TiFlash && expression.CanExprsPushDown(p.ctx.GetSessionVars().StmtCtx, p.Exprs, p.ctx.GetClient(), cop.getStoreType()) { + if len(cop.rootTaskConds) == 0 && expression.CanExprsPushDown(p.ctx.GetSessionVars().StmtCtx, p.Exprs, p.ctx.GetClient(), cop.getStoreType()) { copTask := attachPlan2Task(p, cop) copTask.addCost(p.GetCost(t.count())) p.cost = copTask.cost() @@ -1475,7 +1475,6 @@ func (p *PhysicalProjection) attach2Task(tasks ...task) task { return mpp } } - // TODO: support projection push down for TiKV. t = t.convertToRootTask(p.ctx) t = attachPlan2Task(p, t) t.addCost(p.GetCost(t.count())) diff --git a/planner/core/testdata/integration_suite_in.json b/planner/core/testdata/integration_suite_in.json index a93847591c21a..a2a6c7e655efc 100644 --- a/planner/core/testdata/integration_suite_in.json +++ b/planner/core/testdata/integration_suite_in.json @@ -670,6 +670,52 @@ "explain format = 'brief' select /*+ inl_hash_join(s) */ * from t join s on t.a=s.a and t.a = s.b" ] }, + { + "name": "TestPushDownProjectionForTiKV", + "cases": [ + "desc format = 'brief' select i * 2 from t", + "desc format = 'brief' select DATE_FORMAT(t, '%Y-%m-%d %H') as date from t", + "desc format = 'brief' select md5(s) from t", + "desc format = 'brief' select c from t where a+1=3", + "desc format = 'brief' select /*+ hash_agg()*/ count(b) from (select id + 1 as b from t)A", + "desc format = 'brief' select /*+ hash_agg()*/ count(*) from (select id + 1 as b from t)A", + "desc format = 'brief' select /*+ hash_agg()*/ sum(b) from (select id + 1 as b from t)A", + "desc format = 'brief' select /*+ stream_agg()*/ count(b) from (select id + 1 as b from t)A", + "desc format = 'brief' select /*+ stream_agg()*/ count(*) from (select id + 1 as b from t)A", + "desc format = 'brief' select /*+ stream_agg()*/ sum(b) from (select id + 1 as b from t)A", + "desc format = 'brief' select * from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "desc format = 'brief' select * from t join (select id-2 as b from t) A on A.b=t.id", + "desc format = 'brief' select * from t left join (select id-2 as b from t) A on A.b=t.id", + "desc format = 'brief' select * from t right join (select id-2 as b from t) A on A.b=t.id", + "desc format = 'brief' select A.b, B.b from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "desc format = 'brief' select A.id from t as A where exists (select 1 from t where t.id=A.id)", + "desc format = 'brief' select A.id from t as A where not exists (select 1 from t where t.id=A.id)", + "desc format = 'brief' SELECT FROM_UNIXTIME(name,'%Y-%m-%d') FROM t;" + ] + }, + { + "name": "TestPushDownProjectionForTiFlashCoprocessor", + "cases": [ + "desc format = 'brief' select i * 2 from t", + "desc format = 'brief' select DATE_FORMAT(t, '%Y-%m-%d %H') as date from t", + "desc format = 'brief' select md5(s) from t", + "desc format = 'brief' select c from t where a+1=3", + "desc format = 'brief' select /*+ hash_agg()*/ count(b) from (select id + 1 as b from t)A", + "desc format = 'brief' select /*+ hash_agg()*/ count(*) from (select id + 1 as b from t)A", + "desc format = 'brief' select /*+ hash_agg()*/ sum(b) from (select id + 1 as b from t)A", + "desc format = 'brief' select /*+ stream_agg()*/ count(b) from (select id + 1 as b from t)A", + "desc format = 'brief' select /*+ stream_agg()*/ count(*) from (select id + 1 as b from t)A", + "desc format = 'brief' select /*+ stream_agg()*/ sum(b) from (select id + 1 as b from t)A", + "desc format = 'brief' select * from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "desc format = 'brief' select * from t join (select id-2 as b from t) A on A.b=t.id", + "desc format = 'brief' select * from t left join (select id-2 as b from t) A on A.b=t.id", + "desc format = 'brief' select * from t right join (select id-2 as b from t) A on A.b=t.id", + "desc format = 'brief' select A.b, B.b from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "desc format = 'brief' select A.id from t as A where exists (select 1 from t where t.id=A.id)", + "desc format = 'brief' select A.id from t as A where not exists (select 1 from t where t.id=A.id)", + "desc format = 'brief' SELECT FROM_UNIXTIME(name,'%Y-%m-%d') FROM t;" + ] + }, { "name": "TestPushDownProjectionForTiFlash", "cases": [ diff --git a/planner/core/testdata/integration_suite_out.json b/planner/core/testdata/integration_suite_out.json index 87da8449649af..b4ad84538d8b8 100644 --- a/planner/core/testdata/integration_suite_out.json +++ b/planner/core/testdata/integration_suite_out.json @@ -4335,6 +4335,393 @@ } ] }, + { + "Name": "TestPushDownProjectionForTiKV", + "Cases": [ + { + "SQL": "desc format = 'brief' select i * 2 from t", + "Plan": [ + "TableReader 10000.00 root data:Projection", + "└─Projection 10000.00 cop[tikv] mul(test.t.i, 2)->Column#13", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select DATE_FORMAT(t, '%Y-%m-%d %H') as date from t", + "Plan": [ + "TableReader 10000.00 root data:Projection", + "└─Projection 10000.00 cop[tikv] date_format(test.t.t, %Y-%m-%d %H)->Column#13", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select md5(s) from t", + "Plan": [ + "TableReader 10000.00 root data:Projection", + "└─Projection 10000.00 cop[tikv] md5(test.t.s)->Column#13", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select c from t where a+1=3", + "Plan": [ + "Projection 8000.00 root test.t.c", + "└─TableReader 8000.00 root data:Selection", + " └─Selection 8000.00 cop[tikv] eq(plus(test.t.a, 1), 3)", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ hash_agg()*/ count(b) from (select id + 1 as b from t)A", + "Plan": [ + "HashAgg 1.00 root funcs:count(Column#16)->Column#14", + "└─TableReader 1.00 root data:HashAgg", + " └─HashAgg 1.00 cop[tikv] funcs:count(plus(test.t.id, 1))->Column#16", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ hash_agg()*/ count(*) from (select id + 1 as b from t)A", + "Plan": [ + "HashAgg 1.00 root funcs:count(Column#15)->Column#14", + "└─TableReader 1.00 root data:HashAgg", + " └─HashAgg 1.00 cop[tikv] funcs:count(1)->Column#15", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ hash_agg()*/ sum(b) from (select id + 1 as b from t)A", + "Plan": [ + "HashAgg 1.00 root funcs:sum(Column#16)->Column#14", + "└─TableReader 1.00 root data:HashAgg", + " └─HashAgg 1.00 cop[tikv] funcs:sum(plus(test.t.id, 1))->Column#16", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ stream_agg()*/ count(b) from (select id + 1 as b from t)A", + "Plan": [ + "StreamAgg 1.00 root funcs:count(Column#16)->Column#14", + "└─TableReader 1.00 root data:StreamAgg", + " └─StreamAgg 1.00 cop[tikv] funcs:count(plus(test.t.id, 1))->Column#16", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ stream_agg()*/ count(*) from (select id + 1 as b from t)A", + "Plan": [ + "StreamAgg 1.00 root funcs:count(Column#15)->Column#14", + "└─TableReader 1.00 root data:StreamAgg", + " └─StreamAgg 1.00 cop[tikv] funcs:count(1)->Column#15", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ stream_agg()*/ sum(b) from (select id + 1 as b from t)A", + "Plan": [ + "StreamAgg 1.00 root funcs:sum(Column#16)->Column#14", + "└─TableReader 1.00 root data:StreamAgg", + " └─StreamAgg 1.00 cop[tikv] funcs:sum(plus(test.t.id, 1))->Column#16", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select * from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "Plan": [ + "HashJoin 10000.00 root inner join, equal:[eq(Column#13, Column#26)]", + "├─TableReader(Build) 8000.00 root data:Projection", + "│ └─Projection 8000.00 cop[tikv] minus(test.t.id, 2)->Column#26", + "│ └─Selection 8000.00 cop[tikv] not(isnull(minus(test.t.id, 2)))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "└─TableReader(Probe) 8000.00 root data:Projection", + " └─Projection 8000.00 cop[tikv] minus(test.t.id, 2)->Column#13", + " └─Selection 8000.00 cop[tikv] not(isnull(minus(test.t.id, 2)))", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select * from t join (select id-2 as b from t) A on A.b=t.id", + "Plan": [ + "HashJoin 10000.00 root inner join, equal:[eq(test.t.id, Column#25)]", + "├─TableReader(Build) 8000.00 root data:Projection", + "│ └─Projection 8000.00 cop[tikv] minus(test.t.id, 2)->Column#25", + "│ └─Selection 8000.00 cop[tikv] not(isnull(minus(test.t.id, 2)))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "└─TableReader(Probe) 9990.00 root data:Selection", + " └─Selection 9990.00 cop[tikv] not(isnull(test.t.id))", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select * from t left join (select id-2 as b from t) A on A.b=t.id", + "Plan": [ + "HashJoin 10000.00 root left outer join, equal:[eq(test.t.id, Column#25)]", + "├─TableReader(Build) 8000.00 root data:Projection", + "│ └─Projection 8000.00 cop[tikv] minus(test.t.id, 2)->Column#25", + "│ └─Selection 8000.00 cop[tikv] not(isnull(minus(test.t.id, 2)))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "└─TableReader(Probe) 10000.00 root data:TableFullScan", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select * from t right join (select id-2 as b from t) A on A.b=t.id", + "Plan": [ + "HashJoin 12487.50 root right outer join, equal:[eq(test.t.id, Column#25)]", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tikv] not(isnull(test.t.id))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "└─TableReader(Probe) 10000.00 root data:Projection", + " └─Projection 10000.00 cop[tikv] minus(test.t.id, 2)->Column#25", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select A.b, B.b from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "Plan": [ + "Projection 10000.00 root Column#26, Column#13", + "└─HashJoin 10000.00 root inner join, equal:[eq(Column#13, Column#26)]", + " ├─TableReader(Build) 8000.00 root data:Projection", + " │ └─Projection 8000.00 cop[tikv] minus(test.t.id, 2)->Column#26", + " │ └─Selection 8000.00 cop[tikv] not(isnull(minus(test.t.id, 2)))", + " │ └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + " └─TableReader(Probe) 8000.00 root data:Projection", + " └─Projection 8000.00 cop[tikv] minus(test.t.id, 2)->Column#13", + " └─Selection 8000.00 cop[tikv] not(isnull(minus(test.t.id, 2)))", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select A.id from t as A where exists (select 1 from t where t.id=A.id)", + "Plan": [ + "HashJoin 7992.00 root semi join, equal:[eq(test.t.id, test.t.id)]", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tikv] not(isnull(test.t.id))", + "│ └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "└─TableReader(Probe) 9990.00 root data:Selection", + " └─Selection 9990.00 cop[tikv] not(isnull(test.t.id))", + " └─TableFullScan 10000.00 cop[tikv] table:A keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select A.id from t as A where not exists (select 1 from t where t.id=A.id)", + "Plan": [ + "HashJoin 8000.00 root anti semi join, equal:[eq(test.t.id, test.t.id)]", + "├─TableReader(Build) 10000.00 root data:TableFullScan", + "│ └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo", + "└─TableReader(Probe) 10000.00 root data:TableFullScan", + " └─TableFullScan 10000.00 cop[tikv] table:A keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' SELECT FROM_UNIXTIME(name,'%Y-%m-%d') FROM t;", + "Plan": [ + "Projection 10000.00 root from_unixtime(cast(test.t.name, decimal(65,0) BINARY), %Y-%m-%d)->Column#13", + "└─TableReader 10000.00 root data:TableFullScan", + " └─TableFullScan 10000.00 cop[tikv] table:t keep order:false, stats:pseudo" + ] + } + ] + }, + { + "Name": "TestPushDownProjectionForTiFlashCoprocessor", + "Cases": [ + { + "SQL": "desc format = 'brief' select i * 2 from t", + "Plan": [ + "TableReader 10000.00 root data:Projection", + "└─Projection 10000.00 cop[tiflash] mul(test.t.i, 2)->Column#13", + " └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select DATE_FORMAT(t, '%Y-%m-%d %H') as date from t", + "Plan": [ + "TableReader 10000.00 root data:Projection", + "└─Projection 10000.00 cop[tiflash] date_format(test.t.t, %Y-%m-%d %H)->Column#13", + " └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select md5(s) from t", + "Plan": [ + "Projection 10000.00 root md5(test.t.s)->Column#13", + "└─TableReader 10000.00 root data:TableFullScan", + " └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select c from t where a+1=3", + "Plan": [ + "Projection 8000.00 root test.t.c", + "└─TableReader 8000.00 root data:Selection", + " └─Selection 8000.00 cop[tiflash] eq(plus(test.t.a, 1), 3)", + " └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ hash_agg()*/ count(b) from (select id + 1 as b from t)A", + "Plan": [ + "HashAgg 1.00 root funcs:count(Column#17)->Column#14", + "└─TableReader 1.00 root data:ExchangeSender", + " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(Column#18)->Column#17", + " └─Projection 10000.00 mpp[tiflash] plus(test.t.id, 1)->Column#18", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ hash_agg()*/ count(*) from (select id + 1 as b from t)A", + "Plan": [ + "HashAgg 1.00 root funcs:count(Column#16)->Column#14", + "└─TableReader 1.00 root data:ExchangeSender", + " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 mpp[tiflash] funcs:count(1)->Column#16", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ hash_agg()*/ sum(b) from (select id + 1 as b from t)A", + "Plan": [ + "HashAgg 1.00 root funcs:sum(Column#17)->Column#14", + "└─TableReader 1.00 root data:ExchangeSender", + " └─ExchangeSender 1.00 mpp[tiflash] ExchangeType: PassThrough", + " └─HashAgg 1.00 mpp[tiflash] funcs:sum(Column#18)->Column#17", + " └─Projection 10000.00 mpp[tiflash] cast(plus(test.t.id, 1), decimal(20,0) BINARY)->Column#18", + " └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ stream_agg()*/ count(b) from (select id + 1 as b from t)A", + "Plan": [ + "StreamAgg 1.00 root funcs:count(Column#16)->Column#14", + "└─TableReader 1.00 root data:StreamAgg", + " └─StreamAgg 1.00 batchCop[tiflash] funcs:count(Column#18)->Column#16", + " └─Projection 10000.00 batchCop[tiflash] plus(test.t.id, 1)->Column#18", + " └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ stream_agg()*/ count(*) from (select id + 1 as b from t)A", + "Plan": [ + "StreamAgg 1.00 root funcs:count(Column#15)->Column#14", + "└─TableReader 1.00 root data:StreamAgg", + " └─StreamAgg 1.00 batchCop[tiflash] funcs:count(1)->Column#15", + " └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select /*+ stream_agg()*/ sum(b) from (select id + 1 as b from t)A", + "Plan": [ + "StreamAgg 1.00 root funcs:sum(Column#16)->Column#14", + "└─TableReader 1.00 root data:StreamAgg", + " └─StreamAgg 1.00 batchCop[tiflash] funcs:sum(Column#18)->Column#16", + " └─Projection 10000.00 batchCop[tiflash] cast(plus(test.t.id, 1), decimal(20,0) BINARY)->Column#18", + " └─TableFullScan 10000.00 batchCop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select * from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "Plan": [ + "HashJoin 10000.00 root inner join, equal:[eq(Column#13, Column#26)]", + "├─TableReader(Build) 8000.00 root data:Projection", + "│ └─Projection 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#26", + "│ └─Selection 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + "│ └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + "└─TableReader(Probe) 8000.00 root data:Projection", + " └─Projection 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#13", + " └─Selection 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select * from t join (select id-2 as b from t) A on A.b=t.id", + "Plan": [ + "HashJoin 10000.00 root inner join, equal:[eq(test.t.id, Column#25)]", + "├─TableReader(Build) 8000.00 root data:Projection", + "│ └─Projection 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#25", + "│ └─Selection 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + "│ └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + "└─TableReader(Probe) 9990.00 root data:Selection", + " └─Selection 9990.00 cop[tiflash] not(isnull(test.t.id))", + " └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select * from t left join (select id-2 as b from t) A on A.b=t.id", + "Plan": [ + "HashJoin 10000.00 root left outer join, equal:[eq(test.t.id, Column#25)]", + "├─TableReader(Build) 8000.00 root data:Projection", + "│ └─Projection 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#25", + "│ └─Selection 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + "│ └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + "└─TableReader(Probe) 10000.00 root data:TableFullScan", + " └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select * from t right join (select id-2 as b from t) A on A.b=t.id", + "Plan": [ + "HashJoin 12487.50 root right outer join, equal:[eq(test.t.id, Column#25)]", + "├─TableReader(Build) 9990.00 root data:Selection", + "│ └─Selection 9990.00 cop[tiflash] not(isnull(test.t.id))", + "│ └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + "└─TableReader(Probe) 10000.00 root data:Projection", + " └─Projection 10000.00 cop[tiflash] minus(test.t.id, 2)->Column#25", + " └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select A.b, B.b from (select id-2 as b from t) B join (select id-2 as b from t) A on A.b=B.b", + "Plan": [ + "Projection 10000.00 root Column#26, Column#13", + "└─HashJoin 10000.00 root inner join, equal:[eq(Column#13, Column#26)]", + " ├─TableReader(Build) 8000.00 root data:Projection", + " │ └─Projection 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#26", + " │ └─Selection 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " │ └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo", + " └─TableReader(Probe) 8000.00 root data:Projection", + " └─Projection 8000.00 cop[tiflash] minus(test.t.id, 2)->Column#13", + " └─Selection 8000.00 cop[tiflash] not(isnull(minus(test.t.id, 2)))", + " └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select A.id from t as A where exists (select 1 from t where t.id=A.id)", + "Plan": [ + "TableReader 7992.00 root data:ExchangeSender", + "└─ExchangeSender 7992.00 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 7992.00 mpp[tiflash] semi join, equal:[eq(test.t.id, test.t.id)]", + " ├─ExchangeReceiver(Build) 9990.00 mpp[tiflash] ", + " │ └─ExchangeSender 9990.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─Selection 9990.00 mpp[tiflash] not(isnull(test.t.id))", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", + " └─Selection(Probe) 9990.00 mpp[tiflash] not(isnull(test.t.id))", + " └─TableFullScan 10000.00 mpp[tiflash] table:A keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' select A.id from t as A where not exists (select 1 from t where t.id=A.id)", + "Plan": [ + "TableReader 8000.00 root data:ExchangeSender", + "└─ExchangeSender 8000.00 mpp[tiflash] ExchangeType: PassThrough", + " └─HashJoin 8000.00 mpp[tiflash] anti semi join, equal:[eq(test.t.id, test.t.id)]", + " ├─ExchangeReceiver(Build) 10000.00 mpp[tiflash] ", + " │ └─ExchangeSender 10000.00 mpp[tiflash] ExchangeType: Broadcast", + " │ └─TableFullScan 10000.00 mpp[tiflash] table:t keep order:false, stats:pseudo", + " └─TableFullScan(Probe) 10000.00 mpp[tiflash] table:A keep order:false, stats:pseudo" + ] + }, + { + "SQL": "desc format = 'brief' SELECT FROM_UNIXTIME(name,'%Y-%m-%d') FROM t;", + "Plan": [ + "Projection 10000.00 root from_unixtime(cast(test.t.name, decimal(65,0) BINARY), %Y-%m-%d)->Column#13", + "└─TableReader 10000.00 root data:TableFullScan", + " └─TableFullScan 10000.00 cop[tiflash] table:t keep order:false, stats:pseudo" + ] + } + ] + }, { "Name": "TestPushDownProjectionForTiFlash", "Cases": [ diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 13c27b5562f83..47f75c7b9963f 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -731,6 +731,9 @@ type SessionVars struct { // EnablePipelinedWindowExec enables executing window functions in a pipelined manner. EnablePipelinedWindowExec bool + // AllowProjectionPushDown enables pushdown projection on TiKV. + AllowProjectionPushDown bool + // EnableStrictDoubleTypeCheck enables table field double type check. EnableStrictDoubleTypeCheck bool diff --git a/sessionctx/variable/sysvar.go b/sessionctx/variable/sysvar.go index 493f4a0791cc7..2bd3ab6fb11df 100644 --- a/sessionctx/variable/sysvar.go +++ b/sessionctx/variable/sysvar.go @@ -132,6 +132,10 @@ var defaultSysVars = []*SysVar{ } return nil }}, + {Scope: ScopeSession, Name: TiDBOptProjectionPushDown, Value: BoolToOnOff(config.GetGlobalConfig().Performance.ProjectionPushDown), skipInit: true, Type: TypeBool, SetSession: func(s *SessionVars, val string) error { + s.AllowProjectionPushDown = TiDBOptOn(val) + return nil + }}, {Scope: ScopeSession, Name: TiDBOptAggPushDown, Value: BoolToOnOff(DefOptAggPushDown), Type: TypeBool, skipInit: true, SetSession: func(s *SessionVars, val string) error { s.AllowAggPushDown = TiDBOptOn(val) return nil diff --git a/sessionctx/variable/tidb_vars.go b/sessionctx/variable/tidb_vars.go index 81cb3d7f9e17c..c5e45a5fb1c4d 100644 --- a/sessionctx/variable/tidb_vars.go +++ b/sessionctx/variable/tidb_vars.go @@ -443,6 +443,9 @@ const ( // TiDBEnableStrictDoubleTypeCheck is used to control table field double type syntax check. TiDBEnableStrictDoubleTypeCheck = "tidb_enable_strict_double_type_check" + // TiDBOptProjectionPushDown is used to control whether to pushdown projection to coprocessor. + TiDBOptProjectionPushDown = "tidb_opt_projection_push_down" + // TiDBEnableVectorizedExpression is used to control whether to enable the vectorized expression evaluation. TiDBEnableVectorizedExpression = "tidb_enable_vectorized_expression" diff --git a/tidb-server/main.go b/tidb-server/main.go index a9b09a952d429..bfa626636bf66 100644 --- a/tidb-server/main.go +++ b/tidb-server/main.go @@ -605,6 +605,7 @@ func setGlobalVars() { variable.SetSysVar(variable.TiDBForcePriority, mysql.Priority2Str[priority]) variable.SetSysVar(variable.TiDBOptDistinctAggPushDown, variable.BoolToOnOff(cfg.Performance.DistinctAggPushDown)) + variable.SetSysVar(variable.TiDBOptProjectionPushDown, variable.BoolToOnOff(cfg.Performance.ProjectionPushDown)) variable.SetSysVar(variable.TiDBMemQuotaQuery, strconv.FormatInt(cfg.MemQuotaQuery, 10)) variable.SetSysVar(variable.LogBin, variable.BoolToOnOff(cfg.Binlog.Enable)) variable.SetSysVar(variable.Port, fmt.Sprintf("%d", cfg.Port))