From cb0fa8278ff6318abefa44a97cdadd05e6c6843e Mon Sep 17 00:00:00 2001 From: Ti Chi Robot Date: Tue, 30 Jul 2024 16:26:49 +0800 Subject: [PATCH] planner: derive index filters for mv index paths (#54877) (#55027) close pingcap/tidb#54876 --- pkg/planner/core/indexmerge_path.go | 53 +++++++- .../r/planner/core/indexmerge_path.result | 124 ++++++++++++++++++ .../t/planner/core/indexmerge_path.test | 88 +++++++++++++ 3 files changed, 261 insertions(+), 4 deletions(-) diff --git a/pkg/planner/core/indexmerge_path.go b/pkg/planner/core/indexmerge_path.go index 9b6233e5d5928..0eb1f66dc74e2 100644 --- a/pkg/planner/core/indexmerge_path.go +++ b/pkg/planner/core/indexmerge_path.go @@ -1014,7 +1014,29 @@ func (ds *DataSource) generateIndexMerge4ComposedIndex(normalPathCnt int, indexM remainedCNFs = append(remainedCNFs, CNFItem) } } - mvp := ds.buildPartialPathUp4MVIndex(combinedPartialPaths, true, remainedCNFs, ds.tableStats.HistColl) + + condInIdxFilter := make(map[string]struct{}, len(remainedCNFs)) + // try to derive index filters for each path + for _, path := range combinedPartialPaths { + idxFilters, _ := ds.splitIndexFilterConditions(remainedCNFs, path.FullIdxCols, path.FullIdxColLens) + idxFilters = util.CloneExprs(idxFilters) + path.IndexFilters = append(path.IndexFilters, idxFilters...) + for _, idxFilter := range idxFilters { + condInIdxFilter[string(idxFilter.HashCode())] = struct{}{} + } + } + + // Collect the table filters. + // Since it's the intersection type index merge here, as long as a filter appears in one path, we don't need it in + // the table filters. + var tableFilters []expression.Expression + for _, CNFItem := range remainedCNFs { + if _, ok := condInIdxFilter[string(CNFItem.HashCode())]; !ok { + tableFilters = append(tableFilters, CNFItem) + } + } + + mvp := ds.buildPartialPathUp4MVIndex(combinedPartialPaths, true, tableFilters, ds.tableStats.HistColl) ds.possibleAccessPaths = append(ds.possibleAccessPaths, mvp) return nil @@ -1079,10 +1101,27 @@ func (ds *DataSource) generateIndexMerge4MVIndex(normalPathCnt int, filters []ex continue } + // Here, all partial paths are built from the same MV index, so we can directly use the first one to get the + // metadata. + // And according to buildPartialPaths4MVIndex, there must be at least one partial path if it returns ok. + firstPath := partialPaths[0] + idxFilters, tableFilters := ds.splitIndexFilterConditions( + remainingFilters, + firstPath.FullIdxCols, + firstPath.FullIdxColLens, + ) + + // Add the index filters to every partial path. + // For union type index merge, this is necessary for correctness. + for _, path := range partialPaths { + clonedIdxFilters := util.CloneExprs(idxFilters) + path.IndexFilters = append(path.IndexFilters, clonedIdxFilters...) + } + ds.possibleAccessPaths = append(ds.possibleAccessPaths, ds.buildPartialPathUp4MVIndex( partialPaths, isIntersection, - remainingFilters, + tableFilters, ds.tableStats.HistColl, ), ) @@ -1245,10 +1284,16 @@ func buildPartialPath4MVIndex( partialPath := &util.AccessPath{Index: mvIndex} partialPath.Ranges = ranger.FullRange() for i := 0; i < len(idxCols); i++ { + length := mvIndex.Columns[i].Length + // For full length prefix index, we consider it as non prefix index. + // This behavior is the same as in IndexInfo2Cols(), which is used for non mv index. + if length == idxCols[i].RetType.GetFlen() { + length = types.UnspecifiedLength + } partialPath.IdxCols = append(partialPath.IdxCols, idxCols[i]) - partialPath.IdxColLens = append(partialPath.IdxColLens, mvIndex.Columns[i].Length) + partialPath.IdxColLens = append(partialPath.IdxColLens, length) partialPath.FullIdxCols = append(partialPath.FullIdxCols, idxCols[i]) - partialPath.FullIdxColLens = append(partialPath.FullIdxColLens, mvIndex.Columns[i].Length) + partialPath.FullIdxColLens = append(partialPath.FullIdxColLens, length) } if err := detachCondAndBuildRangeForPath(sctx, partialPath, accessFilters, histColl); err != nil { return nil, false, err diff --git a/tests/integrationtest/r/planner/core/indexmerge_path.result b/tests/integrationtest/r/planner/core/indexmerge_path.result index 696560821af9f..cacad842c3594 100644 --- a/tests/integrationtest/r/planner/core/indexmerge_path.result +++ b/tests/integrationtest/r/planner/core/indexmerge_path.result @@ -1079,3 +1079,127 @@ IndexMerge 0.05 root type: union └─Selection(Probe) 0.05 cop[tikv] eq(planner__core__indexmerge_path.t.a, 1) └─TableRowIDScan 49.94 cop[tikv] table:t keep order:false, stats:pseudo SET @@tidb_opt_fix_control = default; +drop table if exists t, t1; +create table t (a int, b varchar(30), c float, j json, pk int primary key, +key mvi1(c, (cast(j->'$.a' as unsigned array)), b), +key mvi2(a, (cast(j->'$.c' as unsigned array))), +key mvi3((cast(j->'$.d' as unsigned array)), c), +key idx(b, c) +); +insert into t values (1, 'test', 1, '{"a":[3,4,5], "c":[7,8,9], "d":[10,11,12]}', 1); +insert into t values (2, 'text', 1, '{"a":[4,5,6], "c":[10,11,12], "d":[13,14,15]}', 2); +insert into t values (1, 'abcd', 1, '{"a":[7,8,9], "c":[13,14,15], "d":[16,17,18]}', 3); +SELECT /*+ use_index_merge(t, mvi1) */ * from t where +c = 1 and +json_overlaps(j->'$.a', '[4,5,6]') and +b not like '%test%'; +a b c j pk +2 text 1 {"a": [4, 5, 6], "c": [10, 11, 12], "d": [13, 14, 15]} 2 +EXPLAIN format = brief SELECT /*+ use_index_merge(t, mvi1) */ * from t where +c = 1 and +json_overlaps(j->'$.a', '[4,5,6]') and +b not like '%test%'; +id estRows task access object operator info +Selection 0.24 root json_overlaps(json_extract(planner__core__indexmerge_path.t.j, "$.a"), cast("[4,5,6]", json BINARY)) +└─IndexMerge 0.30 root type: union + ├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92)) + │ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 4,1 4], keep order:false, stats:pseudo + ├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92)) + │ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 5,1 5], keep order:false, stats:pseudo + ├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92)) + │ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 6,1 6], keep order:false, stats:pseudo + └─TableRowIDScan(Probe) 0.30 cop[tikv] table:t keep order:false, stats:pseudo +SELECT /*+ use_index_merge(t, mvi1) */ * from t where +c = 1 and +json_contains(j->'$.a', '[4,5]') and +b not like '%test%'; +a b c j pk +2 text 1 {"a": [4, 5, 6], "c": [10, 11, 12], "d": [13, 14, 15]} 2 +EXPLAIN format = brief SELECT /*+ use_index_merge(t, mvi1) */ * from t where +c = 1 and +json_contains(j->'$.a', '[4,5]') and +b not like '%test%'; +id estRows task access object operator info +IndexMerge 0.00 root type: intersection +├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92)) +│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 4,1 4], keep order:false, stats:pseudo +├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92)) +│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[1 5,1 5], keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo +SELECT /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where +a = 1 and +b > 'abc' and +b not like '%test%' and +c = 10 and +3 member of (j->'$.a') and +3 member of (j->'$.c'); +a b c j pk +EXPLAIN format=brief SELECT /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where +a = 1 and +b > 'abc' and +b not like '%test%' and +c = 10 and +3 member of (j->'$.a') and +3 member of (j->'$.c'); +id estRows task access object operator info +IndexMerge 0.00 root type: intersection +├─Selection(Build) 2.67 cop[tikv] eq(planner__core__indexmerge_path.t.c, 10), not(like(planner__core__indexmerge_path.t.b, "%test%", 92)), not(like(planner__core__indexmerge_path.t.b, "%test%", 92)) +│ └─IndexRangeScan 3333.33 cop[tikv] table:t, index:idx(b, c) range:("abc",+inf], keep order:false, stats:pseudo +├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t.b, "%test%", 92)) +│ └─IndexRangeScan 0.10 cop[tikv] table:t, index:mvi1(c, cast(json_extract(`j`, _utf8mb4'$.a') as unsigned array), b) range:[10 3,10 3], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 0.10 cop[tikv] table:t, index:mvi2(a, cast(json_extract(`j`, _utf8mb4'$.c') as unsigned array)) range:[1 3,1 3], keep order:false, stats:pseudo +└─TableRowIDScan(Probe) 0.00 cop[tikv] table:t keep order:false, stats:pseudo +create table t1 ( +a int, +b varchar(30), +c float, +d tinytext, +j json, +key mvi1(b(3), (cast(j as unsigned array))), +key mvi2((cast(j as unsigned array)), b), +key mvi3((cast(j as unsigned array)), d(30)), +key mvi4((cast(j as unsigned array)), d(255)) +); +EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi1) */ * from t1 where +c = 1 and +json_contains(j, '[4,5]') and +b = 'abcdefg'; +id estRows task access object operator info +TableReader 0.00 root data:Selection +└─Selection 0.00 cop[tikv] eq(planner__core__indexmerge_path.t1.b, "abcdefg"), eq(planner__core__indexmerge_path.t1.c, 1), json_contains(planner__core__indexmerge_path.t1.j, cast("[4,5]", json BINARY)) + └─TableFullScan 10000.00 cop[tikv] table:t1 keep order:false, stats:pseudo +EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi2) */ * from t1 where +c = 1 and +json_contains(j, '[4,5]') and +b = 'abcdefg' and +b like '%test%'; +id estRows task access object operator info +IndexMerge 0.00 root type: intersection +├─Selection(Build) 0.00 cop[tikv] like(planner__core__indexmerge_path.t1.b, "%test%", 92) +│ └─IndexRangeScan 0.10 cop[tikv] table:t1, index:mvi2(cast(`j` as unsigned array), b) range:[4 "abcdefg",4 "abcdefg"], keep order:false, stats:pseudo +├─Selection(Build) 0.00 cop[tikv] like(planner__core__indexmerge_path.t1.b, "%test%", 92) +│ └─IndexRangeScan 0.10 cop[tikv] table:t1, index:mvi2(cast(`j` as unsigned array), b) range:[5 "abcdefg",5 "abcdefg"], keep order:false, stats:pseudo +└─Selection(Probe) 0.00 cop[tikv] eq(planner__core__indexmerge_path.t1.c, 1) + └─TableRowIDScan 0.00 cop[tikv] table:t1 keep order:false, stats:pseudo +EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi3) */ * from t1 where +c = 1 and +json_contains(j, '[4,5]') and +d not like '%test%'; +id estRows task access object operator info +IndexMerge 0.01 root type: intersection +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:mvi3(cast(`j` as unsigned array), d) range:[4,4], keep order:false, stats:pseudo +├─IndexRangeScan(Build) 10.00 cop[tikv] table:t1, index:mvi3(cast(`j` as unsigned array), d) range:[5,5], keep order:false, stats:pseudo +└─Selection(Probe) 0.01 cop[tikv] eq(planner__core__indexmerge_path.t1.c, 1), not(like(planner__core__indexmerge_path.t1.d, "%test%", 92)) + └─TableRowIDScan 0.01 cop[tikv] table:t1 keep order:false, stats:pseudo +EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi4) */ * from t1 where +c = 1 and +json_contains(j, '[4,5]') and +d not like '%test%'; +id estRows task access object operator info +IndexMerge 0.01 root type: intersection +├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t1.d, "%test%", 92)) +│ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:mvi4(cast(`j` as unsigned array), d) range:[4,4], keep order:false, stats:pseudo +├─Selection(Build) 0.00 cop[tikv] not(like(planner__core__indexmerge_path.t1.d, "%test%", 92)) +│ └─IndexRangeScan 10.00 cop[tikv] table:t1, index:mvi4(cast(`j` as unsigned array), d) range:[5,5], keep order:false, stats:pseudo +└─Selection(Probe) 0.01 cop[tikv] eq(planner__core__indexmerge_path.t1.c, 1) + └─TableRowIDScan 0.01 cop[tikv] table:t1 keep order:false, stats:pseudo diff --git a/tests/integrationtest/t/planner/core/indexmerge_path.test b/tests/integrationtest/t/planner/core/indexmerge_path.test index e398b39dba679..cf3dcd7871496 100644 --- a/tests/integrationtest/t/planner/core/indexmerge_path.test +++ b/tests/integrationtest/t/planner/core/indexmerge_path.test @@ -425,3 +425,91 @@ EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b = EXPLAIN format = brief SELECT * FROM t WHERE a > 1 AND (b = '2' OR c = 3 OR b = '4' OR c = 5 OR b = '12' OR c = 13); EXPLAIN format = brief SELECT * FROM t WHERE a = 1 AND (c = 13 OR c = 15 OR c = 5 OR b = '12' OR c = 13 OR b = '11'); SET @@tidb_opt_fix_control = default; + +# Test deriving index filters for mv index paths +drop table if exists t, t1; +create table t (a int, b varchar(30), c float, j json, pk int primary key, +key mvi1(c, (cast(j->'$.a' as unsigned array)), b), +key mvi2(a, (cast(j->'$.c' as unsigned array))), +key mvi3((cast(j->'$.d' as unsigned array)), c), +key idx(b, c) +); + +insert into t values (1, 'test', 1, '{"a":[3,4,5], "c":[7,8,9], "d":[10,11,12]}', 1); +insert into t values (2, 'text', 1, '{"a":[4,5,6], "c":[10,11,12], "d":[13,14,15]}', 2); +insert into t values (1, 'abcd', 1, '{"a":[7,8,9], "c":[13,14,15], "d":[16,17,18]}', 3); + +# case 1: union type index merge on single mv index from single condition +SELECT /*+ use_index_merge(t, mvi1) */ * from t where +c = 1 and +json_overlaps(j->'$.a', '[4,5,6]') and +b not like '%test%'; + +EXPLAIN format = brief SELECT /*+ use_index_merge(t, mvi1) */ * from t where +c = 1 and +json_overlaps(j->'$.a', '[4,5,6]') and +b not like '%test%'; + +# case 2: intersection type index merge on single mv index from single condition +SELECT /*+ use_index_merge(t, mvi1) */ * from t where +c = 1 and +json_contains(j->'$.a', '[4,5]') and +b not like '%test%'; + +EXPLAIN format = brief SELECT /*+ use_index_merge(t, mvi1) */ * from t where +c = 1 and +json_contains(j->'$.a', '[4,5]') and +b not like '%test%'; + +# case 3: intersection type index merge on multiple indexes from different conditions +SELECT /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where +a = 1 and +b > 'abc' and +b not like '%test%' and +c = 10 and +3 member of (j->'$.a') and +3 member of (j->'$.c'); + +EXPLAIN format=brief SELECT /*+ use_index_merge(t, mvi1, mvi2, idx) */ * from t where +a = 1 and +b > 'abc' and +b not like '%test%' and +c = 10 and +3 member of (j->'$.a') and +3 member of (j->'$.c'); + +# case 4: test prefix indexes +create table t1 ( +a int, +b varchar(30), +c float, +d tinytext, +j json, +key mvi1(b(3), (cast(j as unsigned array))), +key mvi2((cast(j as unsigned array)), b), +key mvi3((cast(j as unsigned array)), d(30)), +key mvi4((cast(j as unsigned array)), d(255)) +); + + +EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi1) */ * from t1 where +c = 1 and +json_contains(j, '[4,5]') and +b = 'abcdefg'; + +EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi2) */ * from t1 where +c = 1 and +json_contains(j, '[4,5]') and +b = 'abcdefg' and +b like '%test%'; + +EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi3) */ * from t1 where +c = 1 and +json_contains(j, '[4,5]') and +d not like '%test%'; + +EXPLAIN format = brief SELECT /*+ use_index_merge(t1, mvi4) */ * from t1 where +c = 1 and +json_contains(j, '[4,5]') and +d not like '%test%'; +