diff --git a/sessionctx/variable/session.go b/sessionctx/variable/session.go index 1c46e53cb1878..a82db7b45d2df 100644 --- a/sessionctx/variable/session.go +++ b/sessionctx/variable/session.go @@ -950,10 +950,259 @@ type SessionVars struct { // ReadStaleness indicates the staleness duration for the following query ReadStaleness time.Duration +<<<<<<< HEAD // cached is used to optimze the object allocation. cached struct { curr int8 data [2]stmtctx.StatementContext +======= + // cachedStmtCtx is used to optimze the object allocation. + cachedStmtCtx [2]stmtctx.StatementContext + + // Rng stores the rand_seed1 and rand_seed2 for Rand() function + Rng *mathutil.MysqlRng + + // EnablePaging indicates whether enable paging in coprocessor requests. + EnablePaging bool + + // EnableLegacyInstanceScope says if SET SESSION can be used to set an instance + // scope variable. The default is TRUE. + EnableLegacyInstanceScope bool + + // ReadConsistency indicates the read consistency requirement. + ReadConsistency ReadConsistencyLevel + + // StatsLoadSyncWait indicates how long to wait for stats load before timeout. + StatsLoadSyncWait int64 + + // SysdateIsNow indicates whether Sysdate is an alias of Now function + SysdateIsNow bool + // EnableMutationChecker indicates whether to check data consistency for mutations + EnableMutationChecker bool + // AssertionLevel controls how strict the assertions on data mutations should be. + AssertionLevel AssertionLevel + // IgnorePreparedCacheCloseStmt controls if ignore the close-stmt command for prepared statement. + IgnorePreparedCacheCloseStmt bool + // EnableNewCostInterface is a internal switch to indicates whether to use the new cost calculation interface. + EnableNewCostInterface bool + // CostModelVersion is a internal switch to indicates the Cost Model Version. + CostModelVersion int + // IndexJoinDoubleReadPenaltyCostRate indicates whether to add some penalty cost to IndexJoin and how much of it. + IndexJoinDoubleReadPenaltyCostRate float64 + + // BatchPendingTiFlashCount shows the threshold of pending TiFlash tables when batch adding. + BatchPendingTiFlashCount int + // RcWriteCheckTS indicates whether some special write statements don't get latest tso from PD at RC + RcWriteCheckTS bool + // RemoveOrderbyInSubquery indicates whether to remove ORDER BY in subquery. + RemoveOrderbyInSubquery bool + // NonTransactionalIgnoreError indicates whether to ignore error in non-transactional statements. + // When set to false, returns immediately when it meets the first error. + NonTransactionalIgnoreError bool + + // MaxAllowedPacket indicates the maximum size of a packet for the MySQL protocol. + MaxAllowedPacket uint64 + + // TiFlash related optimization, only for MPP. + TiFlashFineGrainedShuffleStreamCount int64 + TiFlashFineGrainedShuffleBatchSize uint64 + + // RequestSourceType is the type of inner request. + RequestSourceType string + + // MemoryDebugModeMinHeapInUse indicated the minimum heapInUse threshold that triggers the memoryDebugMode. + MemoryDebugModeMinHeapInUse int64 + // MemoryDebugModeAlarmRatio indicated the allowable bias ratio of memory tracking accuracy check. + // When `(memory trakced by tidb) * (1+MemoryDebugModeAlarmRatio) < actual heapInUse`, an alarm log will be recorded. + MemoryDebugModeAlarmRatio int64 + + // EnableAnalyzeSnapshot indicates whether to read data on snapshot when collecting statistics. + // When it is false, ANALYZE reads the latest data. + // When it is true, ANALYZE reads data on the snapshot at the beginning of ANALYZE. + EnableAnalyzeSnapshot bool + + // DefaultStrMatchSelectivity adjust the estimation strategy for string matching expressions that can't be estimated by building into range. + // when > 0: it's the selectivity for the expression. + // when = 0: try to use TopN to evaluate the like expression to estimate the selectivity. + DefaultStrMatchSelectivity float64 + + // TiFlashFastScan indicates whether use fast scan in TiFlash + TiFlashFastScan bool + + // PrimaryKeyRequired indicates if sql_require_primary_key sysvar is set + PrimaryKeyRequired bool + + // EnablePreparedPlanCache indicates whether to enable prepared plan cache. + EnablePreparedPlanCache bool + + // PreparedPlanCacheSize controls the size of prepared plan cache. + PreparedPlanCacheSize uint64 + + // PreparedPlanCacheMonitor indicates whether to enable prepared plan cache monitor. + EnablePreparedPlanCacheMemoryMonitor bool + + // EnablePlanCacheForParamLimit controls whether the prepare statement with parameterized limit can be cached + EnablePlanCacheForParamLimit bool + + // EnablePlanCacheForSubquery controls whether the prepare statement with sub query can be cached + EnablePlanCacheForSubquery bool + + // EnableNonPreparedPlanCache indicates whether to enable non-prepared plan cache. + EnableNonPreparedPlanCache bool + + // EnableNonPreparedPlanCacheForDML indicates whether to enable non-prepared plan cache for DML statements. + EnableNonPreparedPlanCacheForDML bool + + // PlanCacheInvalidationOnFreshStats controls if plan cache will be invalidated automatically when + // related stats are analyzed after the plan cache is generated. + PlanCacheInvalidationOnFreshStats bool + + // NonPreparedPlanCacheSize controls the size of non-prepared plan cache. + NonPreparedPlanCacheSize uint64 + + // PlanCacheMaxPlanSize controls the maximum size of a plan that can be cached. + PlanCacheMaxPlanSize uint64 + + // SessionPlanCacheSize controls the size of session plan cache. + SessionPlanCacheSize uint64 + + // ConstraintCheckInPlacePessimistic controls whether to skip the locking of some keys in pessimistic transactions. + // Postpone the conflict check and constraint check to prewrite or later pessimistic locking requests. + ConstraintCheckInPlacePessimistic bool + + // EnableTiFlashReadForWriteStmt indicates whether to enable TiFlash to read for write statements. + EnableTiFlashReadForWriteStmt bool + + // EnableUnsafeSubstitute indicates whether to enable generate column takes unsafe substitute. + EnableUnsafeSubstitute bool + + // ForeignKeyChecks indicates whether to enable foreign key constraint check. + ForeignKeyChecks bool + + // RangeMaxSize is the max memory limit for ranges. When the optimizer estimates that the memory usage of complete + // ranges would exceed the limit, it chooses less accurate ranges such as full range. 0 indicates that there is no + // memory limit for ranges. + RangeMaxSize int64 + + // LastPlanReplayerToken indicates the last plan replayer token + LastPlanReplayerToken string + + // InPlanReplayer means we are now executing a statement for a PLAN REPLAYER SQL. + // Note that PLAN REPLAYER CAPTURE is not included here. + InPlanReplayer bool + + // AnalyzePartitionConcurrency indicates concurrency for partitions in Analyze + AnalyzePartitionConcurrency int + // AnalyzePartitionMergeConcurrency indicates concurrency for merging partition stats + AnalyzePartitionMergeConcurrency int + + // EnableExternalTSRead indicates whether to enable read through external ts + EnableExternalTSRead bool + + HookContext + + // MemTracker indicates the memory tracker of current session. + MemTracker *memory.Tracker + // MemDBDBFootprint tracks the memory footprint of memdb, and is attached to `MemTracker` + MemDBFootprint *memory.Tracker + DiskTracker *memory.Tracker + + // OptPrefixIndexSingleScan indicates whether to do some optimizations to avoid double scan for prefix index. + // When set to true, `col is (not) null`(`col` is index prefix column) is regarded as index filter rather than table filter. + OptPrefixIndexSingleScan bool + + // ChunkPool Several chunks and columns are cached + ChunkPool ReuseChunkPool + // EnableReuseCheck indicates request chunk whether use chunk alloc + EnableReuseCheck bool + + // EnableAdvancedJoinHint indicates whether the join method hint is compatible with join order hint. + EnableAdvancedJoinHint bool + + // preuseChunkAlloc indicates whether pre statement use chunk alloc + // like select @@last_sql_use_alloc + preUseChunkAlloc bool + + // EnablePlanReplayerCapture indicates whether enabled plan replayer capture + EnablePlanReplayerCapture bool + + // EnablePlanReplayedContinuesCapture indicates whether enabled plan replayer continues capture + EnablePlanReplayedContinuesCapture bool + + // PlanReplayerFinishedTaskKey used to record the finished plan replayer task key in order not to record the + // duplicate task in plan replayer continues capture + PlanReplayerFinishedTaskKey map[replayer.PlanReplayerTaskKey]struct{} + + // StoreBatchSize indicates the batch size limit of store batch, set this field to 0 to disable store batch. + StoreBatchSize int + + // shardRand is used by TxnCtx, for the GetCurrentShard() method. + shardRand *rand.Rand + + // Resource group name + ResourceGroupName string + + // PessimisticTransactionFairLocking controls whether fair locking for pessimistic transaction + // is enabled. + PessimisticTransactionFairLocking bool + + // EnableINLJoinInnerMultiPattern indicates whether enable multi pattern for index join inner side + // For now it is not public to user + EnableINLJoinInnerMultiPattern bool + + // Enable late materialization: push down some selection condition to tablescan. + EnableLateMaterialization bool + + // EnableRowLevelChecksum indicates whether row level checksum is enabled. + EnableRowLevelChecksum bool + + // TiFlashComputeDispatchPolicy indicates how to dipatch task to tiflash_compute nodes. + // Only for disaggregated-tiflash mode. + TiFlashComputeDispatchPolicy tiflashcompute.DispatchPolicy + + // SlowTxnThreshold is the threshold of slow transaction logs + SlowTxnThreshold uint64 + + // LoadBasedReplicaReadThreshold is the threshold for the estimated wait duration of a store. + // If exceeding the threshold, try other stores using replica read. + LoadBasedReplicaReadThreshold time.Duration + + // OptOrderingIdxSelThresh is the threshold for optimizer to consider the ordering index. + // If there exists an index whose estimated selectivity is smaller than this threshold, the optimizer won't + // use the ExpectedCnt to adjust the estimated row count for index scan. + OptOrderingIdxSelThresh float64 + + // EnableMPPSharedCTEExecution indicates whether we enable the shared CTE execution strategy on MPP side. + EnableMPPSharedCTEExecution bool + + // OptimizerFixControl control some details of the optimizer behavior through the tidb_opt_fix_control variable. + OptimizerFixControl map[uint64]string + + // HypoIndexes are for the Index Advisor. + HypoIndexes map[string]map[string]map[string]*model.IndexInfo // dbName -> tblName -> idxName -> idxInfo + + // Runtime Filter Group + // Runtime filter type: only support IN or MIN_MAX now. + // Runtime filter type can take multiple values at the same time. + runtimeFilterTypes []RuntimeFilterType + // Runtime filter mode: only support OFF, LOCAL now + runtimeFilterMode RuntimeFilterMode +} + +var ( + // variables below are for the optimizer fix control. + + // TiDBOptFixControl44262 controls whether to allow to use dynamic-mode to access partitioning tables without global-stats (#44262). + TiDBOptFixControl44262 uint64 = 44262 + // TiDBOptFixControl44389 controls whether to consider non-point ranges of some CNF item when building ranges. + TiDBOptFixControl44389 uint64 = 44389 +) + +// GetOptimizerFixControlValue returns the specified value of the optimizer fix control. +func (s *SessionVars) GetOptimizerFixControlValue(key uint64) (value string, exist bool) { + if s.OptimizerFixControl == nil { + return "", false +>>>>>>> 85d6323e3a3 (util/ranger: consider good non-point ranges from CNF item (#44384)) } } diff --git a/util/ranger/BUILD.bazel b/util/ranger/BUILD.bazel new file mode 100644 index 0000000000000..be470b8c15d6c --- /dev/null +++ b/util/ranger/BUILD.bazel @@ -0,0 +1,70 @@ +load("@io_bazel_rules_go//go:def.bzl", "go_library", "go_test") + +go_library( + name = "ranger", + srcs = [ + "checker.go", + "detacher.go", + "points.go", + "ranger.go", + "types.go", + ], + importpath = "github.com/pingcap/tidb/util/ranger", + visibility = ["//visibility:public"], + deps = [ + "//errno", + "//expression", + "//kv", + "//parser/ast", + "//parser/charset", + "//parser/format", + "//parser/model", + "//parser/mysql", + "//parser/terror", + "//sessionctx", + "//sessionctx/stmtctx", + "//sessionctx/variable", + "//types", + "//types/parser_driver", + "//util/chunk", + "//util/codec", + "//util/collate", + "//util/dbterror", + "//util/mathutil", + "@com_github_pingcap_errors//:errors", + "@org_golang_x_exp//slices", + ], +) + +go_test( + name = "ranger_test", + timeout = "short", + srcs = [ + "bench_test.go", + "main_test.go", + "ranger_test.go", + "types_test.go", + ], + data = glob(["testdata/**"]), + flaky = True, + shard_count = 26, + deps = [ + ":ranger", + "//config", + "//expression", + "//parser/ast", + "//parser/model", + "//parser/mysql", + "//planner/core", + "//session", + "//sessionctx", + "//sessionctx/variable", + "//testkit", + "//testkit/testdata", + "//testkit/testsetup", + "//types", + "//util/collate", + "@com_github_stretchr_testify//require", + "@org_uber_go_goleak//:goleak", + ], +) diff --git a/util/ranger/detacher.go b/util/ranger/detacher.go index 442ca51e36bde..197f6ac7e2554 100644 --- a/util/ranger/detacher.go +++ b/util/ranger/detacher.go @@ -24,9 +24,11 @@ import ( "github.com/pingcap/tidb/parser/model" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/sessionctx/stmtctx" + "github.com/pingcap/tidb/sessionctx/variable" "github.com/pingcap/tidb/types" "github.com/pingcap/tidb/util/chunk" "github.com/pingcap/tidb/util/collate" + "github.com/pingcap/tidb/util/mathutil" ) // detachColumnCNFConditions detaches the condition for calculating range from the other conditions. @@ -184,18 +186,71 @@ func getPotentialEqOrInColOffset(expr expression.Expression, cols []*expression. return -1 } -// extractIndexPointRangesForCNF extracts a CNF item from the input CNF expressions, such that the CNF item -// is totally composed of point range filters. +type cnfItemRangeResult struct { + rangeResult *DetachRangeResult + offset int + // sameLenPointRanges means that each range is point range and all of them have the same column numbers(i.e., maxColNum = minColNum). + sameLenPointRanges bool + maxColNum int + minColNum int +} + +func getCNFItemRangeResult(sctx sessionctx.Context, rangeResult *DetachRangeResult, offset int) *cnfItemRangeResult { + sameLenPointRanges := true + var maxColNum, minColNum int + for i, ran := range rangeResult.Ranges { + if !ran.IsPoint(sctx) { + sameLenPointRanges = false + } + if i == 0 { + maxColNum = len(ran.LowVal) + minColNum = len(ran.LowVal) + } else { + maxColNum = mathutil.Max(maxColNum, len(ran.LowVal)) + minColNum = mathutil.Min(minColNum, len(ran.LowVal)) + } + } + if minColNum != maxColNum { + sameLenPointRanges = false + } + return &cnfItemRangeResult{ + rangeResult: rangeResult, + offset: offset, + sameLenPointRanges: sameLenPointRanges, + maxColNum: maxColNum, + minColNum: minColNum, + } +} + +func compareCNFItemRangeResult(curResult, bestResult *cnfItemRangeResult) (curIsBetter bool) { + if curResult.sameLenPointRanges && bestResult.sameLenPointRanges { + return curResult.minColNum > bestResult.minColNum + } + if !curResult.sameLenPointRanges && !bestResult.sameLenPointRanges { + if curResult.minColNum == bestResult.minColNum { + return curResult.maxColNum > bestResult.maxColNum + } + return curResult.minColNum > bestResult.minColNum + } + // Point ranges is better than non-point ranges since we can append subsequent column ranges to point ranges. + return curResult.sameLenPointRanges +} + +// extractBestCNFItemRanges builds ranges for each CNF item from the input CNF expressions and returns the best CNF +// item ranges. // e.g, for input CNF expressions ((a,b) in ((1,1),(2,2))) and a > 1 and ((a,b,c) in (1,1,1),(2,2,2)) // ((a,b,c) in (1,1,1),(2,2,2)) would be extracted. +<<<<<<< HEAD func extractIndexPointRangesForCNF(sctx sessionctx.Context, conds []expression.Expression, cols []*expression.Column, lengths []int) (*DetachRangeResult, int, []*valueInfo, error) { +======= +func extractBestCNFItemRanges(sctx sessionctx.Context, conds []expression.Expression, cols []*expression.Column, + lengths []int, rangeMaxSize int64) (*cnfItemRangeResult, []*valueInfo, error) { +>>>>>>> 85d6323e3a3 (util/ranger: consider good non-point ranges from CNF item (#44384)) if len(conds) < 2 { - return nil, -1, nil, nil + return nil, nil, nil } - var r *DetachRangeResult + var bestRes *cnfItemRangeResult columnValues := make([]*valueInfo, len(cols)) - maxNumCols := int(0) - offset := int(-1) for i, cond := range conds { tmpConds := []expression.Expression{cond} colSets := expression.ExtractColumnSet(cond) @@ -204,16 +259,17 @@ func extractIndexPointRangesForCNF(sctx sessionctx.Context, conds []expression.E } res, err := DetachCondAndBuildRangeForIndex(sctx, tmpConds, cols, lengths) if err != nil { - return nil, -1, nil, err + return nil, nil, err } if len(res.Ranges) == 0 { - return &DetachRangeResult{}, -1, nil, nil + return &cnfItemRangeResult{rangeResult: res, offset: i}, nil, nil } // take the union of the two columnValues columnValues = unionColumnValues(columnValues, res.ColumnValues) if len(res.AccessConds) == 0 || len(res.RemainedConds) > 0 { continue } +<<<<<<< HEAD sameLens, allPoints := true, true numCols := int(0) for j, ran := range res.Ranges { @@ -235,12 +291,17 @@ func extractIndexPointRangesForCNF(sctx sessionctx.Context, conds []expression.E r = res offset = i maxNumCols = numCols +======= + curRes := getCNFItemRangeResult(sctx, res, i) + if bestRes == nil || compareCNFItemRangeResult(curRes, bestRes) { + bestRes = curRes +>>>>>>> 85d6323e3a3 (util/ranger: consider good non-point ranges from CNF item (#44384)) } } - if r != nil { - r.IsDNFCond = false + if bestRes != nil && bestRes.rangeResult != nil { + bestRes.rangeResult.IsDNFCond = false } - return r, offset, columnValues, nil + return bestRes, columnValues, nil } func unionColumnValues(lhs, rhs []*valueInfo) []*valueInfo { @@ -314,27 +375,50 @@ func (d *rangeDetacher) detachCNFCondAndBuildRangeForIndex(conditions []expressi shouldReserve: d.lengths[eqOrInCount] != types.UnspecifiedLength, } if considerDNF { +<<<<<<< HEAD pointRes, offset, columnValues, err := extractIndexPointRangesForCNF(d.sctx, conditions, d.cols, d.lengths) +======= + bestCNFItemRes, columnValues, err := extractBestCNFItemRanges(d.sctx, conditions, d.cols, d.lengths, d.rangeMaxSize) +>>>>>>> 85d6323e3a3 (util/ranger: consider good non-point ranges from CNF item (#44384)) if err != nil { return nil, err } res.ColumnValues = unionColumnValues(res.ColumnValues, columnValues) - if pointRes != nil { - if len(pointRes.Ranges) == 0 { + if bestCNFItemRes != nil && bestCNFItemRes.rangeResult != nil { + if len(bestCNFItemRes.rangeResult.Ranges) == 0 { return &DetachRangeResult{}, nil } - if len(pointRes.Ranges[0].LowVal) > eqOrInCount { - pointRes.ColumnValues = res.ColumnValues - res = pointRes - pointRanges = pointRes.Ranges + if bestCNFItemRes.sameLenPointRanges && bestCNFItemRes.minColNum > eqOrInCount { + bestCNFItemRes.rangeResult.ColumnValues = res.ColumnValues + res = bestCNFItemRes.rangeResult + pointRanges = bestCNFItemRes.rangeResult.Ranges eqOrInCount = len(res.Ranges[0].LowVal) newConditions = newConditions[:0] - newConditions = append(newConditions, conditions[:offset]...) - newConditions = append(newConditions, conditions[offset+1:]...) + newConditions = append(newConditions, conditions[:bestCNFItemRes.offset]...) + newConditions = append(newConditions, conditions[bestCNFItemRes.offset+1:]...) if eqOrInCount == len(d.cols) || len(newConditions) == 0 { res.RemainedConds = append(res.RemainedConds, newConditions...) return res, nil } + } else { + considerCNFItemNonPointRanges := false + fixValue, ok := d.sctx.GetSessionVars().GetOptimizerFixControlValue(variable.TiDBOptFixControl44389) + if ok && variable.TiDBOptOn(fixValue) { + considerCNFItemNonPointRanges = true + } + if considerCNFItemNonPointRanges && !bestCNFItemRes.sameLenPointRanges && eqOrInCount == 0 && bestCNFItemRes.minColNum > 0 && bestCNFItemRes.maxColNum > 1 { + // When eqOrInCount is 0, if we don't enter the IF branch, we would use detachColumnCNFConditions to build + // ranges on the first index column. + // Considering minColNum > 0 and maxColNum > 1, bestCNFItemRes is better than the ranges built by detachColumnCNFConditions + // in most cases. + bestCNFItemRes.rangeResult.ColumnValues = res.ColumnValues + res = bestCNFItemRes.rangeResult + newConditions = newConditions[:0] + newConditions = append(newConditions, conditions[:bestCNFItemRes.offset]...) + newConditions = append(newConditions, conditions[bestCNFItemRes.offset+1:]...) + res.RemainedConds = append(res.RemainedConds, newConditions...) + return res, nil + } } } if eqOrInCount > 0 { diff --git a/util/ranger/ranger_test.go b/util/ranger/ranger_test.go index cd2d42572e5f5..9ef88bb77ab01 100644 --- a/util/ranger/ranger_test.go +++ b/util/ranger/ranger_test.go @@ -1399,6 +1399,36 @@ func TestCompIndexMultiColDNF2(t *testing.T) { } } +<<<<<<< HEAD +======= +func TestIssue41572(t *testing.T) { + store := testkit.CreateMockStore(t) + + testKit := testkit.NewTestKit(t, store) + testKit.MustExec("use test") + testKit.MustExec("drop table if exists t") + testKit.MustExec("create table t(a varchar(100), b int, c int, d int, index idx(a, b, c))") + testKit.MustExec("insert into t values ('t',1,1,1),('t',1,3,3),('t',2,1,3),('t',2,3,1),('w',0,3,3),('z',0,1,1)") + + var input []string + var output []struct { + SQL string + Plan []string + Result []string + } + rangerSuiteData.LoadTestCases(t, &input, &output) + for i, tt := range input { + testdata.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = testdata.ConvertRowsToStrings(testKit.MustQuery("explain " + tt).Rows()) + output[i].Result = testdata.ConvertRowsToStrings(testKit.MustQuery(tt).Sort().Rows()) + }) + testKit.MustQuery("explain " + tt).Check(testkit.Rows(output[i].Plan...)) + testKit.MustQuery(tt).Sort().Check(testkit.Rows(output[i].Result...)) + } +} + +>>>>>>> 85d6323e3a3 (util/ranger: consider good non-point ranges from CNF item (#44384)) func TestPrefixIndexMultiColDNF(t *testing.T) { t.Parallel() dom, store, err := newDomainStoreWithBootstrap(t) @@ -1780,3 +1810,1248 @@ func TestPrefixIndexAppendPointRanges(t *testing.T) { testKit.MustQuery(tt).Check(testkit.Rows(output[i].Result...)) } } +<<<<<<< HEAD +======= + +func TestIndexRange(t *testing.T) { + store := testkit.CreateMockStore(t) + + testKit := testkit.NewTestKit(t, store) + testKit.MustExec("use test") + testKit.MustExec("drop table if exists t") + testKit.MustExec(` +create table t( + a varchar(50), + b int, + c double, + d varchar(10), + e binary(10), + f varchar(10) collate utf8mb4_general_ci, + g enum('A','B','C') collate utf8mb4_general_ci, + h varchar(10) collate utf8_bin, + index idx_ab(a(50), b), + index idx_cb(c, a), + index idx_d(d(2)), + index idx_e(e(2)), + index idx_f(f), + index idx_de(d(2), e), + index idx_g(g), + index idx_h(h(3)) +)`) + + tests := []struct { + indexPos int + exprStr string + accessConds string + filterConds string + resultStr string + }{ + { + indexPos: 0, + exprStr: `a LIKE 'abc%'`, + accessConds: `[like(test.t.a, abc%, 92)]`, + filterConds: "[]", + resultStr: "[[\"abc\",\"abd\")]", + }, + { + indexPos: 0, + exprStr: "a LIKE 'abc_'", + accessConds: "[like(test.t.a, abc_, 92)]", + filterConds: "[like(test.t.a, abc_, 92)]", + resultStr: "[(\"abc\",\"abd\")]", + }, + { + indexPos: 0, + exprStr: "a LIKE 'abc'", + accessConds: "[like(test.t.a, abc, 92)]", + filterConds: "[]", + resultStr: "[[\"abc\",\"abc\"]]", + }, + { + indexPos: 0, + exprStr: `a LIKE "ab\_c"`, + accessConds: "[like(test.t.a, ab\\_c, 92)]", + filterConds: "[]", + resultStr: "[[\"ab_c\",\"ab_c\"]]", + }, + { + indexPos: 0, + exprStr: `a LIKE '%'`, + accessConds: "[]", + filterConds: `[like(test.t.a, %, 92)]`, + resultStr: "[[NULL,+inf]]", + }, + { + indexPos: 0, + exprStr: `a LIKE '\%a'`, + accessConds: "[like(test.t.a, \\%a, 92)]", + filterConds: "[]", + resultStr: `[["%a","%a"]]`, + }, + { + indexPos: 0, + exprStr: `a LIKE "\\"`, + accessConds: "[like(test.t.a, \\, 92)]", + filterConds: "[]", + resultStr: "[[\"\\\\\",\"\\\\\"]]", + }, + { + indexPos: 0, + exprStr: `a LIKE "\\\\a%"`, + accessConds: `[like(test.t.a, \\a%, 92)]`, + filterConds: "[]", + resultStr: "[[\"\\\\a\",\"\\\\b\")]", + }, + { + indexPos: 0, + exprStr: `a > NULL`, + accessConds: "[gt(test.t.a, )]", + filterConds: "[]", + resultStr: `[]`, + }, + { + indexPos: 0, + exprStr: `a = 'a' and b in (1, 2, 3)`, + accessConds: "[eq(test.t.a, a) in(test.t.b, 1, 2, 3)]", + filterConds: "[]", + resultStr: "[[\"a\" 1,\"a\" 1] [\"a\" 2,\"a\" 2] [\"a\" 3,\"a\" 3]]", + }, + { + indexPos: 0, + exprStr: `a = 'a' and b not in (1, 2, 3)`, + accessConds: "[eq(test.t.a, a) not(in(test.t.b, 1, 2, 3))]", + filterConds: "[]", + resultStr: "[(\"a\" NULL,\"a\" 1) (\"a\" 3,\"a\" +inf]]", + }, + { + indexPos: 0, + exprStr: `a in ('a') and b in ('1', 2.0, NULL)`, + accessConds: "[eq(test.t.a, a) in(test.t.b, 1, 2, )]", + filterConds: "[]", + resultStr: `[["a" 1,"a" 1] ["a" 2,"a" 2]]`, + }, + { + indexPos: 1, + exprStr: `c in ('1.1', 1, 1.1) and a in ('1', 'a', NULL)`, + accessConds: "[in(test.t.c, 1.1, 1, 1.1) in(test.t.a, 1, a, )]", + filterConds: "[]", + resultStr: "[[1 \"1\",1 \"1\"] [1 \"a\",1 \"a\"] [1.1 \"1\",1.1 \"1\"] [1.1 \"a\",1.1 \"a\"]]", + }, + { + indexPos: 1, + exprStr: "c in (1, 1, 1, 1, 1, 1, 2, 1, 2, 3, 2, 3, 4, 4, 1, 2)", + accessConds: "[in(test.t.c, 1, 1, 1, 1, 1, 1, 2, 1, 2, 3, 2, 3, 4, 4, 1, 2)]", + filterConds: "[]", + resultStr: "[[1,1] [2,2] [3,3] [4,4]]", + }, + { + indexPos: 1, + exprStr: "c not in (1, 2, 3)", + accessConds: "[not(in(test.t.c, 1, 2, 3))]", + filterConds: "[]", + resultStr: "[(NULL,1) (1,2) (2,3) (3,+inf]]", + }, + { + indexPos: 1, + exprStr: "c in (1, 2) and c in (1, 3)", + accessConds: "[eq(test.t.c, 1)]", + filterConds: "[]", + resultStr: "[[1,1]]", + }, + { + indexPos: 1, + exprStr: "c = 1 and c = 2", + accessConds: "[]", + filterConds: "[]", + resultStr: "[]", + }, + { + indexPos: 0, + exprStr: "a in (NULL)", + accessConds: "[eq(test.t.a, )]", + filterConds: "[]", + resultStr: "[]", + }, + { + indexPos: 0, + exprStr: "a not in (NULL, '1', '2', '3')", + accessConds: "[not(in(test.t.a, , 1, 2, 3))]", + filterConds: "[]", + resultStr: "[]", + }, + { + indexPos: 0, + exprStr: "not (a not in (NULL, '1', '2', '3') and a > '2')", + accessConds: "[or(in(test.t.a, , 1, 2, 3), le(test.t.a, 2))]", + filterConds: "[]", + resultStr: "[[-inf,\"2\"] [\"3\",\"3\"]]", + }, + { + indexPos: 0, + exprStr: "not (a not in (NULL) and a > '2')", + accessConds: "[or(eq(test.t.a, ), le(test.t.a, 2))]", + filterConds: "[]", + resultStr: "[[-inf,\"2\"]]", + }, + { + indexPos: 0, + exprStr: "not (a not in (NULL) or a > '2')", + accessConds: "[and(eq(test.t.a, ), le(test.t.a, 2))]", + filterConds: "[]", + resultStr: "[]", + }, + { + indexPos: 0, + exprStr: "(a > 'b' and a < 'bbb') or (a < 'cb' and a > 'a')", + accessConds: "[or(and(gt(test.t.a, b), lt(test.t.a, bbb)), and(lt(test.t.a, cb), gt(test.t.a, a)))]", + filterConds: "[]", + resultStr: "[(\"a\",\"cb\")]", + }, + { + indexPos: 0, + exprStr: "(a > 'a' and a < 'b') or (a >= 'b' and a < 'c')", + accessConds: "[or(and(gt(test.t.a, a), lt(test.t.a, b)), and(ge(test.t.a, b), lt(test.t.a, c)))]", + filterConds: "[]", + resultStr: "[(\"a\",\"c\")]", + }, + { + indexPos: 0, + exprStr: "(a > 'a' and a < 'b' and b < 1) or (a >= 'b' and a < 'c')", + accessConds: "[or(and(gt(test.t.a, a), lt(test.t.a, b)), and(ge(test.t.a, b), lt(test.t.a, c)))]", + filterConds: "[or(and(and(gt(test.t.a, a), lt(test.t.a, b)), lt(test.t.b, 1)), and(ge(test.t.a, b), lt(test.t.a, c)))]", + resultStr: "[(\"a\",\"c\")]", + }, + { + indexPos: 0, + exprStr: "(a in ('a', 'b') and b < 1) or (a >= 'b' and a < 'c')", + accessConds: "[or(and(in(test.t.a, a, b), lt(test.t.b, 1)), and(ge(test.t.a, b), lt(test.t.a, c)))]", + filterConds: "[]", + resultStr: `[["a" -inf,"a" 1) ["b","c")]`, + }, + { + indexPos: 0, + exprStr: "(a > 'a') or (c > 1)", + accessConds: "[]", + filterConds: "[or(gt(test.t.a, a), gt(test.t.c, 1))]", + resultStr: "[[NULL,+inf]]", + }, + { + indexPos: 2, + exprStr: `d = "你好啊"`, + accessConds: "[eq(test.t.d, 你好啊)]", + filterConds: "[eq(test.t.d, 你好啊)]", + resultStr: "[[\"你好\",\"你好\"]]", + }, + { + indexPos: 3, + exprStr: `e = "你好啊"`, + accessConds: "[eq(test.t.e, 你好啊)]", + filterConds: "[eq(test.t.e, 你好啊)]", + resultStr: "[[0xE4BD,0xE4BD]]", + }, + { + indexPos: 2, + exprStr: `d in ("你好啊", "再见")`, + accessConds: "[in(test.t.d, 你好啊, 再见)]", + filterConds: "[in(test.t.d, 你好啊, 再见)]", + resultStr: "[[\"你好\",\"你好\"] [\"再见\",\"再见\"]]", + }, + { + indexPos: 2, + exprStr: `d not in ("你好啊")`, + accessConds: "[]", + filterConds: "[ne(test.t.d, 你好啊)]", + resultStr: "[[NULL,+inf]]", + }, + { + indexPos: 2, + exprStr: `d < "你好" || d > "你好"`, + accessConds: "[or(lt(test.t.d, 你好), gt(test.t.d, 你好))]", + filterConds: "[or(lt(test.t.d, 你好), gt(test.t.d, 你好))]", + resultStr: "[[-inf,+inf]]", + }, + { + indexPos: 2, + exprStr: `not(d < "你好" || d > "你好")`, + accessConds: "[and(ge(test.t.d, 你好), le(test.t.d, 你好))]", + filterConds: "[and(ge(test.t.d, 你好), le(test.t.d, 你好))]", + resultStr: "[[\"你好\",\"你好\"]]", + }, + { + indexPos: 4, + exprStr: "f >= 'a' and f <= 'B'", + accessConds: "[ge(test.t.f, a) le(test.t.f, B)]", + filterConds: "[]", + resultStr: "[[\"a\",\"B\"]]", + }, + { + indexPos: 4, + exprStr: "f in ('a', 'B')", + accessConds: "[in(test.t.f, a, B)]", + filterConds: "[]", + resultStr: "[[\"a\",\"a\"] [\"B\",\"B\"]]", + }, + { + indexPos: 4, + exprStr: "f = 'a' and f = 'B' collate utf8mb4_bin", + accessConds: "[eq(test.t.f, a)]", + filterConds: "[eq(test.t.f, B)]", + resultStr: "[[\"a\",\"a\"]]", + }, + { + indexPos: 4, + exprStr: "f like '@%' collate utf8mb4_bin", + accessConds: "[]", + filterConds: "[like(test.t.f, @%, 92)]", + resultStr: "[[NULL,+inf]]", + }, + { + indexPos: 5, + exprStr: "d in ('aab', 'aac') and e = 'a'", + accessConds: "[in(test.t.d, aab, aac) eq(test.t.e, a)]", + filterConds: "[in(test.t.d, aab, aac)]", + resultStr: "[[\"aa\" 0x61,\"aa\" 0x61]]", + }, + { + indexPos: 6, + exprStr: "g = 'a'", + accessConds: "[eq(test.t.g, a)]", + filterConds: "[]", + resultStr: "[[\"A\",\"A\"]]", + }, + { + indexPos: 7, + exprStr: `h LIKE 'ÿÿ%'`, + accessConds: `[like(test.t.h, ÿÿ%, 92)]`, + filterConds: "[like(test.t.h, ÿÿ%, 92)]", + resultStr: "[[\"ÿÿ\",\"ÿ\\xc3\\xc0\")]", // The decoding error is ignored. + }, + } + + ctx := context.Background() + for _, tt := range tests { + t.Run(tt.exprStr, func(t *testing.T) { + sql := "select * from t where " + tt.exprStr + sctx := testKit.Session().(sessionctx.Context) + stmts, err := session.Parse(sctx, sql) + require.NoError(t, err) + require.Len(t, stmts, 1) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(context.Background(), sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) + require.NoError(t, err) + p, _, err := plannercore.BuildLogicalPlanForTest(ctx, sctx, stmts[0], ret.InfoSchema) + require.NoError(t, err) + selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) + tbl := selection.Children()[0].(*plannercore.DataSource).TableInfo() + require.NotNil(t, selection) + conds := make([]expression.Expression, len(selection.Conditions)) + for i, cond := range selection.Conditions { + conds[i] = expression.PushDownNot(sctx, cond) + } + cols, lengths := expression.IndexInfo2PrefixCols(tbl.Columns, selection.Schema().Columns, tbl.Indices[tt.indexPos]) + require.NotNil(t, cols) + res, err := ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, 0) + require.NoError(t, err) + require.Equal(t, tt.accessConds, fmt.Sprintf("%s", res.AccessConds)) + require.Equal(t, tt.filterConds, fmt.Sprintf("%s", res.RemainedConds)) + got := fmt.Sprintf("%v", res.Ranges) + require.Equal(t, tt.resultStr, got) + }) + } +} + +func TestTableShardIndex(t *testing.T) { + store := testkit.CreateMockStore(t) + testKit := testkit.NewTestKit(t, store) + config.UpdateGlobal(func(conf *config.Config) { + conf.Experimental.AllowsExpressionIndex = true + }) + testKit.MustExec("use test") + testKit.MustExec("drop table if exists test3") + testKit.MustExec("create table test3(id int primary key clustered, a int, b int, unique key uk_expr((tidb_shard(a)),a))") + testKit.MustExec("create table test33(id int primary key clustered, a int, b int, unique key a(a))") + testKit.MustExec("create table test4(id int primary key clustered, a int, b int, " + + "unique key uk_expr((tidb_shard(a)),a),unique key uk_b_expr((tidb_shard(b)),b))") + testKit.MustExec("create table test5(id int primary key clustered, a int, b int, " + + "unique key uk_expr((tidb_shard(a)),a,b))") + testKit.MustExec("create table test6(id int primary key clustered, a int, b int, c int, " + + "unique key uk_expr((tidb_shard(a)), a))") + testKit.MustExec("create table testx(id int primary key clustered, a int, b int, unique key a(a))") + testKit.MustExec("create table testy(id int primary key clustered, a int, b int, " + + "unique key uk_expr((tidb_shard(b)),a))") + testKit.MustExec("create table testz(id int primary key clustered, a int, b int, " + + "unique key uk_expr((tidb_shard(a+b)),a))") + + tests := []struct { + exprStr string + accessConds string + childLevel int + tableName string + }{ + { + exprStr: "a = 1", + accessConds: "[eq(tidb_shard(test.test3.a), 214) eq(test.test3.a, 1)]", + tableName: "test3", + }, + { + exprStr: "a=100 and (b = 100 or b = 200)", + accessConds: "[or(eq(test.test3.b, 100), eq(test.test3.b, 200)) eq(tidb_shard(test.test3.a), 8) " + + "eq(test.test3.a, 100)]", + tableName: "test3", + }, + { + // don't add prefix + exprStr: " tidb_shard(a) = 8", + accessConds: "[eq(tidb_shard(test.test3.a), 8)]", + tableName: "test3", + }, + { + exprStr: "a=100 or b = 200", + accessConds: "[or(and(eq(tidb_shard(test.test3.a), 8), eq(test.test3.a, 100)), " + + "eq(test.test3.b, 200))]", + tableName: "test3", + }, + { + exprStr: "a=100 or b > 200", + accessConds: "[or(and(eq(tidb_shard(test.test3.a), 8), eq(test.test3.a, 100)), " + + "gt(test.test3.b, 200))]", + tableName: "test3", + }, + { + exprStr: "a=100 or a = 200 or 1", + accessConds: "[or(and(eq(tidb_shard(test.test3.a), 8), eq(test.test3.a, 100)), " + + "or(and(eq(tidb_shard(test.test3.a), 161), eq(test.test3.a, 200)), 1))]", + tableName: "test3", + }, + { + exprStr: "(a=100 and b = 100) or a = 300", + accessConds: "[or(and(eq(test.test3.b, 100), and(eq(tidb_shard(test.test3.a), 8), eq(test.test3.a, 100))), " + + "and(eq(tidb_shard(test.test3.a), 227), eq(test.test3.a, 300)))]", + tableName: "test3", + }, + { + exprStr: "((a=100 and b = 100) or a = 200) or a = 300", + accessConds: "[or(and(eq(test.test3.b, 100), and(eq(tidb_shard(test.test3.a), 8), eq(test.test3.a, 100))), " + + "or(and(eq(tidb_shard(test.test3.a), 161), eq(test.test3.a, 200)), " + + "and(eq(tidb_shard(test.test3.a), 227), eq(test.test3.a, 300))))]", + tableName: "test3", + }, + { + exprStr: "a IN (100, 200, 300)", + accessConds: "[or(or(and(eq(tidb_shard(test.test3.a), 8), eq(test.test3.a, 100)), " + + "and(eq(tidb_shard(test.test3.a), 161), eq(test.test3.a, 200))), and(eq(tidb_shard(test.test3.a), 227), eq(test.test3.a, 300)))]", + tableName: "test3", + }, + { + exprStr: "a IN (100)", + accessConds: "[eq(tidb_shard(test.test3.a), 8) eq(test.test3.a, 100)]", + tableName: "test3", + }, + { + exprStr: "a IN (100, 200, 300) or a = 400", + accessConds: "[or(or(or(and(eq(tidb_shard(test.test3.a), 8), eq(test.test3.a, 100)), " + + "and(eq(tidb_shard(test.test3.a), 161), eq(test.test3.a, 200))), and(eq(tidb_shard(test.test3.a), 227), eq(test.test3.a, 300))), and(eq(tidb_shard(test.test3.a), 85), eq(test.test3.a, 400)))]", + tableName: "test3", + }, + { + // don't add prefix + exprStr: "((a=100 and b = 100) or a = 200) and b = 300", + accessConds: "[or(and(eq(test.test3.a, 100), eq(test.test3.b, 100)), eq(test.test3.a, 200)) " + + "eq(test.test3.b, 300)]", + tableName: "test3", + }, + { + // don't add prefix + exprStr: "a = b", + accessConds: "[eq(test.test3.a, test.test3.b)]", + tableName: "test3", + }, + { + // don't add prefix + exprStr: "a = b and b = 100", + accessConds: "[eq(test.test3.a, test.test3.b) eq(test.test3.b, 100)]", + tableName: "test3", + }, + { + // don't add prefix + exprStr: "a > 900", + accessConds: "[gt(test.test3.a, 900)]", + tableName: "test3", + }, + { + // add prefix + exprStr: "a = 3 or a > 900", + accessConds: "[or(and(eq(tidb_shard(test.test3.a), 156), eq(test.test3.a, 3)), gt(test.test3.a, 900))]", + tableName: "test3", + }, + // two shard index in one table + { + exprStr: "a = 100", + accessConds: "[eq(tidb_shard(test.test4.a), 8) eq(test.test4.a, 100)]", + tableName: "test4", + }, + { + exprStr: "b = 100", + accessConds: "[eq(tidb_shard(test.test4.b), 8) eq(test.test4.b, 100)]", + tableName: "test4", + }, + { + exprStr: "a = 100 and b = 100", + accessConds: "[eq(tidb_shard(test.test4.a), 8) eq(test.test4.a, 100) " + + "eq(tidb_shard(test.test4.b), 8) eq(test.test4.b, 100)]", + tableName: "test4", + }, + { + exprStr: "a = 100 or b = 100", + accessConds: "[or(and(eq(tidb_shard(test.test4.a), 8), eq(test.test4.a, 100)), " + + "and(eq(tidb_shard(test.test4.b), 8), eq(test.test4.b, 100)))]", + tableName: "test4", + }, + // shard index cotans three fields + { + exprStr: "a = 100 and b = 100", + accessConds: "[eq(tidb_shard(test.test5.a), 8) eq(test.test5.a, 100) eq(test.test5.b, 100)]", + tableName: "test5", + }, + { + exprStr: "(a=100 and b = 100) or (a=200 and b = 200)", + accessConds: "[or(and(eq(tidb_shard(test.test5.a), 8), and(eq(test.test5.a, 100), eq(test.test5.b, 100))), " + + "and(eq(tidb_shard(test.test5.a), 161), and(eq(test.test5.a, 200), eq(test.test5.b, 200))))]", + tableName: "test5", + }, + { + exprStr: "(a, b) in ((100, 100), (200, 200))", + accessConds: "[or(and(eq(tidb_shard(test.test5.a), 8), and(eq(test.test5.a, 100), eq(test.test5.b, 100))), " + + "and(eq(tidb_shard(test.test5.a), 161), and(eq(test.test5.a, 200), eq(test.test5.b, 200))))]", + tableName: "test5", + }, + { + exprStr: "(a, b) in ((100, 100))", + accessConds: "[eq(tidb_shard(test.test5.a), 8) eq(test.test5.a, 100) eq(test.test5.b, 100)]", + tableName: "test5", + }, + // don't add prefix + { + exprStr: "a=100", + accessConds: "[eq(test.testy.a, 100)]", + tableName: "testy", + }, + // don't add prefix + { + exprStr: "a=100", + accessConds: "[eq(test.testz.a, 100)]", + tableName: "testz", + }, + // test join + { + exprStr: "test3.a = 100", + accessConds: "[eq(tidb_shard(test.test3.a), 8) eq(test.test3.a, 100)]", + childLevel: 4, + tableName: "test3 JOIN test33 ON test3.b = test33.b", + }, + { + exprStr: "test3.a = 100 and test33.a > 10", + accessConds: "[gt(test.test33.a, 10) eq(tidb_shard(test.test3.a), 8) eq(test.test3.a, 100)]", + childLevel: 4, + tableName: "test3 JOIN test33 ON test3.b = test33.b", + }, + { + exprStr: "test3.a = 100 AND test6.a = 10", + accessConds: "[eq(test.test6.a, 10) eq(tidb_shard(test.test3.a), 8) eq(test.test3.a, 100)]", + childLevel: 4, + tableName: "test3 JOIN test6 ON test3.b = test6.b", + }, + { + exprStr: "test3.a = 100 or test6.a = 10", + accessConds: "[or(and(eq(tidb_shard(test.test3.a), 8), eq(test.test3.a, 100)), eq(test.test6.a, 10))]", + childLevel: 4, + tableName: "test3 JOIN test6 ON test3.b = test6.b", + }, + } + + ctx := context.Background() + for _, tt := range tests { + t.Run(tt.exprStr, func(t *testing.T) { + sql := "select * from " + tt.tableName + " where " + tt.exprStr + sctx := testKit.Session().(sessionctx.Context) + stmts, err := session.Parse(sctx, sql) + require.NoError(t, err) + require.Len(t, stmts, 1) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(context.Background(), sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) + require.NoError(t, err) + p, _, err := plannercore.BuildLogicalPlanForTest(ctx, sctx, stmts[0], ret.InfoSchema) + require.NoError(t, err) + selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) + conds := make([]expression.Expression, len(selection.Conditions)) + for i, cond := range selection.Conditions { + conds[i] = expression.PushDownNot(sctx, cond) + } + ds, ok := selection.Children()[0].(*plannercore.DataSource) + if !ok { + if tt.childLevel == 4 { + ds = selection.Children()[0].Children()[0].Children()[0].(*plannercore.DataSource) + } + } + newConds := ds.AddPrefix4ShardIndexes(ds.SCtx(), conds) + require.Equal(t, tt.accessConds, fmt.Sprintf("%s", newConds)) + }) + } + + // test update statement + t.Run("", func(t *testing.T) { + sql := "update test6 set c = 1000 where a=50 and b = 50" + sctx := testKit.Session().(sessionctx.Context) + stmts, err := session.Parse(sctx, sql) + require.NoError(t, err) + require.Len(t, stmts, 1) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(context.Background(), sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) + require.NoError(t, err) + p, _, err := plannercore.BuildLogicalPlanForTest(ctx, sctx, stmts[0], ret.InfoSchema) + require.NoError(t, err) + selection, ok := p.(*plannercore.Update).SelectPlan.(*plannercore.PhysicalSelection) + require.True(t, ok) + _, ok = selection.Children()[0].(*plannercore.PointGetPlan) + require.True(t, ok) + }) + + // test delete statement + t.Run("", func(t *testing.T) { + sql := "delete from test6 where a = 45 and b = 45;" + sctx := testKit.Session().(sessionctx.Context) + stmts, err := session.Parse(sctx, sql) + require.NoError(t, err) + require.Len(t, stmts, 1) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(context.Background(), sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) + require.NoError(t, err) + p, _, err := plannercore.BuildLogicalPlanForTest(ctx, sctx, stmts[0], ret.InfoSchema) + require.NoError(t, err) + selection, ok := p.(*plannercore.Delete).SelectPlan.(*plannercore.PhysicalSelection) + require.True(t, ok) + _, ok = selection.Children()[0].(*plannercore.PointGetPlan) + require.True(t, ok) + }) +} + +func TestShardIndexFuncSuites(t *testing.T) { + store := testkit.CreateMockStore(t) + testKit := testkit.NewTestKit(t, store) + sctx := testKit.Session().(sessionctx.Context) + + // ------------------------------------------- + // test IsValidShardIndex function + // ------------------------------------------- + longlongType := types.NewFieldType(mysql.TypeLonglong) + col0 := &expression.Column{UniqueID: 0, ID: 0, RetType: longlongType} + col1 := &expression.Column{UniqueID: 1, ID: 1, RetType: longlongType} + // col2 is GC column and VirtualExpr = tidb_shard(col0) + col2 := &expression.Column{UniqueID: 2, ID: 2, RetType: longlongType} + col2.VirtualExpr = expression.NewFunctionInternal(sctx, ast.TiDBShard, col2.RetType, col0) + // col3 is GC column and VirtualExpr = abs(col0) + col3 := &expression.Column{UniqueID: 3, ID: 3, RetType: longlongType} + col3.VirtualExpr = expression.NewFunctionInternal(sctx, ast.Abs, col2.RetType, col0) + col4 := &expression.Column{UniqueID: 4, ID: 4, RetType: longlongType} + + cols := []*expression.Column{col0, col1} + + // input is nil + require.False(t, ranger.IsValidShardIndex(nil)) + // only 1 column + require.False(t, ranger.IsValidShardIndex([]*expression.Column{col2})) + // first col is not expression + require.False(t, ranger.IsValidShardIndex(cols)) + // field in tidb_shard is not the secondary column + require.False(t, ranger.IsValidShardIndex([]*expression.Column{col2, col1})) + // expressioin is abs that is not tidb_shard + require.False(t, ranger.IsValidShardIndex([]*expression.Column{col3, col0})) + // normal case + require.True(t, ranger.IsValidShardIndex([]*expression.Column{col2, col0})) + + // ------------------------------------------- + // test ExtractColumnsFromExpr function + // ------------------------------------------- + // normal case + con1 := &expression.Constant{Value: types.NewDatum(1), RetType: longlongType} + con5 := &expression.Constant{Value: types.NewDatum(5), RetType: longlongType} + exprEq := expression.NewFunctionInternal(sctx, ast.EQ, col0.RetType, col0, con1) + exprIn := expression.NewFunctionInternal(sctx, ast.In, col0.RetType, col0, con1, con5) + require.NotNil(t, exprEq) + require.NotNil(t, exprIn) + // input is nil + require.Equal(t, len(ranger.ExtractColumnsFromExpr(nil)), 0) + // input is column + require.Equal(t, len(ranger.ExtractColumnsFromExpr(exprEq.(*expression.ScalarFunction))), 1) + // (col0 = 1 and col3 > 1) or (col4 < 5 and 5) + exprGt := expression.NewFunctionInternal(sctx, ast.GT, longlongType, col3, con1) + require.NotNil(t, exprGt) + andExpr1 := expression.NewFunctionInternal(sctx, ast.And, longlongType, exprEq, exprGt) + require.NotNil(t, andExpr1) + exprLt := expression.NewFunctionInternal(sctx, ast.LT, longlongType, col4, con5) + andExpr2 := expression.NewFunctionInternal(sctx, ast.And, longlongType, exprLt, con5) + orExpr2 := expression.NewFunctionInternal(sctx, ast.Or, longlongType, andExpr1, andExpr2) + require.Equal(t, len(ranger.ExtractColumnsFromExpr(orExpr2.(*expression.ScalarFunction))), 3) + + // ------------------------------------------- + // test NeedAddColumn4InCond function + // ------------------------------------------- + // normal case + sfIn, ok := exprIn.(*expression.ScalarFunction) + require.True(t, ok) + accessCond := []expression.Expression{nil, exprIn} + shardIndexCols := []*expression.Column{col2, col0} + require.True(t, ranger.NeedAddColumn4InCond(shardIndexCols, accessCond, sfIn)) + + // input nil + require.False(t, ranger.NeedAddColumn4InCond(nil, accessCond, sfIn)) + require.False(t, ranger.NeedAddColumn4InCond(shardIndexCols, nil, sfIn)) + require.False(t, ranger.NeedAddColumn4InCond(shardIndexCols, accessCond, nil)) + + // col1 in (1, 5) + exprIn2 := expression.NewFunctionInternal(sctx, ast.In, col1.RetType, col1, con1, con5) + accessCond[1] = exprIn2 + require.False(t, ranger.NeedAddColumn4InCond(shardIndexCols, accessCond, exprIn2.(*expression.ScalarFunction))) + + // col0 in (1, col1) + exprIn3 := expression.NewFunctionInternal(sctx, ast.In, col0.RetType, col1, con1, col1) + accessCond[1] = exprIn3 + require.False(t, ranger.NeedAddColumn4InCond(shardIndexCols, accessCond, exprIn3.(*expression.ScalarFunction))) + + // ------------------------------------------- + // test NeedAddColumn4EqCond function + // ------------------------------------------- + // ranger.valueInfo is not export by package, we can.t test NeedAddColumn4EqCond + eqAccessCond := []expression.Expression{nil, exprEq} + require.False(t, ranger.NeedAddColumn4EqCond(shardIndexCols, eqAccessCond, nil)) + + // ------------------------------------------- + // test NeedAddGcColumn4ShardIndex function + // ------------------------------------------- + // ranger.valueInfo is not export by package, we can.t test NeedAddGcColumn4ShardIndex + require.False(t, ranger.NeedAddGcColumn4ShardIndex(shardIndexCols, nil, nil)) + + // ------------------------------------------- + // test AddExpr4EqAndInCondition function + // ------------------------------------------- + exprIn4 := expression.NewFunctionInternal(sctx, ast.In, col0.RetType, col0, con1) + test := []struct { + inputConds []expression.Expression + outputConds string + }{ + { + // col0 = 1 => tidb_shard(col0) = 214 and col0 = 1 + inputConds: []expression.Expression{exprEq}, + outputConds: "[eq(Column#2, 214) eq(Column#0, 1)]", + }, + { + // col0 in (1) => cols2 = 214 and col0 = 1 + inputConds: []expression.Expression{exprIn4}, + outputConds: "[and(eq(Column#2, 214), eq(Column#0, 1))]", + }, + { + // col0 in (1, 5) => (cols2 = 214 and col0 = 1) or (cols2 = 122 and col0 = 5) + inputConds: []expression.Expression{exprIn}, + outputConds: "[or(and(eq(Column#2, 214), eq(Column#0, 1)), " + + "and(eq(Column#2, 122), eq(Column#0, 5)))]", + }, + } + + for _, tt := range test { + newConds, _ := ranger.AddExpr4EqAndInCondition(sctx, tt.inputConds, shardIndexCols) + require.Equal(t, fmt.Sprintf("%s", newConds), tt.outputConds) + } +} + +func getSelectionFromQuery(t *testing.T, sctx sessionctx.Context, sql string) *plannercore.LogicalSelection { + ctx := context.Background() + stmts, err := session.Parse(sctx, sql) + require.NoError(t, err) + require.Len(t, stmts, 1) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(context.Background(), sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) + require.NoError(t, err) + p, _, err := plannercore.BuildLogicalPlanForTest(ctx, sctx, stmts[0], ret.InfoSchema) + require.NoError(t, err) + selection, isSelection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) + require.True(t, isSelection) + return selection +} + +func checkDetachRangeResult(t *testing.T, res *ranger.DetachRangeResult, expectedAccessConds, expectedRemainedConds, expectedRanges string) { + require.Equal(t, expectedAccessConds, fmt.Sprintf("%v", res.AccessConds)) + require.Equal(t, expectedRemainedConds, fmt.Sprintf("%v", res.RemainedConds)) + require.Equal(t, expectedRanges, fmt.Sprintf("%v", res.Ranges)) +} + +func checkRangeFallbackAndReset(t *testing.T, sctx sessionctx.Context, expectedRangeFallback bool) { + require.Equal(t, expectedRangeFallback, sctx.GetSessionVars().StmtCtx.RangeFallback) + sctx.GetSessionVars().StmtCtx.RangeFallback = false +} + +func TestRangeFallbackForDetachCondAndBuildRangeForIndex(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t1") + tk.MustExec("create table t1 (a int, b int, c int, d int, index idx(a, b, c))") + tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t1")) + require.NoError(t, err) + tblInfo := tbl.Meta() + sctx := tk.Session().(sessionctx.Context) + + // test CNF condition + sql := "select * from t1 where a in (10,20,30) and b in (40,50,60) and c >= 70 and c <= 80" + selection := getSelectionFromQuery(t, sctx, sql) + conds := selection.Conditions + require.Equal(t, 4, len(conds)) + cols, lengths := expression.IndexInfo2PrefixCols(tblInfo.Columns, selection.Schema().Columns, tblInfo.Indices[0]) + res, err := ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, 0) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[in(test.t1.a, 10, 20, 30) in(test.t1.b, 40, 50, 60) ge(test.t1.c, 70) le(test.t1.c, 80)]", + "[]", + "[[10 40 70,10 40 80] [10 50 70,10 50 80] [10 60 70,10 60 80] [20 40 70,20 40 80] [20 50 70,20 50 80] [20 60 70,20 60 80] [30 40 70,30 40 80] [30 50 70,30 50 80] [30 60 70,30 60 80]]") + checkRangeFallbackAndReset(t, sctx, false) + quota := res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[in(test.t1.a, 10, 20, 30) in(test.t1.b, 40, 50, 60)]", + "[ge(test.t1.c, 70) le(test.t1.c, 80)]", + "[[10 40,10 40] [10 50,10 50] [10 60,10 60] [20 40,20 40] [20 50,20 50] [20 60,20 60] [30 40,30 40] [30 50,30 50] [30 60,30 60]]") + checkRangeFallbackAndReset(t, sctx, true) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[in(test.t1.a, 10, 20, 30)]", + "[in(test.t1.b, 40, 50, 60) ge(test.t1.c, 70) le(test.t1.c, 80)]", + "[[10,10] [20,20] [30,30]]") + checkRangeFallbackAndReset(t, sctx, true) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[]", + "[ge(test.t1.c, 70) le(test.t1.c, 80) in(test.t1.b, 40, 50, 60) in(test.t1.a, 10, 20, 30)]", + "[[NULL,+inf]]") + checkRangeFallbackAndReset(t, sctx, true) + + // test DNF condition + sql = "select * from t1 where a = 10 or a = 20 or a = 30" + selection = getSelectionFromQuery(t, sctx, sql) + conds = selection.Conditions + require.Equal(t, 1, len(conds)) + cols, lengths = expression.IndexInfo2PrefixCols(tblInfo.Columns, selection.Schema().Columns, tblInfo.Indices[0]) + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, 0) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[or(eq(test.t1.a, 10), or(eq(test.t1.a, 20), eq(test.t1.a, 30)))]", + "[]", + "[[10,10] [20,20] [30,30]]") + checkRangeFallbackAndReset(t, sctx, false) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[]", + "[or(or(eq(test.t1.a, 10), eq(test.t1.a, 20)), eq(test.t1.a, 30))]", + "[[NULL,+inf]]") + checkRangeFallbackAndReset(t, sctx, true) + + sql = "select * from t1 where (a = 10 and b = 40) or (a = 20 and b = 50) or (a = 30 and b = 60)" + selection = getSelectionFromQuery(t, sctx, sql) + conds = selection.Conditions + require.Equal(t, 1, len(conds)) + cols, lengths = expression.IndexInfo2PrefixCols(tblInfo.Columns, selection.Schema().Columns, tblInfo.Indices[0]) + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, 0) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[or(and(eq(test.t1.a, 10), eq(test.t1.b, 40)), or(and(eq(test.t1.a, 20), eq(test.t1.b, 50)), and(eq(test.t1.a, 30), eq(test.t1.b, 60))))]", + "[]", + "[[10 40,10 40] [20 50,20 50] [30 60,30 60]]") + checkRangeFallbackAndReset(t, sctx, false) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[]", + "[or(or(and(eq(test.t1.a, 10), eq(test.t1.b, 40)), and(eq(test.t1.a, 20), eq(test.t1.b, 50))), and(eq(test.t1.a, 30), eq(test.t1.b, 60)))]", + "[[NULL,+inf]]") + checkRangeFallbackAndReset(t, sctx, true) + + // test considerDNF code path + sql = "select * from t1 where (a, b) in ((10, 20), (30, 40)) and c = 50" + selection = getSelectionFromQuery(t, sctx, sql) + conds = selection.Conditions + require.Equal(t, 2, len(conds)) + cols, lengths = expression.IndexInfo2PrefixCols(tblInfo.Columns, selection.Schema().Columns, tblInfo.Indices[0]) + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, 0) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[or(and(eq(test.t1.a, 10), eq(test.t1.b, 20)), and(eq(test.t1.a, 30), eq(test.t1.b, 40))) eq(test.t1.c, 50)]", + "[]", + "[[10 20 50,10 20 50] [30 40 50,30 40 50]]") + checkRangeFallbackAndReset(t, sctx, false) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[or(and(eq(test.t1.a, 10), eq(test.t1.b, 20)), and(eq(test.t1.a, 30), eq(test.t1.b, 40)))]", + "[eq(test.t1.c, 50)]", + "[[10 20,10 20] [30 40,30 40]]") + checkRangeFallbackAndReset(t, sctx, true) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[or(eq(test.t1.a, 10), eq(test.t1.a, 30))]", + "[eq(test.t1.c, 50) or(and(eq(test.t1.a, 10), eq(test.t1.b, 20)), and(eq(test.t1.a, 30), eq(test.t1.b, 40)))]", + "[[10,10] [30,30]]") + checkRangeFallbackAndReset(t, sctx, true) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[]", + // Ideal RemainedConds should be [eq(test.t1.c, 50) or(and(eq(test.t1.a, 10), eq(test.t1.b, 20)), and(eq(test.t1.a, 30), eq(test.t1.b, 40)))], but we don't remove redundant or(eq(test.t1.a, 10), eq(test.t1.a, 30)) for now. + "[eq(test.t1.c, 50) or(and(eq(test.t1.a, 10), eq(test.t1.b, 20)), and(eq(test.t1.a, 30), eq(test.t1.b, 40))) or(eq(test.t1.a, 10), eq(test.t1.a, 30))]", + "[[NULL,+inf]]") + checkRangeFallbackAndReset(t, sctx, true) + + // test prefix index + tk.MustExec("drop table if exists t2") + tk.MustExec("create table t2 (a varchar(10), b varchar(10), c varchar(10), d varchar(10), index idx(a(2), b(2), c(2)))") + tbl, err = dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t2")) + require.NoError(t, err) + tblInfo = tbl.Meta() + + // test CNF condition + sql = "select * from t2 where a in ('aaa','bbb','ccc') and b in ('ddd','eee','fff') and c >= 'ggg' and c <= 'iii'" + selection = getSelectionFromQuery(t, sctx, sql) + conds = selection.Conditions + require.Equal(t, 4, len(conds)) + cols, lengths = expression.IndexInfo2PrefixCols(tblInfo.Columns, selection.Schema().Columns, tblInfo.Indices[0]) + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, 0) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[in(test.t2.a, aaa, bbb, ccc) in(test.t2.b, ddd, eee, fff) ge(test.t2.c, ggg) le(test.t2.c, iii)]", + "[in(test.t2.a, aaa, bbb, ccc) in(test.t2.b, ddd, eee, fff) ge(test.t2.c, ggg) le(test.t2.c, iii)]", + "[[\"aa\" \"dd\" \"gg\",\"aa\" \"dd\" \"ii\"] [\"aa\" \"ee\" \"gg\",\"aa\" \"ee\" \"ii\"] [\"aa\" \"ff\" \"gg\",\"aa\" \"ff\" \"ii\"] [\"bb\" \"dd\" \"gg\",\"bb\" \"dd\" \"ii\"] [\"bb\" \"ee\" \"gg\",\"bb\" \"ee\" \"ii\"] [\"bb\" \"ff\" \"gg\",\"bb\" \"ff\" \"ii\"] [\"cc\" \"dd\" \"gg\",\"cc\" \"dd\" \"ii\"] [\"cc\" \"ee\" \"gg\",\"cc\" \"ee\" \"ii\"] [\"cc\" \"ff\" \"gg\",\"cc\" \"ff\" \"ii\"]]") + checkRangeFallbackAndReset(t, sctx, false) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[in(test.t2.a, aaa, bbb, ccc) in(test.t2.b, ddd, eee, fff)]", + "[in(test.t2.a, aaa, bbb, ccc) in(test.t2.b, ddd, eee, fff) ge(test.t2.c, ggg) le(test.t2.c, iii)]", + "[[\"aa\" \"dd\",\"aa\" \"dd\"] [\"aa\" \"ee\",\"aa\" \"ee\"] [\"aa\" \"ff\",\"aa\" \"ff\"] [\"bb\" \"dd\",\"bb\" \"dd\"] [\"bb\" \"ee\",\"bb\" \"ee\"] [\"bb\" \"ff\",\"bb\" \"ff\"] [\"cc\" \"dd\",\"cc\" \"dd\"] [\"cc\" \"ee\",\"cc\" \"ee\"] [\"cc\" \"ff\",\"cc\" \"ff\"]]") + checkRangeFallbackAndReset(t, sctx, true) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[in(test.t2.a, aaa, bbb, ccc)]", + "[in(test.t2.a, aaa, bbb, ccc) in(test.t2.b, ddd, eee, fff) ge(test.t2.c, ggg) le(test.t2.c, iii)]", + "[[\"aa\",\"aa\"] [\"bb\",\"bb\"] [\"cc\",\"cc\"]]") + checkRangeFallbackAndReset(t, sctx, true) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[]", + "[ge(test.t2.c, ggg) le(test.t2.c, iii) in(test.t2.a, aaa, bbb, ccc) in(test.t2.b, ddd, eee, fff)]", + "[[NULL,+inf]]") + checkRangeFallbackAndReset(t, sctx, true) + + // test DNF condition + sql = "select * from t2 where a = 'aaa' or a = 'bbb' or a = 'ccc'" + selection = getSelectionFromQuery(t, sctx, sql) + conds = selection.Conditions + require.Equal(t, 1, len(conds)) + cols, lengths = expression.IndexInfo2PrefixCols(tblInfo.Columns, selection.Schema().Columns, tblInfo.Indices[0]) + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, 0) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[or(eq(test.t2.a, aaa), or(eq(test.t2.a, bbb), eq(test.t2.a, ccc)))]", + "[or(or(eq(test.t2.a, aaa), eq(test.t2.a, bbb)), eq(test.t2.a, ccc))]", + "[[\"aa\",\"aa\"] [\"bb\",\"bb\"] [\"cc\",\"cc\"]]") + checkRangeFallbackAndReset(t, sctx, false) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[]", + "[or(or(eq(test.t2.a, aaa), eq(test.t2.a, bbb)), eq(test.t2.a, ccc))]", + "[[NULL,+inf]]") + checkRangeFallbackAndReset(t, sctx, true) + + sql = "select * from t2 where (a = 'aaa' and b = 'ddd') or (a = 'bbb' and b = 'eee') or (a = 'ccc' and b = 'fff')" + selection = getSelectionFromQuery(t, sctx, sql) + conds = selection.Conditions + require.Equal(t, 1, len(conds)) + cols, lengths = expression.IndexInfo2PrefixCols(tblInfo.Columns, selection.Schema().Columns, tblInfo.Indices[0]) + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, 0) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[or(and(eq(test.t2.a, aaa), eq(test.t2.b, ddd)), or(and(eq(test.t2.a, bbb), eq(test.t2.b, eee)), and(eq(test.t2.a, ccc), eq(test.t2.b, fff))))]", + "[or(or(and(eq(test.t2.a, aaa), eq(test.t2.b, ddd)), and(eq(test.t2.a, bbb), eq(test.t2.b, eee))), and(eq(test.t2.a, ccc), eq(test.t2.b, fff)))]", + "[[\"aa\" \"dd\",\"aa\" \"dd\"] [\"bb\" \"ee\",\"bb\" \"ee\"] [\"cc\" \"ff\",\"cc\" \"ff\"]]") + checkRangeFallbackAndReset(t, sctx, false) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[]", + "[or(or(and(eq(test.t2.a, aaa), eq(test.t2.b, ddd)), and(eq(test.t2.a, bbb), eq(test.t2.b, eee))), and(eq(test.t2.a, ccc), eq(test.t2.b, fff)))]", + "[[NULL,+inf]]") + checkRangeFallbackAndReset(t, sctx, true) + + // test considerDNF code path + sql = "select * from t2 where (a, b) in (('aaa', 'bbb'), ('ccc', 'ddd')) and c = 'eee'" + selection = getSelectionFromQuery(t, sctx, sql) + conds = selection.Conditions + require.Equal(t, 2, len(conds)) + cols, lengths = expression.IndexInfo2PrefixCols(tblInfo.Columns, selection.Schema().Columns, tblInfo.Indices[0]) + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, 0) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[or(eq(test.t2.a, aaa), eq(test.t2.a, ccc))]", + "[eq(test.t2.c, eee) or(and(eq(test.t2.a, aaa), eq(test.t2.b, bbb)), and(eq(test.t2.a, ccc), eq(test.t2.b, ddd)))]", + "[[\"aa\",\"aa\"] [\"cc\",\"cc\"]]") + checkRangeFallbackAndReset(t, sctx, false) + quota = res.Ranges.MemUsage() - 1 + res, err = ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, quota) + require.NoError(t, err) + checkDetachRangeResult(t, res, + "[]", + "[eq(test.t2.c, eee) or(and(eq(test.t2.a, aaa), eq(test.t2.b, bbb)), and(eq(test.t2.a, ccc), eq(test.t2.b, ddd))) or(eq(test.t2.a, aaa), eq(test.t2.a, ccc))]", + "[[NULL,+inf]]") + checkRangeFallbackAndReset(t, sctx, true) +} + +func TestRangeFallbackForBuildTableRange(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a int primary key, b int)") + tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tblInfo := tbl.Meta() + sctx := tk.Session().(sessionctx.Context) + sql := "select * from t where a in (10,20,30,40,50)" + selection := getSelectionFromQuery(t, sctx, sql) + conds := selection.Conditions + require.Equal(t, 1, len(conds)) + col := expression.ColInfo2Col(selection.Schema().Columns, tblInfo.Columns[0]) + var filters []expression.Expression + conds, filters = ranger.DetachCondsForColumn(sctx, conds, col) + require.Equal(t, 1, len(conds)) + require.Equal(t, 0, len(filters)) + ranges, access, remained, err := ranger.BuildTableRange(conds, sctx, col.RetType, 0) + require.NoError(t, err) + require.Equal(t, "[[10,10] [20,20] [30,30] [40,40] [50,50]]", fmt.Sprintf("%v", ranges)) + require.Equal(t, "[in(test.t.a, 10, 20, 30, 40, 50)]", fmt.Sprintf("%v", access)) + require.Equal(t, "[]", fmt.Sprintf("%v", remained)) + checkRangeFallbackAndReset(t, sctx, false) + quota := ranges.MemUsage() - 1 + ranges, access, remained, err = ranger.BuildTableRange(conds, sctx, col.RetType, quota) + require.NoError(t, err) + require.Equal(t, "[[-inf,+inf]]", fmt.Sprintf("%v", ranges)) + require.Equal(t, "[]", fmt.Sprintf("%v", access)) + require.Equal(t, "[in(test.t.a, 10, 20, 30, 40, 50)]", fmt.Sprintf("%v", remained)) + checkRangeFallbackAndReset(t, sctx, true) +} + +func TestRangeFallbackForBuildColumnRange(t *testing.T) { + store, dom := testkit.CreateMockStoreAndDomain(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t (a varchar(20), b int not null)") + tbl, err := dom.InfoSchema().TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + tblInfo := tbl.Meta() + sctx := tk.Session().(sessionctx.Context) + sql := "select * from t where a in ('aaa','bbb','ccc','ddd','eee')" + selection := getSelectionFromQuery(t, sctx, sql) + conds := selection.Conditions + require.Equal(t, 1, len(conds)) + cola := expression.ColInfo2Col(selection.Schema().Columns, tblInfo.Columns[0]) + var filters []expression.Expression + conds, filters = ranger.DetachCondsForColumn(sctx, conds, cola) + require.Equal(t, 1, len(conds)) + require.Equal(t, 0, len(filters)) + ranges, access, remained, err := ranger.BuildColumnRange(conds, sctx, cola.RetType, types.UnspecifiedLength, 0) + require.NoError(t, err) + require.Equal(t, "[[\"aaa\",\"aaa\"] [\"bbb\",\"bbb\"] [\"ccc\",\"ccc\"] [\"ddd\",\"ddd\"] [\"eee\",\"eee\"]]", fmt.Sprintf("%v", ranges)) + require.Equal(t, "[in(test.t.a, aaa, bbb, ccc, ddd, eee)]", fmt.Sprintf("%v", access)) + require.Equal(t, "[]", fmt.Sprintf("%v", remained)) + checkRangeFallbackAndReset(t, sctx, false) + quota := ranges.MemUsage() - 1 + ranges, access, remained, err = ranger.BuildColumnRange(conds, sctx, cola.RetType, types.UnspecifiedLength, quota) + require.NoError(t, err) + require.Equal(t, "[[NULL,+inf]]", fmt.Sprintf("%v", ranges)) + require.Equal(t, "[]", fmt.Sprintf("%v", access)) + require.Equal(t, "[in(test.t.a, aaa, bbb, ccc, ddd, eee)]", fmt.Sprintf("%v", remained)) + checkRangeFallbackAndReset(t, sctx, true) + sql = "select * from t where b in (10,20,30)" + selection = getSelectionFromQuery(t, sctx, sql) + conds = selection.Conditions + require.Equal(t, 1, len(conds)) + colb := expression.ColInfo2Col(selection.Schema().Columns, tblInfo.Columns[1]) + conds, filters = ranger.DetachCondsForColumn(sctx, conds, colb) + require.Equal(t, 1, len(conds)) + require.Equal(t, 0, len(filters)) + ranges, access, remained, err = ranger.BuildColumnRange(conds, sctx, colb.RetType, types.UnspecifiedLength, 1) + require.NoError(t, err) + require.Equal(t, "[[-inf,+inf]]", fmt.Sprintf("%v", ranges)) + require.Equal(t, "[]", fmt.Sprintf("%v", access)) + require.Equal(t, "[in(test.t.b, 10, 20, 30)]", fmt.Sprintf("%v", remained)) +} + +func TestPrefixIndexRange(t *testing.T) { + store := testkit.CreateMockStore(t) + tk := testkit.NewTestKit(t, store) + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec(` +create table t( + a varchar(50), + b varchar(50), + c text(50), + d varbinary(50), + index idx_a(a(2)), + index idx_ab(a(2), b(2)), + index idx_c(c(2)), + index idx_d(d(2)) +)`) + tk.MustExec("set tidb_opt_prefix_index_single_scan = 1") + + tests := []struct { + indexPos int + exprStr string + accessConds string + filterConds string + resultStr string + }{ + { + indexPos: 0, + exprStr: "a is null", + accessConds: "[isnull(test.t.a)]", + filterConds: "[]", + resultStr: "[[NULL,NULL]]", + }, + { + indexPos: 0, + exprStr: "a is not null", + accessConds: "[not(isnull(test.t.a))]", + filterConds: "[]", + resultStr: "[[-inf,+inf]]", + }, + { + indexPos: 1, + exprStr: "a = 'a' and b is null", + accessConds: "[eq(test.t.a, a) isnull(test.t.b)]", + filterConds: "[eq(test.t.a, a)]", + resultStr: "[[\"a\" NULL,\"a\" NULL]]", + }, + { + indexPos: 1, + exprStr: "a = 'a' and b is not null", + accessConds: "[eq(test.t.a, a) not(isnull(test.t.b))]", + filterConds: "[eq(test.t.a, a)]", + resultStr: "[[\"a\" -inf,\"a\" +inf]]", + }, + { + indexPos: 2, + exprStr: "c is null", + accessConds: "[isnull(test.t.c)]", + filterConds: "[]", + resultStr: "[[NULL,NULL]]", + }, + { + indexPos: 2, + exprStr: "c is not null", + accessConds: "[not(isnull(test.t.c))]", + filterConds: "[]", + resultStr: "[[-inf,+inf]]", + }, + { + indexPos: 3, + exprStr: "d is null", + accessConds: "[isnull(test.t.d)]", + filterConds: "[]", + resultStr: "[[NULL,NULL]]", + }, + { + indexPos: 3, + exprStr: "d is not null", + accessConds: "[not(isnull(test.t.d))]", + filterConds: "[]", + resultStr: "[[-inf,+inf]]", + }, + } + + collate.SetNewCollationEnabledForTest(true) + defer func() { collate.SetNewCollationEnabledForTest(false) }() + ctx := context.Background() + for _, tt := range tests { + sql := "select * from t where " + tt.exprStr + sctx := tk.Session() + stmts, err := session.Parse(sctx, sql) + require.NoError(t, err, fmt.Sprintf("error %v, for expr %s", err, tt.exprStr)) + require.Len(t, stmts, 1) + ret := &plannercore.PreprocessorReturn{} + err = plannercore.Preprocess(context.Background(), sctx, stmts[0], plannercore.WithPreprocessorReturn(ret)) + require.NoError(t, err, fmt.Sprintf("error %v, for resolve name, expr %s", err, tt.exprStr)) + p, _, err := plannercore.BuildLogicalPlanForTest(ctx, sctx, stmts[0], ret.InfoSchema) + require.NoError(t, err, fmt.Sprintf("error %v, for build plan, expr %s", err, tt.exprStr)) + selection := p.(plannercore.LogicalPlan).Children()[0].(*plannercore.LogicalSelection) + tbl := selection.Children()[0].(*plannercore.DataSource).TableInfo() + require.NotNil(t, selection, fmt.Sprintf("expr:%v", tt.exprStr)) + conds := make([]expression.Expression, len(selection.Conditions)) + for i, cond := range selection.Conditions { + conds[i] = expression.PushDownNot(sctx, cond) + } + cols, lengths := expression.IndexInfo2PrefixCols(tbl.Columns, selection.Schema().Columns, tbl.Indices[tt.indexPos]) + require.NotNil(t, cols) + res, err := ranger.DetachCondAndBuildRangeForIndex(sctx, conds, cols, lengths, 0) + require.NoError(t, err) + require.Equal(t, tt.accessConds, fmt.Sprintf("%s", res.AccessConds), fmt.Sprintf("wrong access conditions for expr: %s", tt.exprStr)) + require.Equal(t, tt.filterConds, fmt.Sprintf("%s", res.RemainedConds), fmt.Sprintf("wrong filter conditions for expr: %s", tt.exprStr)) + got := fmt.Sprintf("%v", res.Ranges) + require.Equal(t, tt.resultStr, got, fmt.Sprintf("different for expr %s", tt.exprStr)) + } +} + +func TestIssue44389(t *testing.T) { + store := testkit.CreateMockStore(t) + + testKit := testkit.NewTestKit(t, store) + testKit.MustExec("use test") + testKit.MustExec("drop table if exists t") + testKit.MustExec("create table t(a varchar(100), b int, c int, index idx_ab(a, b))") + testKit.MustExec("insert into t values ('kk', 1, 10), ('kk', 1, 20), ('hh', 2, 10), ('hh', 3, 10), ('xx', 4, 10), ('yy', 5, 10), ('yy', 6, 20), ('zz', 7, 10)") + testKit.MustExec("set @@tidb_opt_fix_control = '44389:ON'") + + var input []string + var output []struct { + SQL string + Plan []string + Result []string + } + rangerSuiteData.LoadTestCases(t, &input, &output) + for i, tt := range input { + testdata.OnRecord(func() { + output[i].SQL = tt + output[i].Plan = testdata.ConvertRowsToStrings(testKit.MustQuery("explain " + tt).Rows()) + output[i].Result = testdata.ConvertRowsToStrings(testKit.MustQuery(tt).Sort().Rows()) + }) + testKit.MustQuery("explain " + tt).Check(testkit.Rows(output[i].Plan...)) + testKit.MustQuery(tt).Sort().Check(testkit.Rows(output[i].Result...)) + } +} +>>>>>>> 85d6323e3a3 (util/ranger: consider good non-point ranges from CNF item (#44384)) diff --git a/util/ranger/testdata/ranger_suite_in.json b/util/ranger/testdata/ranger_suite_in.json index c1641c0f251f7..beb004ff1a521 100644 --- a/util/ranger/testdata/ranger_suite_in.json +++ b/util/ranger/testdata/ranger_suite_in.json @@ -109,5 +109,12 @@ "select * from IDT_20755 use index (u_m_col) where col1 = \"xxxxxxxxxxxxxxx\" and col2 in (72, 73) and col3 != \"2024-10-19 08:55:32\"", "select * from IDT_20755 use index (u_m_col) where col1 = \"xxxxxxxxxxxxxxx\" and col2 in (72, 73, 74) and col3 != \"2024-10-19 08:55:32\"" ] + }, + { + "name": "TestIssue44389", + "cases": [ + "select * from t where c = 10 and (a = 'xx' or (a = 'kk' and b = 1))", + "select * from t where c = 10 and ((a = 'xx' or a = 'yy') or ((a = 'kk' and b = 1) or (a = 'hh' and b = 2)))" + ] } ] diff --git a/util/ranger/testdata/ranger_suite_out.json b/util/ranger/testdata/ranger_suite_out.json index d6a9769cc97b8..5eaddaf678981 100644 --- a/util/ranger/testdata/ranger_suite_out.json +++ b/util/ranger/testdata/ranger_suite_out.json @@ -689,5 +689,38 @@ ] } ] + }, + { + "Name": "TestIssue44389", + "Cases": [ + { + "SQL": "select * from t where c = 10 and (a = 'xx' or (a = 'kk' and b = 1))", + "Plan": [ + "IndexLookUp_11 0.01 root ", + "├─IndexRangeScan_8(Build) 10.10 cop[tikv] table:t, index:idx_ab(a, b) range:[\"kk\" 1,\"kk\" 1], [\"xx\",\"xx\"], keep order:false, stats:pseudo", + "└─Selection_10(Probe) 0.01 cop[tikv] eq(test.t.c, 10)", + " └─TableRowIDScan_9 10.10 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Result": [ + "kk 1 10", + "xx 4 10" + ] + }, + { + "SQL": "select * from t where c = 10 and ((a = 'xx' or a = 'yy') or ((a = 'kk' and b = 1) or (a = 'hh' and b = 2)))", + "Plan": [ + "IndexLookUp_11 0.02 root ", + "├─IndexRangeScan_8(Build) 20.20 cop[tikv] table:t, index:idx_ab(a, b) range:[\"hh\" 2,\"hh\" 2], [\"kk\" 1,\"kk\" 1], [\"xx\",\"xx\"], [\"yy\",\"yy\"], keep order:false, stats:pseudo", + "└─Selection_10(Probe) 0.02 cop[tikv] eq(test.t.c, 10)", + " └─TableRowIDScan_9 20.20 cop[tikv] table:t keep order:false, stats:pseudo" + ], + "Result": [ + "hh 2 10", + "kk 1 10", + "xx 4 10", + "yy 5 10" + ] + } + ] } ]