Skip to content

Commit

Permalink
This is an automated cherry-pick of pingcap#54179
Browse files Browse the repository at this point in the history
Signed-off-by: ti-chi-bot <[email protected]>
  • Loading branch information
elsa0520 authored and ti-chi-bot committed Oct 30, 2024
1 parent 7ddeb88 commit 7aad36c
Show file tree
Hide file tree
Showing 51 changed files with 1,127 additions and 249,679 deletions.
38 changes: 38 additions & 0 deletions expression/constant_propagation.go
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
package expression

import (
<<<<<<< HEAD:expression/constant_propagation.go
"errors"

"github.com/pingcap/tidb/parser/ast"
Expand All @@ -26,6 +27,17 @@ import (
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/disjointset"
"github.com/pingcap/tidb/util/logutil"
=======
exprctx "github.com/pingcap/tidb/pkg/expression/context"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/types"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/collate"
"github.com/pingcap/tidb/pkg/util/disjointset"
"github.com/pingcap/tidb/pkg/util/logutil"
>>>>>>> 521c99967d5 (planner: The length function could not be substitute when collation of mapped column is utfxxx_bin (#54179)):pkg/expression/constant_propagation.go
"go.uber.org/zap"
)

Expand All @@ -38,7 +50,11 @@ type basePropConstSolver struct {
eqList []*Constant // if eqList[i] != nil, it means col_i = eqList[i]
unionSet *disjointset.IntSet // unionSet stores the relations like col_i = col_j
columns []*Column // columns stores all columns appearing in the conditions
<<<<<<< HEAD:expression/constant_propagation.go
ctx sessionctx.Context
=======
ctx exprctx.ExprContext
>>>>>>> 521c99967d5 (planner: The length function could not be substitute when collation of mapped column is utfxxx_bin (#54179)):pkg/expression/constant_propagation.go
}

func (s *basePropConstSolver) getColID(col *Column) int {
Expand Down Expand Up @@ -357,8 +373,14 @@ func (s *propConstSolver) solve(conditions []Expression) []Expression {
}

// PropagateConstant propagate constant values of deterministic predicates in a condition.
<<<<<<< HEAD:expression/constant_propagation.go
func PropagateConstant(ctx sessionctx.Context, conditions []Expression) []Expression {
return newPropConstSolver().PropagateConstant(ctx, conditions)
=======
// This is a constant propagation logic for expression list such as ['a=1', 'a=b']
func PropagateConstant(ctx exprctx.ExprContext, conditions []Expression) []Expression {
return newPropConstSolver().PropagateConstant(exprctx.WithConstantPropagateCheck(ctx), conditions)
>>>>>>> 521c99967d5 (planner: The length function could not be substitute when collation of mapped column is utfxxx_bin (#54179)):pkg/expression/constant_propagation.go
}

type propOuterJoinConstSolver struct {
Expand Down Expand Up @@ -662,7 +684,11 @@ func (s *propOuterJoinConstSolver) solve(joinConds, filterConds []Expression) ([
}

// propagateConstantDNF find DNF item from CNF, and propagate constant inside DNF.
<<<<<<< HEAD:expression/constant_propagation.go
func propagateConstantDNF(ctx sessionctx.Context, conds []Expression) []Expression {
=======
func propagateConstantDNF(ctx exprctx.ExprContext, conds []Expression) []Expression {
>>>>>>> 521c99967d5 (planner: The length function could not be substitute when collation of mapped column is utfxxx_bin (#54179)):pkg/expression/constant_propagation.go
for i, cond := range conds {
if dnf, ok := cond.(*ScalarFunction); ok && dnf.FuncName.L == ast.LogicOr {
dnfItems := SplitDNFItems(cond)
Expand All @@ -681,7 +707,11 @@ func propagateConstantDNF(ctx sessionctx.Context, conds []Expression) []Expressi
// Second step is to extract `outerCol = innerCol` from join conditions, and derive new join
// conditions based on this column equal condition and `outerCol` related
// expressions in join conditions and filter conditions;
<<<<<<< HEAD:expression/constant_propagation.go
func PropConstOverOuterJoin(ctx sessionctx.Context, joinConds, filterConds []Expression,
=======
func PropConstOverOuterJoin(ctx exprctx.ExprContext, joinConds, filterConds []Expression,
>>>>>>> 521c99967d5 (planner: The length function could not be substitute when collation of mapped column is utfxxx_bin (#54179)):pkg/expression/constant_propagation.go
outerSchema, innerSchema *Schema, nullSensitive bool) ([]Expression, []Expression) {
solver := &propOuterJoinConstSolver{
outerSchema: outerSchema,
Expand All @@ -695,7 +725,11 @@ func PropConstOverOuterJoin(ctx sessionctx.Context, joinConds, filterConds []Exp

// PropagateConstantSolver is a constant propagate solver.
type PropagateConstantSolver interface {
<<<<<<< HEAD:expression/constant_propagation.go
PropagateConstant(ctx sessionctx.Context, conditions []Expression) []Expression
=======
PropagateConstant(ctx exprctx.ExprContext, conditions []Expression) []Expression
>>>>>>> 521c99967d5 (planner: The length function could not be substitute when collation of mapped column is utfxxx_bin (#54179)):pkg/expression/constant_propagation.go
}

// newPropConstSolver returns a PropagateConstantSolver.
Expand All @@ -706,7 +740,11 @@ func newPropConstSolver() PropagateConstantSolver {
}

// PropagateConstant propagate constant values of deterministic predicates in a condition.
<<<<<<< HEAD:expression/constant_propagation.go
func (s *propConstSolver) PropagateConstant(ctx sessionctx.Context, conditions []Expression) []Expression {
=======
func (s *propConstSolver) PropagateConstant(ctx exprctx.ExprContext, conditions []Expression) []Expression {
>>>>>>> 521c99967d5 (planner: The length function could not be substitute when collation of mapped column is utfxxx_bin (#54179)):pkg/expression/constant_propagation.go
s.ctx = ctx
return s.solve(conditions)
}
39 changes: 39 additions & 0 deletions expression/util.go
Original file line number Diff line number Diff line change
Expand Up @@ -25,6 +25,7 @@ import (

"github.com/pingcap/errors"
"github.com/pingcap/failpoint"
<<<<<<< HEAD:expression/util.go
"github.com/pingcap/tidb/kv"
"github.com/pingcap/tidb/parser/ast"
"github.com/pingcap/tidb/parser/mysql"
Expand All @@ -37,6 +38,22 @@ import (
"github.com/pingcap/tidb/util/collate"
"github.com/pingcap/tidb/util/logutil"
"github.com/pingcap/tidb/util/sqlexec"
=======
"github.com/pingcap/tidb/pkg/expression/contextopt"
"github.com/pingcap/tidb/pkg/kv"
"github.com/pingcap/tidb/pkg/parser/ast"
"github.com/pingcap/tidb/pkg/parser/charset"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/parser/opcode"
"github.com/pingcap/tidb/pkg/parser/terror"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/types"
driver "github.com/pingcap/tidb/pkg/types/parser_driver"
"github.com/pingcap/tidb/pkg/util/chunk"
"github.com/pingcap/tidb/pkg/util/collate"
"github.com/pingcap/tidb/pkg/util/intset"
"github.com/pingcap/tidb/pkg/util/logutil"
>>>>>>> 521c99967d5 (planner: The length function could not be substitute when collation of mapped column is utfxxx_bin (#54179)):pkg/expression/util.go
"go.uber.org/zap"
"golang.org/x/tools/container/intsets"
)
Expand Down Expand Up @@ -460,6 +477,28 @@ func ColumnSubstituteImpl(expr Expression, schema *Schema, newExprs []Expression
}
return false, false, v
}
// If the collation of the column is PAD SPACE,
// we can't propagate the constant to the length function.
// For example, schema = ['name'], newExprs = ['a'], v = length(name).
// We can't substitute name with 'a' in length(name) because the collation of name is PAD SPACE.
// TODO: We will fix it here temporarily, and redesign the logic if we encounter more similar functions or situations later.
// Fixed issue #53730
if ctx.IsConstantPropagateCheck() && v.FuncName.L == ast.Length {
arg0, isColumn := v.GetArgs()[0].(*Column)
if isColumn {
id := schema.ColumnIndex(arg0)
if id != -1 {
_, isConstant := newExprs[id].(*Constant)
if isConstant {
mappedNewColumnCollate := schema.Columns[id].GetStaticType().GetCollate()
if mappedNewColumnCollate == charset.CollationUTF8MB4 ||
mappedNewColumnCollate == charset.CollationUTF8 {
return false, false, v
}
}
}
}
}
// cowExprRef is a copy-on-write util, args array allocation happens only
// when expr in args is changed
refExprArr := cowExprRef{v.GetArgs(), nil}
Expand Down
235 changes: 235 additions & 0 deletions pkg/expression/context/context.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,235 @@
// Copyright 2024 PingCAP, Inc.
//
// Licensed under the Apache License, Version 2.0 (the "License");
// you may not use this file except in compliance with the License.
// You may obtain a copy of the License at
//
// http://www.apache.org/licenses/LICENSE-2.0
//
// Unless required by applicable law or agreed to in writing, software
// distributed under the License is distributed on an "AS IS" BASIS,
// WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
// See the License for the specific language governing permissions and
// limitations under the License.

package context

import (
"sync/atomic"
"time"

"github.com/pingcap/tidb/pkg/errctx"
"github.com/pingcap/tidb/pkg/parser/mysql"
"github.com/pingcap/tidb/pkg/sessionctx/variable"
"github.com/pingcap/tidb/pkg/types"
contextutil "github.com/pingcap/tidb/pkg/util/context"
"github.com/pingcap/tidb/pkg/util/intest"
"github.com/pingcap/tidb/pkg/util/mathutil"
)

// PlanColumnIDAllocator allocates column id for plan.
type PlanColumnIDAllocator interface {
// AllocPlanColumnID allocates column id for plan.
AllocPlanColumnID() int64
}

// SimplePlanColumnIDAllocator implements PlanColumnIDAllocator
type SimplePlanColumnIDAllocator struct {
id atomic.Int64
}

// NewSimplePlanColumnIDAllocator creates a new SimplePlanColumnIDAllocator.
func NewSimplePlanColumnIDAllocator(offset int64) *SimplePlanColumnIDAllocator {
alloc := &SimplePlanColumnIDAllocator{}
alloc.id.Store(offset)
return alloc
}

// AllocPlanColumnID allocates column id for plan.
func (a *SimplePlanColumnIDAllocator) AllocPlanColumnID() int64 {
return a.id.Add(1)
}

// EvalContext is used to evaluate an expression
type EvalContext interface {
contextutil.WarnHandler
// CtxID indicates the id of the context.
CtxID() uint64
// SQLMode returns the sql mode
SQLMode() mysql.SQLMode
// TypeCtx returns the types.Context
TypeCtx() types.Context
// ErrCtx returns the errctx.Context
ErrCtx() errctx.Context
// Location returns the timezone info
Location() *time.Location
// CurrentDB return the current database name
CurrentDB() string
// CurrentTime returns the current time.
// Multiple calls for CurrentTime() should return the same value for the same `CtxID`.
CurrentTime() (time.Time, error)
// GetMaxAllowedPacket returns the value of the 'max_allowed_packet' system variable.
GetMaxAllowedPacket() uint64
// GetDefaultWeekFormatMode returns the value of the 'default_week_format' system variable.
GetDefaultWeekFormatMode() string
// GetDivPrecisionIncrement returns the specified value of DivPrecisionIncrement.
GetDivPrecisionIncrement() int
// RequestVerification verifies user privilege
RequestVerification(db, table, column string, priv mysql.PrivilegeType) bool
// RequestDynamicVerification verifies user privilege for a DYNAMIC privilege.
RequestDynamicVerification(privName string, grantable bool) bool
// GetOptionalPropSet returns the optional properties provided by this context.
GetOptionalPropSet() OptionalEvalPropKeySet
// GetOptionalPropProvider gets the optional property provider by key
GetOptionalPropProvider(OptionalEvalPropKey) (OptionalEvalPropProvider, bool)
// GetParamValue returns the value of the parameter by index.
GetParamValue(idx int) types.Datum
}

// BuildContext is used to build an expression
type BuildContext interface {
// GetEvalCtx returns the EvalContext.
GetEvalCtx() EvalContext
// GetCharsetInfo gets charset and collation for current context.
GetCharsetInfo() (string, string)
// GetDefaultCollationForUTF8MB4 returns the default collation of UTF8MB4.
GetDefaultCollationForUTF8MB4() string
// GetBlockEncryptionMode returns the variable `block_encryption_mode`.
GetBlockEncryptionMode() string
// GetSysdateIsNow returns a bool to determine whether Sysdate is an alias of Now function.
// It is the value of variable `tidb_sysdate_is_now`.
GetSysdateIsNow() bool
// GetNoopFuncsMode returns the noop function mode: OFF/ON/WARN values as 0/1/2.
GetNoopFuncsMode() int
// Rng is used to generate random values.
Rng() *mathutil.MysqlRng
// IsUseCache indicates whether to cache the build expression in plan cache.
IsUseCache() bool
// SetSkipPlanCache sets to skip the plan cache and records the reason.
SetSkipPlanCache(reason string)
// AllocPlanColumnID allocates column id for plan.
AllocPlanColumnID() int64
// IsInNullRejectCheck returns the flag to indicate whether the expression is in null reject check.
// It should always return `false` in most implementations because we do not want to do null reject check
// in most cases except for the method `isNullRejected` in planner.
// See the comments for `isNullRejected` in planner for more details.
IsInNullRejectCheck() bool
// IsConstantPropagateCheck returns the flag to indicate whether the expression is in constant propagate check.
// It should be true only when we are doing constant propagation in rule_predicate_push_down.
IsConstantPropagateCheck() bool
// ConnectionID indicates the connection ID of the current session.
// If the context is not in a session, it should return 0.
ConnectionID() uint64
}

// ExprContext contains full context for expression building and evaluating.
// It also provides some additional information for to build aggregate functions.
type ExprContext interface {
BuildContext
// GetWindowingUseHighPrecision determines whether to compute window operations without loss of precision.
// see https://dev.mysql.com/doc/refman/8.0/en/window-function-optimization.html for more details.
GetWindowingUseHighPrecision() bool
// GetGroupConcatMaxLen returns the value of the 'group_concat_max_len' system variable.
GetGroupConcatMaxLen() uint64
}

// NullRejectCheckExprContext is a wrapper to return true for `IsInNullRejectCheck`.
type NullRejectCheckExprContext struct {
ExprContext
}

// WithNullRejectCheck returns a new `NullRejectCheckExprContext` with the given `ExprContext`.
func WithNullRejectCheck(ctx ExprContext) *NullRejectCheckExprContext {
return &NullRejectCheckExprContext{ExprContext: ctx}
}

// IsInNullRejectCheck always returns true for `NullRejectCheckExprContext`
func (ctx *NullRejectCheckExprContext) IsInNullRejectCheck() bool {
return true
}

// ConstantPropagateCheckContext is a wrapper to return true for `IsConstantPropagateCheck`.
type ConstantPropagateCheckContext struct {
ExprContext
}

// WithConstantPropagateCheck returns a new `ConstantPropagateCheckContext` with the given `ExprContext`.
func WithConstantPropagateCheck(ctx ExprContext) *ConstantPropagateCheckContext {
return &ConstantPropagateCheckContext{ExprContext: ctx}
}

// IsConstantPropagateCheck always returns true for `ConstantPropagateCheckContext`
func (ctx *ConstantPropagateCheckContext) IsConstantPropagateCheck() bool {
return true
}

type innerOverrideEvalContext struct {
EvalContext
typeCtx types.Context
errCtx errctx.Context
}

// TypeCtx implements EvalContext.TypeCtx
func (ctx *innerOverrideEvalContext) TypeCtx() types.Context {
return ctx.typeCtx
}

// ErrCtx implements EvalContext.GetEvalCtx
func (ctx *innerOverrideEvalContext) ErrCtx() errctx.Context {
return ctx.errCtx
}

type innerOverrideBuildContext struct {
BuildContext
evalCtx EvalContext
}

// GetEvalCtx implements BuildContext.GetEvalCtx
func (ctx *innerOverrideBuildContext) GetEvalCtx() EvalContext {
return ctx.evalCtx
}

// CtxWithHandleTruncateErrLevel returns a new BuildContext with the specified level for handling truncate error.
func CtxWithHandleTruncateErrLevel(ctx BuildContext, level errctx.Level) BuildContext {
truncateAsWarnings, ignoreTruncate := false, false
switch level {
case errctx.LevelWarn:
truncateAsWarnings = true
case errctx.LevelIgnore:
ignoreTruncate = true
default:
}

evalCtx := ctx.GetEvalCtx()
tc, ec := evalCtx.TypeCtx(), evalCtx.ErrCtx()

flags := tc.Flags().
WithTruncateAsWarning(truncateAsWarnings).
WithIgnoreTruncateErr(ignoreTruncate)

if tc.Flags() == flags && ec.LevelForGroup(errctx.ErrGroupTruncate) == level {
// We do not need to create a new context if the flags and level are the same.
return ctx
}

return &innerOverrideBuildContext{
BuildContext: ctx,
evalCtx: &innerOverrideEvalContext{
EvalContext: evalCtx,
typeCtx: tc.WithFlags(flags),
errCtx: ec.WithErrGroupLevel(errctx.ErrGroupTruncate, level),
},
}
}

// AssertLocationWithSessionVars asserts the location in the context and session variables are the same.
// It is only used for testing.
func AssertLocationWithSessionVars(ctxLoc *time.Location, vars *variable.SessionVars) {
ctxLocStr := ctxLoc.String()
varsLocStr := vars.Location().String()
stmtLocStr := vars.StmtCtx.TimeZone().String()
intest.Assert(ctxLocStr == varsLocStr && ctxLocStr == stmtLocStr,
"location mismatch, ctxLoc: %s, varsLoc: %s, stmtLoc: %s",
ctxLoc.String(), ctxLocStr, stmtLocStr,
)
}
Loading

0 comments on commit 7aad36c

Please sign in to comment.