From 6cbf5284578663455f13f4cb7bffa28b372f5080 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Mon, 10 Dec 2018 11:03:21 +0800 Subject: [PATCH 1/7] expression: constraint propagate for '>' and monotonous function --- expression/constant_test.go | 97 ++++++++++- expression/constraint_propagation.go | 234 +++++++++++++++++++++++++-- 2 files changed, 315 insertions(+), 16 deletions(-) diff --git a/expression/constant_test.go b/expression/constant_test.go index f07047a5f9bbe..b788363c97972 100644 --- a/expression/constant_test.go +++ b/expression/constant_test.go @@ -32,12 +32,16 @@ var _ = Suite(&testExpressionSuite{}) type testExpressionSuite struct{} func newColumn(id int) *Column { + return newColumnWithType(id, types.NewFieldType(mysql.TypeLonglong)) +} + +func newColumnWithType(id int, t *types.FieldType) *Column { return &Column{ UniqueID: int64(id), ColName: model.NewCIStr(fmt.Sprint(id)), TblName: model.NewCIStr("t"), DBName: model.NewCIStr("test"), - RetType: types.NewFieldType(mysql.TypeLonglong), + RetType: t, } } @@ -48,6 +52,15 @@ func newLonglong(value int64) *Constant { } } +func newDate(year, month, day int) *Constant { + var tmp types.Datum + tmp.SetMysqlTime(types.Time{types.FromDate(year, month, day, 0, 0, 0, 0), mysql.TypeDate, 0}) + return &Constant{ + Value: tmp, + RetType: types.NewFieldType(mysql.TypeDate), + } +} + func newFunction(funcName string, args ...Expression) Expression { typeLong := types.NewFieldType(mysql.TypeLonglong) return NewFunctionInternal(mock.NewContext(), funcName, typeLong, args...) @@ -180,6 +193,88 @@ func (*testExpressionSuite) TestConstantPropagation(c *C) { } } +func (*testExpressionSuite) TestConstraintPropagation(c *C) { + defer testleak.AfterTest(c)() + col1 := newColumnWithType(1, types.NewFieldType(mysql.TypeDate)) + tests := []struct { + solver constraintSolver + conditions []Expression + result string + }{ + { + solver: newConstraintSolver(ruleColumnGTConst), + conditions: []Expression{ + newFunction(ast.GT, newColumn(0), newLonglong(5)), + newFunction(ast.GT, newColumn(0), newLonglong(7)), + }, + result: "gt(test.t.0, 7)", + }, + { + solver: newConstraintSolver(ruleColumnGTConst), + conditions: []Expression{ + newFunction(ast.GT, newColumn(0), newLonglong(5)), + newFunction(ast.LT, newColumn(0), newLonglong(5)), + }, + result: "0", + }, + { + solver: newConstraintSolver(ruleColumnGTConst), + conditions: []Expression{ + newFunction(ast.GT, newColumn(0), newLonglong(7)), + newFunction(ast.LT, newColumn(0), newLonglong(5)), + }, + result: "0", + }, + { + solver: newConstraintSolver(ruleColumnGTConst), + // col1 > '2018-12-11' and to_days(col1) < 5 => false + conditions: []Expression{ + newFunction(ast.GT, col1, newDate(2018, 12, 11)), + newFunction(ast.LT, newFunction(ast.ToDays, col1), newLonglong(5)), + }, + result: "0", + }, + { + solver: newConstraintSolver(ruleColumnLTConst), + conditions: []Expression{ + newFunction(ast.LT, newColumn(0), newLonglong(5)), + newFunction(ast.LT, newColumn(0), newLonglong(7)), + }, + result: "lt(test.t.0, 5)", + }, + { + solver: newConstraintSolver(ruleColumnLTConst), + conditions: []Expression{ + newFunction(ast.LT, newColumn(0), newLonglong(5)), + newFunction(ast.GT, newColumn(0), newLonglong(5)), + }, + result: "0", + }, + { + solver: newConstraintSolver(ruleColumnLTConst), + conditions: []Expression{ + newFunction(ast.LT, newColumn(0), newLonglong(5)), + newFunction(ast.GT, newColumn(0), newLonglong(7)), + }, + result: "0", + }, + } + for _, tt := range tests { + ctx := mock.NewContext() + conds := make([]Expression, 0, len(tt.conditions)) + for _, cd := range tt.conditions { + conds = append(conds, FoldConstant(cd)) + } + newConds := tt.solver.Solve(ctx, conds) + var result []string + for _, v := range newConds { + result = append(result, v.String()) + } + sort.Strings(result) + c.Assert(strings.Join(result, ", "), Equals, tt.result, Commentf("different for expr %s", tt.conditions)) + } +} + func (*testExpressionSuite) TestConstantFolding(c *C) { defer testleak.AfterTest(c)() tests := []struct { diff --git a/expression/constraint_propagation.go b/expression/constraint_propagation.go index 17454df5a35e3..f30ba13590150 100644 --- a/expression/constraint_propagation.go +++ b/expression/constraint_propagation.go @@ -15,7 +15,10 @@ package expression import ( "bytes" + "fmt" + "github.com/pingcap/errors" + "github.com/pingcap/parser/ast" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/sessionctx" "github.com/pingcap/tidb/types" @@ -78,24 +81,35 @@ func newExprSet(conditions []Expression) *exprSet { return &exprs } +type constraintSolver []constraintPropagateRule + +func newConstraintSolver(rules ...constraintPropagateRule) constraintSolver { + return constraintSolver(rules) +} + type pgSolver2 struct{} -// PropagateConstant propagate constant values of deterministic predicates in a condition. -func (s pgSolver2) PropagateConstant(ctx sessionctx.Context, conditions []Expression) []Expression { +func (_ pgSolver2) PropagateConstant(ctx sessionctx.Context, conditions []Expression) []Expression { + solver := newConstraintSolver(ruleConstantFalse, ruleColumnEQConst) + return solver.Solve(ctx, conditions) +} + +// Solve propagate constraint according to the rules in the constraintSolver. +func (s constraintSolver) Solve(ctx sessionctx.Context, conditions []Expression) []Expression { exprs := newExprSet(conditions) s.fixPoint(ctx, exprs) return exprs.Slice() } -// fixPoint is the core of the constant propagation algorithm. +// fixPoint is the core of the constraint propagation algorithm. // It will iterate the expression set over and over again, pick two expressions, // apply one to another. // If new conditions can be infered, they will be append into the expression set. // Until no more conditions can be infered from the set, the algorithm finish. -func (s pgSolver2) fixPoint(ctx sessionctx.Context, exprs *exprSet) { +func (s constraintSolver) fixPoint(ctx sessionctx.Context, exprs *exprSet) { for { saveLen := len(exprs.data) - iterOnce(ctx, exprs) + s.iterOnce(ctx, exprs) if saveLen == len(exprs.data) { break } @@ -104,7 +118,7 @@ func (s pgSolver2) fixPoint(ctx sessionctx.Context, exprs *exprSet) { } // iterOnce picks two expressions from the set, try to propagate new conditions from them. -func iterOnce(ctx sessionctx.Context, exprs *exprSet) { +func (s constraintSolver) iterOnce(ctx sessionctx.Context, exprs *exprSet) { for i := 0; i < len(exprs.data); i++ { if exprs.tombstone[i] { continue @@ -116,24 +130,19 @@ func iterOnce(ctx sessionctx.Context, exprs *exprSet) { if i == j { continue } - solve(ctx, i, j, exprs) + s.solve(ctx, i, j, exprs) } } } // solve uses exprs[i] exprs[j] to propagate new conditions. -func solve(ctx sessionctx.Context, i, j int, exprs *exprSet) { - for _, rule := range rules { +func (s constraintSolver) solve(ctx sessionctx.Context, i, j int, exprs *exprSet) { + for _, rule := range s { rule(ctx, i, j, exprs) } } -type constantPropagateRule func(ctx sessionctx.Context, i, j int, exprs *exprSet) - -var rules = []constantPropagateRule{ - ruleConstantFalse, - ruleColumnEQConst, -} +type constraintPropagateRule func(ctx sessionctx.Context, i, j int, exprs *exprSet) // ruleConstantFalse propagates from CNF condition that false plus anything returns false. // false, a = 1, b = c ... => false @@ -164,3 +173,198 @@ func ruleColumnEQConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { } } } + +// ruleColumnGTConst propagates the "column > const" condition. +func ruleColumnGTConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { + cond := exprs.data[i] + f1, ok := cond.(*ScalarFunction) + if !ok || f1.FuncName.L != ast.GT { + return + } + var col1 *Column + var con1 *Constant + col1, ok = f1.GetArgs()[0].(*Column) + if !ok { + return + } + con1, ok = f1.GetArgs()[1].(*Constant) + if !ok { + return + } + + expr := exprs.data[j] + f2, ok := expr.(*ScalarFunction) + if !ok { + return + } + + // col > c1, col > c2, c1 > c2 => col > c1 + if f2.FuncName.L == ast.GT { + col2, ok := f2.GetArgs()[0].(*Column) + if !ok || !col1.Equal(ctx, col2) { + return + } + con2, ok := f2.GetArgs()[1].(*Constant) + if !ok { + return + } + if !con1.RetType.Equal(con2.RetType) { + return + } + + v, isNull, err := compareConstant(ctx, ast.GT, con1, con2) + if err != nil { + log.Warn(err) + return + } + if !isNull && v > 0 { + exprs.tombstone[j] = true + } + return + } + + // The simple case: + // col > c1, col < c2, c1 >= c2 => false + // + // The extended case: + // col > c1, f(col) < c2, f is monotonous, f(c1) <= c2 => false + // + // Prove: + // col > c1, f is monotonous => f(col) > f(c1) + // f(col) > f(c1), f(col) < c2, f(c1) >= c2 => false + if f2.FuncName.L == ast.LT { + con2, ok := f2.GetArgs()[1].(*Constant) + if !ok { + return + } + arg0 := f2.GetArgs()[0] + // The simple case. + var fc1 Expression + col2, ok := arg0.(*Column) + if ok { + fc1 = con1 + } else { + // The extended case. + scalarFunc, ok := arg0.(*ScalarFunction) + if !ok { + return + } + _, ok = monotoneFuncs[scalarFunc.FuncName.L] + if !ok { + return + } + col2, ok = scalarFunc.GetArgs()[0].(*Column) + if !ok { + return + } + var err error + fc1, err = NewFunction(ctx, scalarFunc.FuncName.L, scalarFunc.RetType, con1) + if err != nil { + log.Warn(err) + return + } + } + if !col1.Equal(ctx, col2) { + return + } + v, isNull, err := compareConstant(ctx, ast.GE, fc1, con2) + if err != nil { + log.Warn(err) + return + } + if !isNull && v > 0 { + exprs.SetConstFalse() + } + return + } + return +} + +var monotoneFuncs = map[string]struct{}{ + ast.ToDays: struct{}{}, +} + +// compareConstant compares two expressions. c1 and c2 should be constant with the same type. +func compareConstant(ctx sessionctx.Context, fn string, c1, c2 Expression) (int64, bool, error) { + cmp, err := NewFunction(ctx, fn, c1.GetType(), c1, c2) + if err != nil { + return 0, false, errors.Trace(err) + } + return cmp.EvalInt(ctx, chunk.Row{}) +} + +// ruleColumnLTConst propagates the "column < const" condition. +func ruleColumnLTConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { + cond := exprs.data[i] + f1, ok := cond.(*ScalarFunction) + if !ok || f1.FuncName.L != ast.LT { + return + } + var col1 *Column + var con1 *Constant + col1, ok = f1.GetArgs()[0].(*Column) + if !ok { + return + } + con1, ok = f1.GetArgs()[1].(*Constant) + if !ok { + return + } + + expr := exprs.data[j] + f2, ok := expr.(*ScalarFunction) + if !ok { + return + } + // col < c1, col < c2, c1 < c2 => col < c1 + if f2.FuncName.L == ast.LT { + col2, ok := f2.GetArgs()[0].(*Column) + if !ok || !col1.Equal(ctx, col2) { + return + } + con2, ok := f2.GetArgs()[1].(*Constant) + if !ok { + return + } + if !con1.RetType.Equal(con2.RetType) { + return + } + + v, isNull, err := compareConstant(ctx, ast.LT, con1, con2) + if err != nil { + log.Warn(err) + return + } + if !isNull && v > 0 { + exprs.tombstone[j] = true + } + return + } + + // col < c1, col > c2, c1 <= c2 => false + if f2.FuncName.L == ast.GT { + col2, ok := f2.GetArgs()[0].(*Column) + if !ok || !col1.Equal(ctx, col2) { + return + } + con2, ok := f2.GetArgs()[1].(*Constant) + if !ok { + return + } + + if !con1.RetType.Equal(con2.RetType) { + return + } + + v, isNull, err := compareConstant(ctx, ast.LE, con1, con2) + if err != nil { + log.Warn(err) + return + } + if !isNull && v > 0 { + exprs.SetConstFalse() + } + return + } + return +} From e11f0ed789497e8497b9a5d4fb04afd108137985 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 11 Dec 2018 10:49:00 +0800 Subject: [PATCH 2/7] Merge master --- expression/constant_test.go | 9 +++ expression/constraint_propagation.go | 98 +++++----------------------- go.mod | 2 +- go.sum | 4 +- 4 files changed, 28 insertions(+), 85 deletions(-) diff --git a/expression/constant_test.go b/expression/constant_test.go index b788363c97972..acd1240b4aaad 100644 --- a/expression/constant_test.go +++ b/expression/constant_test.go @@ -258,6 +258,15 @@ func (*testExpressionSuite) TestConstraintPropagation(c *C) { }, result: "0", }, + { + solver: newConstraintSolver(ruleColumnLTConst), + // col1 < '2018-12-11' and to_days(col1) > 737999 => false + conditions: []Expression{ + newFunction(ast.LT, col1, newDate(2018, 12, 11)), + newFunction(ast.GT, newFunction(ast.ToDays, col1), newLonglong(737999)), + }, + result: "0", + }, } for _, tt := range tests { ctx := mock.NewContext() diff --git a/expression/constraint_propagation.go b/expression/constraint_propagation.go index f30ba13590150..317d403d5b21a 100644 --- a/expression/constraint_propagation.go +++ b/expression/constraint_propagation.go @@ -174,11 +174,21 @@ func ruleColumnEQConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { } } -// ruleColumnGTConst propagates the "column > const" condition. +// ruleColumnLTConst propagates the "column > const" condition. func ruleColumnGTConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { + ruleColumnXXConst(ctx, i, j, exprs, ast.GT, ast.LT, ast.GE) +} + +// ruleColumnLTConst propagates the "column < const" condition. +func ruleColumnLTConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { + ruleColumnXXConst(ctx, i, j, exprs, ast.LT, ast.GT, ast.LE) +} + +// ruleColumnXXConst propagates the "column OP const" condition, where op may be '>' and '<' +func ruleColumnXXConst(ctx sessionctx.Context, i, j int, exprs *exprSet, GT string, LT string, GE string) { cond := exprs.data[i] f1, ok := cond.(*ScalarFunction) - if !ok || f1.FuncName.L != ast.GT { + if !ok || f1.FuncName.L != GT { return } var col1 *Column @@ -199,7 +209,7 @@ func ruleColumnGTConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { } // col > c1, col > c2, c1 > c2 => col > c1 - if f2.FuncName.L == ast.GT { + if f2.FuncName.L == GT { col2, ok := f2.GetArgs()[0].(*Column) if !ok || !col1.Equal(ctx, col2) { return @@ -212,7 +222,7 @@ func ruleColumnGTConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { return } - v, isNull, err := compareConstant(ctx, ast.GT, con1, con2) + v, isNull, err := compareConstant(ctx, GT, con1, con2) if err != nil { log.Warn(err) return @@ -232,7 +242,7 @@ func ruleColumnGTConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { // Prove: // col > c1, f is monotonous => f(col) > f(c1) // f(col) > f(c1), f(col) < c2, f(c1) >= c2 => false - if f2.FuncName.L == ast.LT { + if f2.FuncName.L == LT { con2, ok := f2.GetArgs()[1].(*Constant) if !ok { return @@ -267,7 +277,7 @@ func ruleColumnGTConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { if !col1.Equal(ctx, col2) { return } - v, isNull, err := compareConstant(ctx, ast.GE, fc1, con2) + v, isNull, err := compareConstant(ctx, GE, fc1, con2) if err != nil { log.Warn(err) return @@ -292,79 +302,3 @@ func compareConstant(ctx sessionctx.Context, fn string, c1, c2 Expression) (int6 } return cmp.EvalInt(ctx, chunk.Row{}) } - -// ruleColumnLTConst propagates the "column < const" condition. -func ruleColumnLTConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { - cond := exprs.data[i] - f1, ok := cond.(*ScalarFunction) - if !ok || f1.FuncName.L != ast.LT { - return - } - var col1 *Column - var con1 *Constant - col1, ok = f1.GetArgs()[0].(*Column) - if !ok { - return - } - con1, ok = f1.GetArgs()[1].(*Constant) - if !ok { - return - } - - expr := exprs.data[j] - f2, ok := expr.(*ScalarFunction) - if !ok { - return - } - // col < c1, col < c2, c1 < c2 => col < c1 - if f2.FuncName.L == ast.LT { - col2, ok := f2.GetArgs()[0].(*Column) - if !ok || !col1.Equal(ctx, col2) { - return - } - con2, ok := f2.GetArgs()[1].(*Constant) - if !ok { - return - } - if !con1.RetType.Equal(con2.RetType) { - return - } - - v, isNull, err := compareConstant(ctx, ast.LT, con1, con2) - if err != nil { - log.Warn(err) - return - } - if !isNull && v > 0 { - exprs.tombstone[j] = true - } - return - } - - // col < c1, col > c2, c1 <= c2 => false - if f2.FuncName.L == ast.GT { - col2, ok := f2.GetArgs()[0].(*Column) - if !ok || !col1.Equal(ctx, col2) { - return - } - con2, ok := f2.GetArgs()[1].(*Constant) - if !ok { - return - } - - if !con1.RetType.Equal(con2.RetType) { - return - } - - v, isNull, err := compareConstant(ctx, ast.LE, con1, con2) - if err != nil { - log.Warn(err) - return - } - if !isNull && v > 0 { - exprs.SetConstFalse() - } - return - } - return -} diff --git a/go.mod b/go.mod index c1ed756e0b4b8..08135b2bc61e1 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/pingcap/errors v0.11.0 github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20181105061835-1b5d69cd1d26 - github.com/pingcap/parser v0.0.0-20181210061630-27e9d3e251d4 + github.com/pingcap/parser v0.0.0-20181211024540-4e6d047fcaae github.com/pingcap/pd v2.1.0-rc.4+incompatible github.com/pingcap/tidb-tools v0.0.0-20181112132202-4860a0d5de03 github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323 diff --git a/go.sum b/go.sum index 6a8c51c3ac0a9..be57fc9899053 100644 --- a/go.sum +++ b/go.sum @@ -107,8 +107,8 @@ github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rG github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20181105061835-1b5d69cd1d26 h1:JK4VLNYbSn36QSbCnqALi2ySXdH0DfcMssT/zmLf4Ls= github.com/pingcap/kvproto v0.0.0-20181105061835-1b5d69cd1d26/go.mod h1:0gwbe1F2iBIjuQ9AH0DbQhL+Dpr5GofU8fgYyXk+ykk= -github.com/pingcap/parser v0.0.0-20181210061630-27e9d3e251d4 h1:2rCHDk4h8VZw0fiC2CFJffOlXU3iMuz1kOt5wTMCemY= -github.com/pingcap/parser v0.0.0-20181210061630-27e9d3e251d4/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20181211024540-4e6d047fcaae h1:RD98+89F/yakFLnztEL4Pi9f+RkOAm2vgRJcB1p6tTw= +github.com/pingcap/parser v0.0.0-20181211024540-4e6d047fcaae/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v2.1.0-rc.4+incompatible h1:/buwGk04aHO5odk/+O8ZOXGs4qkUjYTJ2UpCJXna8NE= github.com/pingcap/pd v2.1.0-rc.4+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/tidb-tools v0.0.0-20181112132202-4860a0d5de03 h1:xVuo5U+l6XAWHsb+xhkZ8zz3jerIwDfCHAO6kR2Kaog= From 12d92128bf798ce3d3d53a2e379aebe315133932 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 11 Dec 2018 11:26:34 +0800 Subject: [PATCH 3/7] gofmt --- expression/constraint_propagation.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/expression/constraint_propagation.go b/expression/constraint_propagation.go index 317d403d5b21a..e6b67314a6faa 100644 --- a/expression/constraint_propagation.go +++ b/expression/constraint_propagation.go @@ -291,7 +291,7 @@ func ruleColumnXXConst(ctx sessionctx.Context, i, j int, exprs *exprSet, GT stri } var monotoneFuncs = map[string]struct{}{ - ast.ToDays: struct{}{}, + ast.ToDays: {}, } // compareConstant compares two expressions. c1 and c2 should be constant with the same type. From 337657194e7a9667114dff80817bf16b17ed1b18 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 11 Dec 2018 11:36:00 +0800 Subject: [PATCH 4/7] fix CI --- expression/constraint_propagation.go | 1 - 1 file changed, 1 deletion(-) diff --git a/expression/constraint_propagation.go b/expression/constraint_propagation.go index e6b67314a6faa..4ea22f63ff99c 100644 --- a/expression/constraint_propagation.go +++ b/expression/constraint_propagation.go @@ -15,7 +15,6 @@ package expression import ( "bytes" - "fmt" "github.com/pingcap/errors" "github.com/pingcap/parser/ast" From 9a3659bb0dde6267a88df95f842de1959be61857 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Tue, 11 Dec 2018 12:35:38 +0800 Subject: [PATCH 5/7] fix CI --- expression/constant_test.go | 5 ++++- expression/constraint_propagation.go | 2 +- go.mod | 2 +- go.sum | 4 ++-- 4 files changed, 8 insertions(+), 5 deletions(-) diff --git a/expression/constant_test.go b/expression/constant_test.go index acd1240b4aaad..0a34770e6de53 100644 --- a/expression/constant_test.go +++ b/expression/constant_test.go @@ -54,7 +54,10 @@ func newLonglong(value int64) *Constant { func newDate(year, month, day int) *Constant { var tmp types.Datum - tmp.SetMysqlTime(types.Time{types.FromDate(year, month, day, 0, 0, 0, 0), mysql.TypeDate, 0}) + tmp.SetMysqlTime(types.Time{ + Time: types.FromDate(year, month, day, 0, 0, 0, 0), + Type: mysql.TypeDate, + }) return &Constant{ Value: tmp, RetType: types.NewFieldType(mysql.TypeDate), diff --git a/expression/constraint_propagation.go b/expression/constraint_propagation.go index 4ea22f63ff99c..225078e267ada 100644 --- a/expression/constraint_propagation.go +++ b/expression/constraint_propagation.go @@ -88,7 +88,7 @@ func newConstraintSolver(rules ...constraintPropagateRule) constraintSolver { type pgSolver2 struct{} -func (_ pgSolver2) PropagateConstant(ctx sessionctx.Context, conditions []Expression) []Expression { +func (s pgSolver2) PropagateConstant(ctx sessionctx.Context, conditions []Expression) []Expression { solver := newConstraintSolver(ruleConstantFalse, ruleColumnEQConst) return solver.Solve(ctx, conditions) } diff --git a/go.mod b/go.mod index 08135b2bc61e1..c1ed756e0b4b8 100644 --- a/go.mod +++ b/go.mod @@ -47,7 +47,7 @@ require ( github.com/pingcap/errors v0.11.0 github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e github.com/pingcap/kvproto v0.0.0-20181105061835-1b5d69cd1d26 - github.com/pingcap/parser v0.0.0-20181211024540-4e6d047fcaae + github.com/pingcap/parser v0.0.0-20181210061630-27e9d3e251d4 github.com/pingcap/pd v2.1.0-rc.4+incompatible github.com/pingcap/tidb-tools v0.0.0-20181112132202-4860a0d5de03 github.com/pingcap/tipb v0.0.0-20181012112600-11e33c750323 diff --git a/go.sum b/go.sum index be57fc9899053..6a8c51c3ac0a9 100644 --- a/go.sum +++ b/go.sum @@ -107,8 +107,8 @@ github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e h1:P73/4dPCL96rG github.com/pingcap/goleveldb v0.0.0-20171020122428-b9ff6c35079e/go.mod h1:O17XtbryoCJhkKGbT62+L2OlrniwqiGLSqrmdHCMzZw= github.com/pingcap/kvproto v0.0.0-20181105061835-1b5d69cd1d26 h1:JK4VLNYbSn36QSbCnqALi2ySXdH0DfcMssT/zmLf4Ls= github.com/pingcap/kvproto v0.0.0-20181105061835-1b5d69cd1d26/go.mod h1:0gwbe1F2iBIjuQ9AH0DbQhL+Dpr5GofU8fgYyXk+ykk= -github.com/pingcap/parser v0.0.0-20181211024540-4e6d047fcaae h1:RD98+89F/yakFLnztEL4Pi9f+RkOAm2vgRJcB1p6tTw= -github.com/pingcap/parser v0.0.0-20181211024540-4e6d047fcaae/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= +github.com/pingcap/parser v0.0.0-20181210061630-27e9d3e251d4 h1:2rCHDk4h8VZw0fiC2CFJffOlXU3iMuz1kOt5wTMCemY= +github.com/pingcap/parser v0.0.0-20181210061630-27e9d3e251d4/go.mod h1:1FNvfp9+J0wvc4kl8eGNh7Rqrxveg15jJoWo/a0uHwA= github.com/pingcap/pd v2.1.0-rc.4+incompatible h1:/buwGk04aHO5odk/+O8ZOXGs4qkUjYTJ2UpCJXna8NE= github.com/pingcap/pd v2.1.0-rc.4+incompatible/go.mod h1:nD3+EoYes4+aNNODO99ES59V83MZSI+dFbhyr667a0E= github.com/pingcap/tidb-tools v0.0.0-20181112132202-4860a0d5de03 h1:xVuo5U+l6XAWHsb+xhkZ8zz3jerIwDfCHAO6kR2Kaog= From b3287fcc6bceb6a22fafb271ac5463a736076164 Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Sat, 29 Dec 2018 14:38:30 +0800 Subject: [PATCH 6/7] address comment --- expression/constraint_propagation.go | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/expression/constraint_propagation.go b/expression/constraint_propagation.go index 225078e267ada..d89347779126a 100644 --- a/expression/constraint_propagation.go +++ b/expression/constraint_propagation.go @@ -16,7 +16,6 @@ package expression import ( "bytes" - "github.com/pingcap/errors" "github.com/pingcap/parser/ast" "github.com/pingcap/parser/mysql" "github.com/pingcap/tidb/sessionctx" @@ -238,7 +237,7 @@ func ruleColumnXXConst(ctx sessionctx.Context, i, j int, exprs *exprSet, GT stri // The extended case: // col > c1, f(col) < c2, f is monotonous, f(c1) <= c2 => false // - // Prove: + // Proof: // col > c1, f is monotonous => f(col) > f(c1) // f(col) > f(c1), f(col) < c2, f(c1) >= c2 => false if f2.FuncName.L == LT { @@ -258,7 +257,7 @@ func ruleColumnXXConst(ctx sessionctx.Context, i, j int, exprs *exprSet, GT stri if !ok { return } - _, ok = monotoneFuncs[scalarFunc.FuncName.L] + _, ok = monotoneIncFuncs[scalarFunc.FuncName.L] if !ok { return } @@ -289,15 +288,16 @@ func ruleColumnXXConst(ctx sessionctx.Context, i, j int, exprs *exprSet, GT stri return } -var monotoneFuncs = map[string]struct{}{ +// monotoneIncFuncs are those functions that for any x y, if x > y => f(x) > f(y) +var monotoneIncFuncs = map[string]struct{}{ ast.ToDays: {}, } // compareConstant compares two expressions. c1 and c2 should be constant with the same type. func compareConstant(ctx sessionctx.Context, fn string, c1, c2 Expression) (int64, bool, error) { - cmp, err := NewFunction(ctx, fn, c1.GetType(), c1, c2) + cmp, err := NewFunction(ctx, fn, types.NewFieldType(mysql.TypeTiny), c1, c2) if err != nil { - return 0, false, errors.Trace(err) + return 0, false, err } return cmp.EvalInt(ctx, chunk.Row{}) } From bdcf79971ca9f06718af02128991d10078da57fb Mon Sep 17 00:00:00 2001 From: tiancaiamao Date: Wed, 2 Jan 2019 11:43:36 +0800 Subject: [PATCH 7/7] make the code more general and remove useless part --- expression/constant_test.go | 38 +++---- expression/constraint_propagation.go | 147 ++++++++++++++------------- 2 files changed, 90 insertions(+), 95 deletions(-) diff --git a/expression/constant_test.go b/expression/constant_test.go index 0a34770e6de53..e0eba43757412 100644 --- a/expression/constant_test.go +++ b/expression/constant_test.go @@ -204,16 +204,18 @@ func (*testExpressionSuite) TestConstraintPropagation(c *C) { conditions []Expression result string }{ + // Don't propagate this any more, because it makes the code more complex but not + // useful for partition pruning. + // { + // solver: newConstraintSolver(ruleColumnGTConst), + // conditions: []Expression{ + // newFunction(ast.GT, newColumn(0), newLonglong(5)), + // newFunction(ast.GT, newColumn(0), newLonglong(7)), + // }, + // result: "gt(test.t.0, 7)", + // }, { - solver: newConstraintSolver(ruleColumnGTConst), - conditions: []Expression{ - newFunction(ast.GT, newColumn(0), newLonglong(5)), - newFunction(ast.GT, newColumn(0), newLonglong(7)), - }, - result: "gt(test.t.0, 7)", - }, - { - solver: newConstraintSolver(ruleColumnGTConst), + solver: newConstraintSolver(ruleColumnOPConst), conditions: []Expression{ newFunction(ast.GT, newColumn(0), newLonglong(5)), newFunction(ast.LT, newColumn(0), newLonglong(5)), @@ -221,7 +223,7 @@ func (*testExpressionSuite) TestConstraintPropagation(c *C) { result: "0", }, { - solver: newConstraintSolver(ruleColumnGTConst), + solver: newConstraintSolver(ruleColumnOPConst), conditions: []Expression{ newFunction(ast.GT, newColumn(0), newLonglong(7)), newFunction(ast.LT, newColumn(0), newLonglong(5)), @@ -229,7 +231,7 @@ func (*testExpressionSuite) TestConstraintPropagation(c *C) { result: "0", }, { - solver: newConstraintSolver(ruleColumnGTConst), + solver: newConstraintSolver(ruleColumnOPConst), // col1 > '2018-12-11' and to_days(col1) < 5 => false conditions: []Expression{ newFunction(ast.GT, col1, newDate(2018, 12, 11)), @@ -238,15 +240,7 @@ func (*testExpressionSuite) TestConstraintPropagation(c *C) { result: "0", }, { - solver: newConstraintSolver(ruleColumnLTConst), - conditions: []Expression{ - newFunction(ast.LT, newColumn(0), newLonglong(5)), - newFunction(ast.LT, newColumn(0), newLonglong(7)), - }, - result: "lt(test.t.0, 5)", - }, - { - solver: newConstraintSolver(ruleColumnLTConst), + solver: newConstraintSolver(ruleColumnOPConst), conditions: []Expression{ newFunction(ast.LT, newColumn(0), newLonglong(5)), newFunction(ast.GT, newColumn(0), newLonglong(5)), @@ -254,7 +248,7 @@ func (*testExpressionSuite) TestConstraintPropagation(c *C) { result: "0", }, { - solver: newConstraintSolver(ruleColumnLTConst), + solver: newConstraintSolver(ruleColumnOPConst), conditions: []Expression{ newFunction(ast.LT, newColumn(0), newLonglong(5)), newFunction(ast.GT, newColumn(0), newLonglong(7)), @@ -262,7 +256,7 @@ func (*testExpressionSuite) TestConstraintPropagation(c *C) { result: "0", }, { - solver: newConstraintSolver(ruleColumnLTConst), + solver: newConstraintSolver(ruleColumnOPConst), // col1 < '2018-12-11' and to_days(col1) > 737999 => false conditions: []Expression{ newFunction(ast.LT, col1, newDate(2018, 12, 11)), diff --git a/expression/constraint_propagation.go b/expression/constraint_propagation.go index 959c222351d45..aafbf1660d306 100644 --- a/expression/constraint_propagation.go +++ b/expression/constraint_propagation.go @@ -172,23 +172,19 @@ func ruleColumnEQConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { } } -// ruleColumnLTConst propagates the "column > const" condition. -func ruleColumnGTConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { - ruleColumnXXConst(ctx, i, j, exprs, ast.GT, ast.LT, ast.GE) -} - -// ruleColumnLTConst propagates the "column < const" condition. -func ruleColumnLTConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { - ruleColumnXXConst(ctx, i, j, exprs, ast.LT, ast.GT, ast.LE) -} - -// ruleColumnXXConst propagates the "column OP const" condition, where op may be '>' and '<' -func ruleColumnXXConst(ctx sessionctx.Context, i, j int, exprs *exprSet, GT string, LT string, GE string) { +// ruleColumnOPConst propagates the "column OP const" condition. +func ruleColumnOPConst(ctx sessionctx.Context, i, j int, exprs *exprSet) { cond := exprs.data[i] f1, ok := cond.(*ScalarFunction) - if !ok || f1.FuncName.L != GT { + if !ok { + return + } + if f1.FuncName.L != ast.GE && f1.FuncName.L != ast.GT && + f1.FuncName.L != ast.LE && f1.FuncName.L != ast.LT { return } + OP1 := f1.FuncName.L + var col1 *Column var con1 *Constant col1, ok = f1.GetArgs()[0].(*Column) @@ -206,88 +202,93 @@ func ruleColumnXXConst(ctx sessionctx.Context, i, j int, exprs *exprSet, GT stri return } - // col > c1, col > c2, c1 > c2 => col > c1 - if f2.FuncName.L == GT { - col2, ok := f2.GetArgs()[0].(*Column) - if !ok || !col1.Equal(ctx, col2) { - return - } - con2, ok := f2.GetArgs()[1].(*Constant) - if !ok { - return - } - if !con1.RetType.Equal(con2.RetType) { - return - } - - v, isNull, err := compareConstant(ctx, GT, con1, con2) - if err != nil { - log.Warn(err) - return - } - if !isNull && v > 0 { - exprs.tombstone[j] = true - } - return - } - // The simple case: - // col > c1, col < c2, c1 >= c2 => false + // col >= c1, col < c2, c1 >= c2 => false + // col >= c1, col <= c2, c1 > c2 => false + // col >= c1, col OP c2, c1 ^OP c2, where OP in [< , <=] => false + // col OP1 c1 where OP1 in [>= , <], col OP2 c2 where OP1 opsite OP2, c1 ^OP2 c2 => false // // The extended case: - // col > c1, f(col) < c2, f is monotonous, f(c1) <= c2 => false + // col >= c1, f(col) < c2, f is monotonous, f(c1) >= c2 => false // // Proof: // col > c1, f is monotonous => f(col) > f(c1) // f(col) > f(c1), f(col) < c2, f(c1) >= c2 => false - if f2.FuncName.L == LT { - con2, ok := f2.GetArgs()[1].(*Constant) + OP2 := f2.FuncName.L + if !opsiteOP(OP1, OP2) { + return + } + + con2, ok := f2.GetArgs()[1].(*Constant) + if !ok { + return + } + arg0 := f2.GetArgs()[0] + // The simple case. + var fc1 Expression + col2, ok := arg0.(*Column) + if ok { + fc1 = con1 + } else { + // The extended case. + scalarFunc, ok := arg0.(*ScalarFunction) if !ok { return } - arg0 := f2.GetArgs()[0] - // The simple case. - var fc1 Expression - col2, ok := arg0.(*Column) - if ok { - fc1 = con1 - } else { - // The extended case. - scalarFunc, ok := arg0.(*ScalarFunction) - if !ok { - return - } - _, ok = monotoneIncFuncs[scalarFunc.FuncName.L] - if !ok { - return - } - col2, ok = scalarFunc.GetArgs()[0].(*Column) - if !ok { - return - } - var err error - fc1, err = NewFunction(ctx, scalarFunc.FuncName.L, scalarFunc.RetType, con1) - if err != nil { - log.Warn(err) - return - } + _, ok = monotoneIncFuncs[scalarFunc.FuncName.L] + if !ok { + return } - if !col1.Equal(ctx, col2) { + col2, ok = scalarFunc.GetArgs()[0].(*Column) + if !ok { return } - v, isNull, err := compareConstant(ctx, GE, fc1, con2) + var err error + fc1, err = NewFunction(ctx, scalarFunc.FuncName.L, scalarFunc.RetType, con1) if err != nil { log.Warn(err) return } - if !isNull && v > 0 { - exprs.SetConstFalse() - } + } + if !col1.Equal(ctx, col2) { + return + } + v, isNull, err := compareConstant(ctx, negOP(OP2), fc1, con2) + if err != nil { + log.Warn(err) return } + if !isNull && v > 0 { + exprs.SetConstFalse() + } return } +// opsiteOP the opsite direction of a compare operation, used in ruleColumnOPConst. +func opsiteOP(op1, op2 string) bool { + switch { + case op1 == ast.GE || op1 == ast.GT: + return op2 == ast.LT || op2 == ast.LE + case op1 == ast.LE || op1 == ast.LT: + return op2 == ast.GT || op2 == ast.GE + } + return false +} + +func negOP(cmp string) string { + switch cmp { + case ast.LT: + return ast.GE + case ast.LE: + return ast.GT + case ast.GT: + return ast.LE + case ast.GE: + return ast.LT + } + return "" +} + // monotoneIncFuncs are those functions that for any x y, if x > y => f(x) > f(y) var monotoneIncFuncs = map[string]struct{}{ ast.ToDays: {},