ranger: fix the range construction behavior when the column's type is YEAR (#23559)

types/datum.go

@@ -848,7 +848,7 @@ func (d *Datum) ConvertTo(sc *stmtctx.StatementContext, target *FieldType) (Datu
 	case mysql.TypeNewDecimal:
 		return d.convertToMysqlDecimal(sc, target)
 	case mysql.TypeYear:
-		return d.convertToMysqlYear(sc, target)
+		return d.ConvertToMysqlYear(sc, target)
 	case mysql.TypeEnum:
 		return d.convertToMysqlEnum(sc, target)
 	case mysql.TypeBit:
@@ -1366,7 +1366,8 @@ func ProduceDecWithSpecifiedTp(dec *MyDecimal, tp *FieldType, sc *stmtctx.Statem
 	return dec, err
 }
 
-func (d *Datum) convertToMysqlYear(sc *stmtctx.StatementContext, target *FieldType) (Datum, error) {
+// ConvertToMysqlYear converts a datum to MySQLYear.
+func (d *Datum) ConvertToMysqlYear(sc *stmtctx.StatementContext, target *FieldType) (Datum, error) {
 	var (
 		ret    Datum
 		y      int64
@@ -1411,54 +1412,6 @@ func (d *Datum) convertToMysqlYear(sc *stmtctx.StatementContext, target *FieldTy
 	return ret, errors.Trace(err)
 }
 
-// ConvertDatumToFloatYear converts datum into MySQL year with float type
-func ConvertDatumToFloatYear(sc *stmtctx.StatementContext, d Datum) (Datum, error) {
-	return d.convertToMysqlFloatYear(sc, types.NewFieldType(mysql.TypeYear))
-}
-
-func (d *Datum) convertToMysqlFloatYear(sc *stmtctx.StatementContext, target *FieldType) (Datum, error) {
-	var (
-		ret    Datum
-		y      float64
-		err    error
-		adjust bool
-	)
-	switch d.k {
-	case KindString, KindBytes:
-		s := d.GetString()
-		trimS := strings.TrimSpace(s)
-		y, err = StrToFloat(sc, trimS, false)
-		if err != nil {
-			ret.SetFloat64(0)
-			return ret, errors.Trace(err)
-		}
-		// condition:
-		// parsed to 0, not a string of length 4, the first valid char is a 0 digit
-		if len(s) != 4 && y == 0 && strings.HasPrefix(trimS, "0") {
-			adjust = true
-		}
-	case KindMysqlTime:
-		y = float64(d.GetMysqlTime().Year())
-	case KindMysqlDuration:
-		y = float64(time.Now().Year())
-	case KindNull:
-		// if datum is NULL, we should keep it as it is, instead of setting it to zero or any other value.
-		ret = *d
-		return ret, nil
-	default:
-		ret, err = d.convertToFloat(sc, NewFieldType(mysql.TypeDouble))
-		if err != nil {
-			_, err = invalidConv(d, target.Tp)
-			ret.SetFloat64(0)
-			return ret, err
-		}
-		y = ret.GetFloat64()
-	}
-	y = adjustYearForFloat(y, adjust)
-	ret.SetFloat64(y)
-	return ret, err
-}
-
 func (d *Datum) convertStringToMysqlBit(sc *stmtctx.StatementContext) (uint64, error) {
 	bitStr, err := ParseBitStr(BinaryLiteral(d.b).ToString())
 	if err != nil {

types/time.go

@@ -1260,18 +1260,6 @@ func AdjustYear(y int64, adjustZero bool) (int64, error) {
 	return y, nil
 }
 
-func adjustYearForFloat(y float64, shouldAdjust bool) float64 {
-	if y == 0 && !shouldAdjust {
-		return y
-	}
-	if y >= 0 && y <= 69 {
-		y = 2000 + y
-	} else if y >= 70 && y <= 99 {
-		y = 1900 + y
-	}
-	return y
-}
-
 // NewDuration construct duration with time.
 func NewDuration(hour, minute, second, microsecond int, fsp int8) Duration {
 	return Duration{

util/ranger/points.go

@@ -135,6 +135,13 @@ func getFullRange() []*point {
 	}
 }
 
+func getNotNullFullRange() []*point {
+	return []*point{
+		{value: types.MinNotNullDatum(), start: true},
+		{value: types.MaxValueDatum()},
+	}
+}
+
 // FullIntRange is used for table range. Since table range cannot accept MaxValueDatum as the max value.
 // So we need to set it to MaxInt64.
 func FullIntRange(isUnsigned bool) []*Range {
@@ -221,31 +228,54 @@ func (r *builder) buildFormBinOp(expr *expression.ScalarFunction) []*point {
 		ft    *types.FieldType
 	)
 
-	// refineValue refines the constant datum:
+	// refineValueAndOp refines the constant datum and operator:
 	// 1. for string type since we may eval the constant to another collation instead of its own collation.
 	// 2. for year type since 2-digit year value need adjustment, see https://dev.mysql.com/doc/refman/5.6/en/year.html
-	refineValue := func(col *expression.Column, value *types.Datum) (err error) {
+
+	refineValueAndOp := func(col *expression.Column, value *types.Datum, op *string) (err error) {
 		if col.RetType.EvalType() == types.ETString && (value.Kind() == types.KindString || value.Kind() == types.KindBinaryLiteral) {
 			value.SetString(value.GetString(), col.RetType.Collate)
 		}
 		if col.GetType().Tp == mysql.TypeYear {
-			*value, err = types.ConvertDatumToFloatYear(r.sc, *value)
+			// If the original value is adjusted, we need to change the condition.
+			// For example, col < 2156. Since the max year is 2155, 2156 is changed to 2155.
+			// col < 2155 is wrong. It should be col <= 2155.
+			preValue, err1 := value.ToInt64(r.sc)
+			if err1 != nil {
+				return err1
+			}
+			*value, err = value.ConvertToMysqlYear(r.sc, col.RetType)
+			if errors.ErrorEqual(err, types.ErrInvalidYear) {
+				// Keep err for EQ and NE.
+				switch *op {
+				case ast.GT:
+					if value.GetInt64() > preValue {
+						*op = ast.GE
+					}
+					err = nil
+				case ast.LT:
+					if value.GetInt64() < preValue {
+						*op = ast.LE
+					}
+					err = nil
+				case ast.GE, ast.LE:
+					err = nil
+				}
+			}
 		}
 		return
 	}
-	if col, ok := expr.GetArgs()[0].(*expression.Column); ok {
+	var col *expression.Column
+	var ok bool
+	if col, ok = expr.GetArgs()[0].(*expression.Column); ok {
 		ft = col.RetType
 		value, err = expr.GetArgs()[1].Eval(chunk.Row{})
 		if err != nil {
 			return nil
 		}
-		err = refineValue(col, &value)
-		if err != nil {
-			return nil
-		}
 		op = expr.FuncName.L
 	} else {
-		col, ok := expr.GetArgs()[1].(*expression.Column)
+		col, ok = expr.GetArgs()[1].(*expression.Column)
 		if !ok {
 			return nil
 		}
@@ -254,11 +284,6 @@ func (r *builder) buildFormBinOp(expr *expression.ScalarFunction) []*point {
 		if err != nil {
 			return nil
 		}
-		err = refineValue(col, &value)
-		if err != nil {
-			return nil
-		}
-
 		switch expr.FuncName.L {
 		case ast.GE:
 			op = ast.LE
@@ -275,6 +300,15 @@ func (r *builder) buildFormBinOp(expr *expression.ScalarFunction) []*point {
 	if op != ast.NullEQ && value.IsNull() {
 		return nil
 	}
+	err = refineValueAndOp(col, &value, &op)
+	if err != nil {
+		if op == ast.NE {
+			// col != an impossible value (not valid year)
+			return getNotNullFullRange()
+		}
+		// col = an impossible value (not valid year)
+		return nil
+	}
 
 	value, op, isValidRange := handleUnsignedCol(ft, value, op)
 	if !isValidRange {
@@ -472,10 +506,10 @@ func (r *builder) buildFromIn(expr *expression.ScalarFunction) ([]*point, bool)
 			dt.SetString(dt.GetString(), colCollate)
 		}
 		if expr.GetArgs()[0].GetType().Tp == mysql.TypeYear {
-			dt, err = types.ConvertDatumToFloatYear(r.sc, dt)
+			dt, err = dt.ConvertToMysqlYear(r.sc, expr.GetArgs()[0].GetType())
 			if err != nil {
-				r.err = ErrUnsupportedType.GenWithStack("expr:%v is not converted to year", e)
-				return getFullRange(), hasNull
+				// in (..., an impossible value (not valid year), ...), the range is empty, so skip it.
+				continue
 			}
 		}
 		var startValue, endValue types.Datum

util/ranger/ranger_test.go

@@ -1538,7 +1538,7 @@ func (s *testRangerSuite) TestIndexRangeForYear(c *C) {
 			exprStr:     `a not in (-1, 1, 2)`,
 			accessConds: "[not(in(test.t.a, -1, 1, 2))]",
 			filterConds: "[]",
-			resultStr:   `[(NULL,0) [0,2001) (2002,+inf]]`,
+			resultStr:   `[(NULL,2001) (2002,+inf]]`,
 		},
 		{
 			indexPos:    0,
@@ -1566,7 +1566,7 @@ func (s *testRangerSuite) TestIndexRangeForYear(c *C) {
 			exprStr:     `a not in (1, 2, 15698)`,
 			accessConds: "[not(in(test.t.a, 1, 2, 15698))]",
 			filterConds: "[]",
-			resultStr:   `[(NULL,2001) (2002,2155] (2155,+inf]]`,
+			resultStr:   `[(NULL,2001) (2002,+inf]]`,
 		},
 		{
 			indexPos:    0,
@@ -1594,7 +1594,7 @@ func (s *testRangerSuite) TestIndexRangeForYear(c *C) {
 			exprStr:     `a != 2156`,
 			accessConds: "[ne(test.t.a, 2156)]",
 			filterConds: "[]",
-			resultStr:   `[[-inf,2155] (2155,+inf]]`,
+			resultStr:   `[[-inf,+inf]]`,
 		},
 		{
 			exprStr:     "a < 99 or a > 01",