From 289dcfefdd159345972f71cf42c9d6e594255290 Mon Sep 17 00:00:00 2001
From: Shenghui Wu <793703860@qq.com>
Date: Thu, 6 May 2021 23:05:52 +0800
Subject: [PATCH] util: fix enum index range for in/not in clause. (#24431)

---
 expression/integration_test.go | 17 +++++++++++++++++
 util/ranger/points.go          | 18 ++++++++++++++++++
 2 files changed, 35 insertions(+)

diff --git a/expression/integration_test.go b/expression/integration_test.go
index 587b8903fb075..76ba37b49d4bd 100644
--- a/expression/integration_test.go
+++ b/expression/integration_test.go
@@ -9318,4 +9318,21 @@ func (s *testIntegrationSuite) TestEnumIndex(c *C) {
 		result := tk.MustQuery("select * from t where " + cond).Sort().Rows()
 		tk.MustQuery("select * from tidx where " + cond).Sort().Check(result)
 	}
+
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(e enum('d','c','b','a'), a int, index idx(e));")
+	tk.MustExec("insert into t values(1,1),(2,2),(3,3),(4,4);")
+	tk.MustQuery("select /*+ use_index(t, idx) */ * from t where e not in ('a','d') and a = 2;").Check(
+		testkit.Rows("c 2"))
+
+	// issue 24419
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t02")
+	tk.MustExec("CREATE TABLE `t02` (  `COL1` enum('^YSQT0]V@9TFN>^WB6G?NG@S8>VYOM;BSC@<BCQ6','TKZQQ=C1@IH9W>64=ZISGS?O[JDFBI5M]QXJYQNSKU>NGAWLXS26LMTZ2YNN`XKIUGKY0IHDWV>E[BJJCABOKH1M^CB5E@DLS7Q88PWZTEAY]1ZQMN5NX[I<KBBK','PXWTHJ?R]P=`Y','OFJHCEKCQGT:MXI7P3[YO4N0DF=2XJWJ4Z9Z;HQ8TMUTZV8YLQAHWJ4BDZHR3A','@[ETQPEKKDD;9INXAQISU0O65J86AWQ2SZ8=ZZW6TKT4GCF_O13^ZQW_S>FIYA983K:E4N77@FINM5HVGQCUCVNF5WLOOOEORAM=_JLMVFURMUASTVDBE','NL3V:J9LM4U5KUCV<RIJ_RKMZ4;CXD_0:K`HCO=P1YNYTHX8KYZRQ?PL01HLNSUC_R7:I5<V[HV0BIDEBZAPT73R7`DP43XXPLQCEI8>R;P','M5=T5FLQEZMPZAXH]4G:TSYYYVQ7O@4S6C3N8WPFKSP;SRD6VW@94BBH8XCT','P]I52Y46F?@RMOOF6;FWDTO`7FIT]R:]ELHD[CNLDSHC7FPBYOOJXLZSBV^5C^AAF6J5BCKE4V9==@H=4C]GMZXPNM','ECIQWH>?MK=ARGI0WVJNIBZFCFVJHFIUYJ:2?2WWZBNBWTPFNQPLLBFP9R_','E<<T9UUF2?XM8TWS_','W[5E_U1J?YSOQISL1KD','M@V^`^8I','5UTEJUZIQ^ZJOJU_D6@V2DSVOIK@LUT^E?RTL>_Y9OT@SOPYR72VIJVMBWIVPF@TTBZ@8ZPBZL=LXZF`WM4V2?K>AT','PZ@PR6XN28JL`B','ZOHBSCRMZPOI`IVTSEZAIDAF7DS@1TT20AP9','QLDIOY[Y:JZR@OL__I^@FBO=O_?WOOR:2BE:QJC','BI^TGJ_N<H:7OW8XXITM@FBWDNJ=KA`X:9@BUY4UHKSHFP`EAWR9_QS^HR2AI39MGVXWVD]RUI46SHU=GXAX;RT765X:CU7M4XOD^S9JFZI=HTTS?C0CT','M@HGGFM43C7','@M`IHSJQ8HBTGOS`=VW]QBMLVWN`SP;E>EEXYKV1POHTOJQPGCPVR=TYZMGWABUQR07J8U::W4','N`ZN4P@9T[JW;FR6=FA4WP@APNPG[XQVIK4]F]2>EC>JEIOXC``;;?OHP') DEFAULT NULL,  `COL2` tinyint DEFAULT NULL,  `COL3` time DEFAULT NULL,  KEY `U_M_COL4` (`COL1`,`COL2`),  KEY `U_M_COL5` (`COL3`,`COL2`)) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4 COLLATE=utf8mb4_bin;")
+	tk.MustExec("insert into t02(col1, col2) values ('OFJHCEKCQGT:MXI7P3[YO4N0DF=2XJWJ4Z9Z;HQ8TMUTZV8YLQAHWJ4BDZHR3A', 39), ('OFJHCEKCQGT:MXI7P3[YO4N0DF=2XJWJ4Z9Z;HQ8TMUTZV8YLQAHWJ4BDZHR3A', 51), ('OFJHCEKCQGT:MXI7P3[YO4N0DF=2XJWJ4Z9Z;HQ8TMUTZV8YLQAHWJ4BDZHR3A', 55), ('OFJHCEKCQGT:MXI7P3[YO4N0DF=2XJWJ4Z9Z;HQ8TMUTZV8YLQAHWJ4BDZHR3A', -30), ('ZOHBSCRMZPOI`IVTSEZAIDAF7DS@1TT20AP9', -30);")
+	tk.MustQuery("select * from t02 where col1 not in (\"W1Rgd74pbJaGX47h1MPjpr0XSKJNCnwEleJ50Vbpl9EmbHJX6D6BXYKT2UAbl1uDw3ZGeYykhzG6Gld0wKdOiT4Gv5j9upHI0Q7vrXij4N9WNFJvB\", \"N`ZN4P@9T[JW;FR6=FA4WP@APNPG[XQVIK4]F]2>EC>JEIOXC``;;?OHP\") and col2 = -30;").Check(
+		testkit.Rows(
+			"OFJHCEKCQGT:MXI7P3[YO4N0DF=2XJWJ4Z9Z;HQ8TMUTZV8YLQAHWJ4BDZHR3A -30 <nil>",
+			"ZOHBSCRMZPOI`IVTSEZAIDAF7DS@1TT20AP9 -30 <nil>"))
 }
diff --git a/util/ranger/points.go b/util/ranger/points.go
index 19d7c18d9ecf7..d98b548dcbb7a 100644
--- a/util/ranger/points.go
+++ b/util/ranger/points.go
@@ -102,6 +102,9 @@ func (r *pointSorter) Less(i, j int) bool {
 }
 
 func rangePointLess(sc *stmtctx.StatementContext, a, b *point) (bool, error) {
+	if a.value.Kind() == types.KindMysqlEnum && b.value.Kind() == types.KindMysqlEnum {
+		return rangePointEnumLess(sc, a, b)
+	}
 	cmp, err := a.value.CompareDatum(sc, &b.value)
 	if cmp != 0 {
 		return cmp < 0, nil
@@ -109,6 +112,14 @@ func rangePointLess(sc *stmtctx.StatementContext, a, b *point) (bool, error) {
 	return rangePointEqualValueLess(a, b), errors.Trace(err)
 }
 
+func rangePointEnumLess(sc *stmtctx.StatementContext, a, b *point) (bool, error) {
+	cmp := types.CompareInt64(a.value.GetInt64(), b.value.GetInt64())
+	if cmp != 0 {
+		return cmp < 0, nil
+	}
+	return rangePointEqualValueLess(a, b), nil
+}
+
 func rangePointEqualValueLess(a, b *point) bool {
 	if a.start && b.start {
 		return !a.excl && b.excl
@@ -552,6 +563,13 @@ func (r *builder) buildFromIn(expr *expression.ScalarFunction) ([]*point, bool)
 		if dt.Kind() == types.KindString || dt.Kind() == types.KindBinaryLiteral {
 			dt.SetString(dt.GetString(), colCollate)
 		}
+		if expr.GetArgs()[0].GetType().Tp == mysql.TypeEnum {
+			dt, err = dt.ConvertTo(r.sc, expr.GetArgs()[0].GetType())
+			if err != nil {
+				// in (..., an impossible value (not valid enum), ...), the range is empty, so skip it.
+				continue
+			}
+		}
 		if expr.GetArgs()[0].GetType().Tp == mysql.TypeYear {
 			dt, err = dt.ConvertToMysqlYear(r.sc, expr.GetArgs()[0].GetType())
 			if err != nil {