cherry pick pingcap#33146 to release-5.3

Signed-off-by: ti-srebot <[email protected]>
ti-srebot · Mar 16, 2022 · d58b2af · d58b2af
1 parent 459917c
commit d58b2af
Show file tree

Hide file tree

Showing 2 changed files with 101 additions and 0 deletions.
diff --git a/statistics/handle/dump.go b/statistics/handle/dump.go
@@ -296,10 +296,18 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *J
 			}
 			hist := statistics.HistogramFromProto(jsonCol.Histogram)
 			sc := &stmtctx.StatementContext{TimeZone: time.UTC}
+			// Deal with sortKey, the length of sortKey maybe longer than the column's length.
+			orgLen := colInfo.FieldType.Flen
+			if types.IsString(colInfo.FieldType.Tp) {
+				colInfo.Flen = types.UnspecifiedLength
+			}
 			hist, err := hist.ConvertTo(sc, &colInfo.FieldType)
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
+			if types.IsString(colInfo.FieldType.Tp) {
+				colInfo.Flen = orgLen
+			}
 			cm, topN := statistics.CMSketchAndTopNFromProto(jsonCol.CMSketch)
 			fms := statistics.FMSketchFromProto(jsonCol.FMSketch)
 			hist.ID, hist.NullCount, hist.LastUpdateVersion, hist.TotColSize, hist.Correlation = colInfo.ID, jsonCol.NullCount, jsonCol.LastUpdateVersion, jsonCol.TotColSize, jsonCol.Correlation

diff --git a/statistics/handle/dump_test.go b/statistics/handle/dump_test.go
@@ -336,5 +336,98 @@ func (s *testStatsSuite) TestDumpVer2Stats(c *C) {
 	statsCacheTbl = h.GetTableStats(tableInfo.Meta())
 	// assert that after the JSONTable above loaded into storage then updated into the stats cache,
 	// the statistics.Table in the stats cache is the same as the unmarshalled statistics.Table
+<<<<<<< HEAD
 	assertTableEqual(c, statsCacheTbl, loadTbl)
+=======
+	requireTableEqual(t, statsCacheTbl, loadTbl)
+}
+
+func TestLoadStatsForNewCollation(t *testing.T) {
+	// This test is almost the same as TestDumpVer2Stats, except that: b varchar(10) => b varchar(3) collate utf8mb4_unicode_ci
+	store, dom, clean := testkit.CreateMockStoreAndDomain(t)
+	defer clean()
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("set @@tidb_analyze_version = 2")
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b varchar(3) collate utf8mb4_unicode_ci)")
+	tk.MustExec("insert into t value(1, 'aaa'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc')")
+	// mark column stats as needed
+	tk.MustExec("select * from t where a = 3")
+	tk.MustExec("select * from t where b = 'bbb'")
+	tk.MustExec("alter table t add index single(a)")
+	tk.MustExec("alter table t add index multi(a, b)")
+	tk.MustExec("analyze table t with 2 topn")
+	h := dom.StatsHandle()
+	is := dom.InfoSchema()
+	tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	require.NoError(t, err)
+
+	storageTbl, err := h.TableStatsFromStorage(tableInfo.Meta(), tableInfo.Meta().ID, false, 0)
+	require.NoError(t, err)
+
+	dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil)
+	require.NoError(t, err)
+
+	jsonBytes, err := json.MarshalIndent(dumpJSONTable, "", " ")
+	require.NoError(t, err)
+
+	loadJSONTable := &handle.JSONTable{}
+	err = json.Unmarshal(jsonBytes, loadJSONTable)
+	require.NoError(t, err)
+
+	loadTbl, err := handle.TableStatsFromJSON(tableInfo.Meta(), tableInfo.Meta().ID, loadJSONTable)
+	require.NoError(t, err)
+
+	// assert that a statistics.Table from storage dumped into JSON text and then unmarshalled into a statistics.Table keeps unchanged
+	requireTableEqual(t, loadTbl, storageTbl)
+
+	// assert that this statistics.Table is the same as the one in stats cache
+	statsCacheTbl := h.GetTableStats(tableInfo.Meta())
+	requireTableEqual(t, loadTbl, statsCacheTbl)
+
+	err = h.LoadStatsFromJSON(is, loadJSONTable)
+	require.NoError(t, err)
+	require.Nil(t, h.Update(is))
+	statsCacheTbl = h.GetTableStats(tableInfo.Meta())
+	// assert that after the JSONTable above loaded into storage then updated into the stats cache,
+	// the statistics.Table in the stats cache is the same as the unmarshalled statistics.Table
+	requireTableEqual(t, statsCacheTbl, loadTbl)
+}
+
+func TestJSONTableToBlocks(t *testing.T) {
+	store, dom, clean := testkit.CreateMockStoreAndDomain(t)
+	defer clean()
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("set @@tidb_analyze_version = 2")
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b varchar(10))")
+	tk.MustExec("insert into t value(1, 'aaa'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc')")
+	// mark column stats as needed
+	tk.MustExec("select * from t where a = 3")
+	tk.MustExec("select * from t where b = 'bbb'")
+	tk.MustExec("alter table t add index single(a)")
+	tk.MustExec("alter table t add index multi(a, b)")
+	tk.MustExec("analyze table t with 2 topn")
+	h := dom.StatsHandle()
+	is := dom.InfoSchema()
+	tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	require.NoError(t, err)
+
+	dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil)
+	require.NoError(t, err)
+	jsOrigin, _ := json.Marshal(dumpJSONTable)
+
+	blockSize := 30
+	js, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil)
+	require.NoError(t, err)
+	dumpJSONBlocks, err := handle.JSONTableToBlocks(js, blockSize)
+	require.NoError(t, err)
+	jsConverted, err := handle.BlocksToJSONTable(dumpJSONBlocks)
+	require.NoError(t, err)
+	jsonStr, err := json.Marshal(jsConverted)
+	require.NoError(t, err)
+	require.JSONEq(t, string(jsOrigin), string(jsonStr))
+>>>>>>> 128dc1682... statistics: fix load stat for new collation column (#33146)
 }