statistics: fix load stat for new collation column

statistics/handle/dump.go

@@ -300,10 +300,18 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *J
 			}
 			hist := statistics.HistogramFromProto(jsonCol.Histogram)
 			sc := &stmtctx.StatementContext{TimeZone: time.UTC}
+			// Deal with sortKey, the length of sortKey maybe longer than the column's length.
+			orgLen := colInfo.FieldType.Flen
+			if types.IsString(colInfo.FieldType.Tp) {
+				colInfo.Flen = types.UnspecifiedLength
+			}
 			hist, err := hist.ConvertTo(sc, &colInfo.FieldType)
 			if err != nil {
 				return nil, errors.Trace(err)
 			}
+			if types.IsString(colInfo.FieldType.Tp) {
+				colInfo.Flen = orgLen
+			}
 			cm, topN := statistics.CMSketchAndTopNFromProto(jsonCol.CMSketch)
 			fms := statistics.FMSketchFromProto(jsonCol.FMSketch)
 			hist.ID, hist.NullCount, hist.LastUpdateVersion, hist.TotColSize, hist.Correlation = colInfo.ID, jsonCol.NullCount, jsonCol.LastUpdateVersion, jsonCol.TotColSize, jsonCol.Correlation

statistics/handle/dump_test.go

@@ -391,6 +391,59 @@ func TestDumpVer2Stats(t *testing.T) {
 	requireTableEqual(t, statsCacheTbl, loadTbl)
 }
 
+func TestLoadStatsForNewCollation(t *testing.T) {
+	// This test is almost the same as TestDumpVer2Stats, except that: b varchar(10) => b varchar(3) collate utf8mb4_unicode_ci
+	store, dom, clean := testkit.CreateMockStoreAndDomain(t)
+	defer clean()
+	tk := testkit.NewTestKit(t, store)
+	tk.MustExec("set @@tidb_analyze_version = 2")
+	tk.MustExec("use test")
+	tk.MustExec("drop table if exists t")
+	tk.MustExec("create table t(a int, b varchar(3) collate utf8mb4_unicode_ci)")
+	tk.MustExec("insert into t value(1, 'aaa'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc')")
+	// mark column stats as needed
+	tk.MustExec("select * from t where a = 3")
+	tk.MustExec("select * from t where b = 'bbb'")
+	tk.MustExec("alter table t add index single(a)")
+	tk.MustExec("alter table t add index multi(a, b)")
+	tk.MustExec("analyze table t with 2 topn")
+	h := dom.StatsHandle()
+	is := dom.InfoSchema()
+	tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
+	require.NoError(t, err)
+
+	storageTbl, err := h.TableStatsFromStorage(tableInfo.Meta(), tableInfo.Meta().ID, false, 0)
+	require.NoError(t, err)
+
+	dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil)
+	require.NoError(t, err)
+
+	jsonBytes, err := json.MarshalIndent(dumpJSONTable, "", " ")
+	require.NoError(t, err)
+
+	loadJSONTable := &handle.JSONTable{}
+	err = json.Unmarshal(jsonBytes, loadJSONTable)
+	require.NoError(t, err)
+
+	loadTbl, err := handle.TableStatsFromJSON(tableInfo.Meta(), tableInfo.Meta().ID, loadJSONTable)
+	require.NoError(t, err)
+
+	// assert that a statistics.Table from storage dumped into JSON text and then unmarshalled into a statistics.Table keeps unchanged
+	requireTableEqual(t, loadTbl, storageTbl)
+
+	// assert that this statistics.Table is the same as the one in stats cache
+	statsCacheTbl := h.GetTableStats(tableInfo.Meta())
+	requireTableEqual(t, loadTbl, statsCacheTbl)
+
+	err = h.LoadStatsFromJSON(is, loadJSONTable)
+	require.NoError(t, err)
+	require.Nil(t, h.Update(is))
+	statsCacheTbl = h.GetTableStats(tableInfo.Meta())
+	// assert that after the JSONTable above loaded into storage then updated into the stats cache,
+	// the statistics.Table in the stats cache is the same as the unmarshalled statistics.Table
+	requireTableEqual(t, statsCacheTbl, loadTbl)
+}
+
 func TestJSONTableToBlocks(t *testing.T) {
 	store, dom, clean := testkit.CreateMockStoreAndDomain(t)
 	defer clean()