From d58b2af70d302da4a6fbebfad8f37d687b5ad88c Mon Sep 17 00:00:00 2001 From: wjHuang Date: Wed, 16 Mar 2022 17:29:53 +0800 Subject: [PATCH] cherry pick #33146 to release-5.3 Signed-off-by: ti-srebot --- statistics/handle/dump.go | 8 +++ statistics/handle/dump_test.go | 93 ++++++++++++++++++++++++++++++++++ 2 files changed, 101 insertions(+) diff --git a/statistics/handle/dump.go b/statistics/handle/dump.go index 4628c18ac1c3a..3b2de7b992d56 100644 --- a/statistics/handle/dump.go +++ b/statistics/handle/dump.go @@ -296,10 +296,18 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *J } hist := statistics.HistogramFromProto(jsonCol.Histogram) sc := &stmtctx.StatementContext{TimeZone: time.UTC} + // Deal with sortKey, the length of sortKey maybe longer than the column's length. + orgLen := colInfo.FieldType.Flen + if types.IsString(colInfo.FieldType.Tp) { + colInfo.Flen = types.UnspecifiedLength + } hist, err := hist.ConvertTo(sc, &colInfo.FieldType) if err != nil { return nil, errors.Trace(err) } + if types.IsString(colInfo.FieldType.Tp) { + colInfo.Flen = orgLen + } cm, topN := statistics.CMSketchAndTopNFromProto(jsonCol.CMSketch) fms := statistics.FMSketchFromProto(jsonCol.FMSketch) hist.ID, hist.NullCount, hist.LastUpdateVersion, hist.TotColSize, hist.Correlation = colInfo.ID, jsonCol.NullCount, jsonCol.LastUpdateVersion, jsonCol.TotColSize, jsonCol.Correlation diff --git a/statistics/handle/dump_test.go b/statistics/handle/dump_test.go index 2ce75f722d2d0..37481addc722b 100644 --- a/statistics/handle/dump_test.go +++ b/statistics/handle/dump_test.go @@ -336,5 +336,98 @@ func (s *testStatsSuite) TestDumpVer2Stats(c *C) { statsCacheTbl = h.GetTableStats(tableInfo.Meta()) // assert that after the JSONTable above loaded into storage then updated into the stats cache, // the statistics.Table in the stats cache is the same as the unmarshalled statistics.Table +<<<<<<< HEAD assertTableEqual(c, statsCacheTbl, loadTbl) +======= + requireTableEqual(t, statsCacheTbl, loadTbl) +} + +func TestLoadStatsForNewCollation(t *testing.T) { + // This test is almost the same as TestDumpVer2Stats, except that: b varchar(10) => b varchar(3) collate utf8mb4_unicode_ci + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@tidb_analyze_version = 2") + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b varchar(3) collate utf8mb4_unicode_ci)") + tk.MustExec("insert into t value(1, 'aaa'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc')") + // mark column stats as needed + tk.MustExec("select * from t where a = 3") + tk.MustExec("select * from t where b = 'bbb'") + tk.MustExec("alter table t add index single(a)") + tk.MustExec("alter table t add index multi(a, b)") + tk.MustExec("analyze table t with 2 topn") + h := dom.StatsHandle() + is := dom.InfoSchema() + tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + + storageTbl, err := h.TableStatsFromStorage(tableInfo.Meta(), tableInfo.Meta().ID, false, 0) + require.NoError(t, err) + + dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil) + require.NoError(t, err) + + jsonBytes, err := json.MarshalIndent(dumpJSONTable, "", " ") + require.NoError(t, err) + + loadJSONTable := &handle.JSONTable{} + err = json.Unmarshal(jsonBytes, loadJSONTable) + require.NoError(t, err) + + loadTbl, err := handle.TableStatsFromJSON(tableInfo.Meta(), tableInfo.Meta().ID, loadJSONTable) + require.NoError(t, err) + + // assert that a statistics.Table from storage dumped into JSON text and then unmarshalled into a statistics.Table keeps unchanged + requireTableEqual(t, loadTbl, storageTbl) + + // assert that this statistics.Table is the same as the one in stats cache + statsCacheTbl := h.GetTableStats(tableInfo.Meta()) + requireTableEqual(t, loadTbl, statsCacheTbl) + + err = h.LoadStatsFromJSON(is, loadJSONTable) + require.NoError(t, err) + require.Nil(t, h.Update(is)) + statsCacheTbl = h.GetTableStats(tableInfo.Meta()) + // assert that after the JSONTable above loaded into storage then updated into the stats cache, + // the statistics.Table in the stats cache is the same as the unmarshalled statistics.Table + requireTableEqual(t, statsCacheTbl, loadTbl) +} + +func TestJSONTableToBlocks(t *testing.T) { + store, dom, clean := testkit.CreateMockStoreAndDomain(t) + defer clean() + tk := testkit.NewTestKit(t, store) + tk.MustExec("set @@tidb_analyze_version = 2") + tk.MustExec("use test") + tk.MustExec("drop table if exists t") + tk.MustExec("create table t(a int, b varchar(10))") + tk.MustExec("insert into t value(1, 'aaa'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc')") + // mark column stats as needed + tk.MustExec("select * from t where a = 3") + tk.MustExec("select * from t where b = 'bbb'") + tk.MustExec("alter table t add index single(a)") + tk.MustExec("alter table t add index multi(a, b)") + tk.MustExec("analyze table t with 2 topn") + h := dom.StatsHandle() + is := dom.InfoSchema() + tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t")) + require.NoError(t, err) + + dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil) + require.NoError(t, err) + jsOrigin, _ := json.Marshal(dumpJSONTable) + + blockSize := 30 + js, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil) + require.NoError(t, err) + dumpJSONBlocks, err := handle.JSONTableToBlocks(js, blockSize) + require.NoError(t, err) + jsConverted, err := handle.BlocksToJSONTable(dumpJSONBlocks) + require.NoError(t, err) + jsonStr, err := json.Marshal(jsConverted) + require.NoError(t, err) + require.JSONEq(t, string(jsOrigin), string(jsonStr)) +>>>>>>> 128dc1682... statistics: fix load stat for new collation column (#33146) }