Skip to content

Commit

Permalink
cherry pick pingcap#33146 to release-4.0
Browse files Browse the repository at this point in the history 
Signed-off-by: ti-srebot <[email protected]>
  • Loading branch information
@wjhuang2016 @ti-srebot
wjhuang2016 authored and ti-srebot committed Mar 16, 2022
1 parent 75f81d2 commit 18e9f1b
Show file tree
Hide file tree
Showing 2 changed files with 194 additions and 0 deletions.
13 changes: 13 additions & 0 deletions statistics/handle/dump.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -219,10 +219,23 @@ func TableStatsFromJSON(tableInfo *model.TableInfo, physicalID int64, jsonTbl *J
hist := statistics.HistogramFromProto(jsonCol.Histogram)
count := int64(hist.TotalRowCount())
sc := &stmtctx.StatementContext{TimeZone: time.UTC}
// Deal with sortKey, the length of sortKey maybe longer than the column's length.
orgLen := colInfo.FieldType.Flen
if types.IsString(colInfo.FieldType.Tp) {
colInfo.Flen = types.UnspecifiedLength
}
hist, err := hist.ConvertTo(sc, &colInfo.FieldType)
if err != nil {
return nil, errors.Trace(err)
}
<<<<<<< HEAD
=======
if types.IsString(colInfo.FieldType.Tp) {
colInfo.Flen = orgLen
}
cm, topN := statistics.CMSketchAndTopNFromProto(jsonCol.CMSketch)
fms := statistics.FMSketchFromProto(jsonCol.FMSketch)
>>>>>>> 128dc1682... statistics: fix load stat for new collation column (#33146)
hist.ID, hist.NullCount, hist.LastUpdateVersion, hist.TotColSize, hist.Correlation = colInfo.ID, jsonCol.NullCount, jsonCol.LastUpdateVersion, jsonCol.TotColSize, jsonCol.Correlation
col := &statistics.Column{
PhysicalID: physicalID,
Expand Down
181 changes: 181 additions & 0 deletions statistics/handle/dump_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -182,5 +182,186 @@ func (s *testStatsSuite) TestDumpPseudoColumns(c *C) {
c.Assert(err, IsNil)
h := s.do.StatsHandle()
_, err = h.DumpStatsToJSON("test", tbl.Meta(), nil)
<<<<<<< HEAD
c.Assert(err, IsNil)
=======
require.NoError(t, err)
}

func TestDumpExtendedStats(t *testing.T) {
store, dom, clean := testkit.CreateMockStoreAndDomain(t)
defer clean()
tk := testkit.NewTestKit(t, store)
tk.MustExec("set session tidb_enable_extended_stats = on")
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int, b int)")
tk.MustExec("insert into t values(1,5),(2,4),(3,3),(4,2),(5,1)")
h := dom.StatsHandle()
require.Nil(t, h.DumpStatsDeltaToKV(handle.DumpAll))
tk.MustExec("alter table t add stats_extended s1 correlation(a,b)")
tk.MustExec("analyze table t")

is := dom.InfoSchema()
tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)
tbl := h.GetTableStats(tableInfo.Meta())
jsonTbl, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil)
require.NoError(t, err)
loadTbl, err := handle.TableStatsFromJSON(tableInfo.Meta(), tableInfo.Meta().ID, jsonTbl)
require.NoError(t, err)
requireTableEqual(t, loadTbl, tbl)

cleanStats(tk, dom)
wg := util.WaitGroupWrapper{}
wg.Run(func() {
require.Nil(t, h.Update(is))
})
err = h.LoadStatsFromJSON(is, jsonTbl)
wg.Wait()
require.NoError(t, err)
loadTblInStorage := h.GetTableStats(tableInfo.Meta())
requireTableEqual(t, loadTblInStorage, tbl)
}

func TestDumpVer2Stats(t *testing.T) {
store, dom, clean := testkit.CreateMockStoreAndDomain(t)
defer clean()
tk := testkit.NewTestKit(t, store)
tk.MustExec("set @@tidb_analyze_version = 2")
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int, b varchar(10))")
tk.MustExec("insert into t value(1, 'aaa'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc')")
// mark column stats as needed
tk.MustExec("select * from t where a = 3")
tk.MustExec("select * from t where b = 'bbb'")
tk.MustExec("alter table t add index single(a)")
tk.MustExec("alter table t add index multi(a, b)")
tk.MustExec("analyze table t with 2 topn")
h := dom.StatsHandle()
is := dom.InfoSchema()
tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)

storageTbl, err := h.TableStatsFromStorage(tableInfo.Meta(), tableInfo.Meta().ID, false, 0)
require.NoError(t, err)

dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil)
require.NoError(t, err)

jsonBytes, err := json.MarshalIndent(dumpJSONTable, "", " ")
require.NoError(t, err)

loadJSONTable := &handle.JSONTable{}
err = json.Unmarshal(jsonBytes, loadJSONTable)
require.NoError(t, err)

loadTbl, err := handle.TableStatsFromJSON(tableInfo.Meta(), tableInfo.Meta().ID, loadJSONTable)
require.NoError(t, err)

// assert that a statistics.Table from storage dumped into JSON text and then unmarshalled into a statistics.Table keeps unchanged
requireTableEqual(t, loadTbl, storageTbl)

// assert that this statistics.Table is the same as the one in stats cache
statsCacheTbl := h.GetTableStats(tableInfo.Meta())
requireTableEqual(t, loadTbl, statsCacheTbl)

err = h.LoadStatsFromJSON(is, loadJSONTable)
require.NoError(t, err)
require.Nil(t, h.Update(is))
statsCacheTbl = h.GetTableStats(tableInfo.Meta())
// assert that after the JSONTable above loaded into storage then updated into the stats cache,
// the statistics.Table in the stats cache is the same as the unmarshalled statistics.Table
requireTableEqual(t, statsCacheTbl, loadTbl)
}

func TestLoadStatsForNewCollation(t *testing.T) {
// This test is almost the same as TestDumpVer2Stats, except that: b varchar(10) => b varchar(3) collate utf8mb4_unicode_ci
store, dom, clean := testkit.CreateMockStoreAndDomain(t)
defer clean()
tk := testkit.NewTestKit(t, store)
tk.MustExec("set @@tidb_analyze_version = 2")
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int, b varchar(3) collate utf8mb4_unicode_ci)")
tk.MustExec("insert into t value(1, 'aaa'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc')")
// mark column stats as needed
tk.MustExec("select * from t where a = 3")
tk.MustExec("select * from t where b = 'bbb'")
tk.MustExec("alter table t add index single(a)")
tk.MustExec("alter table t add index multi(a, b)")
tk.MustExec("analyze table t with 2 topn")
h := dom.StatsHandle()
is := dom.InfoSchema()
tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)

storageTbl, err := h.TableStatsFromStorage(tableInfo.Meta(), tableInfo.Meta().ID, false, 0)
require.NoError(t, err)

dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil)
require.NoError(t, err)

jsonBytes, err := json.MarshalIndent(dumpJSONTable, "", " ")
require.NoError(t, err)

loadJSONTable := &handle.JSONTable{}
err = json.Unmarshal(jsonBytes, loadJSONTable)
require.NoError(t, err)

loadTbl, err := handle.TableStatsFromJSON(tableInfo.Meta(), tableInfo.Meta().ID, loadJSONTable)
require.NoError(t, err)

// assert that a statistics.Table from storage dumped into JSON text and then unmarshalled into a statistics.Table keeps unchanged
requireTableEqual(t, loadTbl, storageTbl)

// assert that this statistics.Table is the same as the one in stats cache
statsCacheTbl := h.GetTableStats(tableInfo.Meta())
requireTableEqual(t, loadTbl, statsCacheTbl)

err = h.LoadStatsFromJSON(is, loadJSONTable)
require.NoError(t, err)
require.Nil(t, h.Update(is))
statsCacheTbl = h.GetTableStats(tableInfo.Meta())
// assert that after the JSONTable above loaded into storage then updated into the stats cache,
// the statistics.Table in the stats cache is the same as the unmarshalled statistics.Table
requireTableEqual(t, statsCacheTbl, loadTbl)
}

func TestJSONTableToBlocks(t *testing.T) {
store, dom, clean := testkit.CreateMockStoreAndDomain(t)
defer clean()
tk := testkit.NewTestKit(t, store)
tk.MustExec("set @@tidb_analyze_version = 2")
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
tk.MustExec("create table t(a int, b varchar(10))")
tk.MustExec("insert into t value(1, 'aaa'), (3, 'aab'), (5, 'bba'), (2, 'bbb'), (4, 'cca'), (6, 'ccc')")
// mark column stats as needed
tk.MustExec("select * from t where a = 3")
tk.MustExec("select * from t where b = 'bbb'")
tk.MustExec("alter table t add index single(a)")
tk.MustExec("alter table t add index multi(a, b)")
tk.MustExec("analyze table t with 2 topn")
h := dom.StatsHandle()
is := dom.InfoSchema()
tableInfo, err := is.TableByName(model.NewCIStr("test"), model.NewCIStr("t"))
require.NoError(t, err)

dumpJSONTable, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil)
require.NoError(t, err)
jsOrigin, _ := json.Marshal(dumpJSONTable)

blockSize := 30
js, err := h.DumpStatsToJSON("test", tableInfo.Meta(), nil)
require.NoError(t, err)
dumpJSONBlocks, err := handle.JSONTableToBlocks(js, blockSize)
require.NoError(t, err)
jsConverted, err := handle.BlocksToJSONTable(dumpJSONBlocks)
require.NoError(t, err)
jsonStr, err := json.Marshal(jsConverted)
require.NoError(t, err)
require.JSONEq(t, string(jsOrigin), string(jsonStr))
>>>>>>> 128dc1682... statistics: fix load stat for new collation column (#33146)
}

0 comments on commit 18e9f1b

Please sign in to comment.