Skip to content

Commit

Permalink
stats: support show stats for partition table (#8023)
Browse files Browse the repository at this point in the history
  • Loading branch information
@alivxxx
alivxxx authored Oct 24, 2018
1 parent b87b5f0 commit cc1da16
Show file tree
Hide file tree
Showing 4 changed files with 143 additions and 63 deletions.
135 changes: 92 additions & 43 deletions executor/show_stats.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -30,45 +30,69 @@ func (e *ShowExec) fetchShowStatsMeta() error {
dbs := do.InfoSchema().AllSchemas()
for _, db := range dbs {
for _, tbl := range db.Tables {
statsTbl := h.GetTableStats(tbl)
if !statsTbl.Pseudo {
e.appendRow([]interface{}{
db.Name.O,
tbl.Name.O,
e.versionToTime(statsTbl.Version),
statsTbl.ModifyCount,
statsTbl.Count,
})
pi := tbl.GetPartitionInfo()
if pi == nil {
e.appendTableForStatsMeta(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl))
} else {
for _, def := range pi.Definitions {
e.appendTableForStatsMeta(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID))
}
}
}
}
return nil
}

func (e *ShowExec) appendTableForStatsMeta(dbName, tblName, partitionName string, statsTbl *statistics.Table) {
if statsTbl.Pseudo {
return
}
e.appendRow([]interface{}{
dbName,
tblName,
partitionName,
e.versionToTime(statsTbl.Version),
statsTbl.ModifyCount,
statsTbl.Count,
})
}

func (e *ShowExec) fetchShowStatsHistogram() error {
do := domain.GetDomain(e.ctx)
h := do.StatsHandle()
dbs := do.InfoSchema().AllSchemas()
for _, db := range dbs {
for _, tbl := range db.Tables {
statsTbl := h.GetTableStats(tbl)
if !statsTbl.Pseudo {
for _, col := range statsTbl.Columns {
e.histogramToRow(db.Name.O, tbl.Name.O, col.Info.Name.O, 0, col.Histogram, col.AvgColSize(statsTbl.Count))
}
for _, idx := range statsTbl.Indices {
e.histogramToRow(db.Name.O, tbl.Name.O, idx.Info.Name.O, 1, idx.Histogram, 0)
pi := tbl.GetPartitionInfo()
if pi == nil {
e.appendTableForStatsHistograms(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl))
} else {
for _, def := range pi.Definitions {
e.appendTableForStatsHistograms(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID))
}
}
}
}
return nil
}

func (e *ShowExec) histogramToRow(dbName string, tblName string, colName string, isIndex int, hist statistics.Histogram, avgColSize float64) {
func (e *ShowExec) appendTableForStatsHistograms(dbName, tblName, partitionName string, statsTbl *statistics.Table) {
if statsTbl.Pseudo {
return
}
for _, col := range statsTbl.Columns {
e.histogramToRow(dbName, tblName, partitionName, col.Info.Name.O, 0, col.Histogram, col.AvgColSize(statsTbl.Count))
}
for _, idx := range statsTbl.Indices {
e.histogramToRow(dbName, tblName, partitionName, idx.Info.Name.O, 1, idx.Histogram, 0)
}
}

func (e *ShowExec) histogramToRow(dbName, tblName, partitionName, colName string, isIndex int, hist statistics.Histogram, avgColSize float64) {
e.appendRow([]interface{}{
dbName,
tblName,
partitionName,
colName,
isIndex,
e.versionToTime(hist.LastUpdateVersion),
Expand All @@ -89,29 +113,41 @@ func (e *ShowExec) fetchShowStatsBuckets() error {
dbs := do.InfoSchema().AllSchemas()
for _, db := range dbs {
for _, tbl := range db.Tables {
statsTbl := h.GetTableStats(tbl)
if !statsTbl.Pseudo {
for _, col := range statsTbl.Columns {
err := e.bucketsToRows(db.Name.O, tbl.Name.O, col.Info.Name.O, 0, col.Histogram)
if err != nil {
return errors.Trace(err)
}
}
for _, idx := range statsTbl.Indices {
err := e.bucketsToRows(db.Name.O, tbl.Name.O, idx.Info.Name.O, len(idx.Info.Columns), idx.Histogram)
if err != nil {
return errors.Trace(err)
}
pi := tbl.GetPartitionInfo()
if pi == nil {
e.appendTableForStatsBuckets(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl))
} else {
for _, def := range pi.Definitions {
e.appendTableForStatsBuckets(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID))
}
}
}
}
return nil
}

func (e *ShowExec) appendTableForStatsBuckets(dbName, tblName, partitionName string, statsTbl *statistics.Table) error {
if statsTbl.Pseudo {
return nil
}
for _, col := range statsTbl.Columns {
err := e.bucketsToRows(dbName, tblName, partitionName, col.Info.Name.O, 0, col.Histogram)
if err != nil {
return errors.Trace(err)
}
}
for _, idx := range statsTbl.Indices {
err := e.bucketsToRows(dbName, tblName, partitionName, idx.Info.Name.O, len(idx.Info.Columns), idx.Histogram)
if err != nil {
return errors.Trace(err)
}
}
return nil
}

// bucketsToRows converts histogram buckets to rows. If the histogram is built from index, then numOfCols equals to number
// of index columns, else numOfCols is 0.
func (e *ShowExec) bucketsToRows(dbName, tblName, colName string, numOfCols int, hist statistics.Histogram) error {
func (e *ShowExec) bucketsToRows(dbName, tblName, partitionName, colName string, numOfCols int, hist statistics.Histogram) error {
isIndex := 0
if numOfCols > 0 {
isIndex = 1
Expand All @@ -128,6 +164,7 @@ func (e *ShowExec) bucketsToRows(dbName, tblName, colName string, numOfCols int,
e.appendRow([]interface{}{
dbName,
tblName,
partitionName,
colName,
isIndex,
i,
Expand All @@ -146,20 +183,32 @@ func (e *ShowExec) fetchShowStatsHealthy() {
dbs := do.InfoSchema().AllSchemas()
for _, db := range dbs {
for _, tbl := range db.Tables {
statsTbl := h.GetTableStats(tbl)
if !statsTbl.Pseudo {
var healthy int64
if statsTbl.ModifyCount < statsTbl.Count {
healthy = int64((1.0 - float64(statsTbl.ModifyCount)/float64(statsTbl.Count)) * 100.0)
} else if statsTbl.ModifyCount == 0 {
healthy = 100
pi := tbl.GetPartitionInfo()
if pi == nil {
e.appendTableForStatsHealthy(db.Name.O, tbl.Name.O, "", h.GetTableStats(tbl))
} else {
for _, def := range pi.Definitions {
e.appendTableForStatsHealthy(db.Name.O, tbl.Name.O, def.Name.O, h.GetPartitionStats(tbl, def.ID))
}
e.appendRow([]interface{}{
db.Name.O,
tbl.Name.O,
healthy,
})
}
}
}
}

func (e *ShowExec) appendTableForStatsHealthy(dbName, tblName, partitionName string, statsTbl *statistics.Table) {
if statsTbl.Pseudo {
return
}
var healthy int64
if statsTbl.ModifyCount < statsTbl.Count {
healthy = int64((1.0 - float64(statsTbl.ModifyCount)/float64(statsTbl.Count)) * 100.0)
} else if statsTbl.ModifyCount == 0 {
healthy = 100
}
e.appendRow([]interface{}{
dbName,
tblName,
partitionName,
healthy,
})
}
53 changes: 42 additions & 11 deletions executor/show_stats_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -44,11 +44,11 @@ func (s *testSuite) TestShowStatsHistograms(c *C) {
tk.MustExec("analyze table t")
result := tk.MustQuery("show stats_histograms").Sort()
c.Assert(len(result.Rows()), Equals, 2)
c.Assert(result.Rows()[0][2], Equals, "a")
c.Assert(result.Rows()[1][2], Equals, "b")
c.Assert(result.Rows()[0][3], Equals, "a")
c.Assert(result.Rows()[1][3], Equals, "b")
result = tk.MustQuery("show stats_histograms where column_name = 'a'")
c.Assert(len(result.Rows()), Equals, 1)
c.Assert(result.Rows()[0][2], Equals, "a")
c.Assert(result.Rows()[0][3], Equals, "a")
}

func (s *testSuite) TestShowStatsBuckets(c *C) {
Expand All @@ -60,9 +60,9 @@ func (s *testSuite) TestShowStatsBuckets(c *C) {
tk.MustExec("insert into t values (1,1)")
tk.MustExec("analyze table t")
result := tk.MustQuery("show stats_buckets").Sort()
result.Sort().Check(testkit.Rows("test t a 0 0 1 1 1 1", "test t b 0 0 1 1 1 1", "test t idx 1 0 1 1 (1, 1) (1, 1)"))
result.Check(testkit.Rows("test t a 0 0 1 1 1 1", "test t b 0 0 1 1 1 1", "test t idx 1 0 1 1 (1, 1) (1, 1)"))
result = tk.MustQuery("show stats_buckets where column_name = 'idx'")
result.Check(testkit.Rows("test t idx 1 0 1 1 (1, 1) (1, 1)"))
result.Check(testkit.Rows("test t idx 1 0 1 1 (1, 1) (1, 1)"))
}

func (s *testSuite) TestShowStatsHealthy(c *C) {
Expand All @@ -72,22 +72,22 @@ func (s *testSuite) TestShowStatsHealthy(c *C) {
tk.MustExec("create table t (a int)")
tk.MustExec("create index idx on t(a)")
tk.MustExec("analyze table t")
tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100"))
tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100"))
tk.MustExec("insert into t values (1), (2)")
do, _ := session.GetDomain(s.store)
do.StatsHandle().DumpStatsDeltaToKV(statistics.DumpAll)
tk.MustExec("analyze table t")
tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100"))
tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100"))
tk.MustExec("insert into t values (3), (4), (5), (6), (7), (8), (9), (10)")
do.StatsHandle().DumpStatsDeltaToKV(statistics.DumpAll)
do.StatsHandle().Update(do.InfoSchema())
tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 19"))
tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 19"))
tk.MustExec("analyze table t")
tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100"))
tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 100"))
tk.MustExec("delete from t")
do.StatsHandle().DumpStatsDeltaToKV(statistics.DumpAll)
do.StatsHandle().Update(do.InfoSchema())
tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 0"))
tk.MustQuery("show stats_healthy").Check(testkit.Rows("test t 0"))
}

func (s *testSuite) TestShowStatsHasNullValue(c *C) {
Expand All @@ -96,5 +96,36 @@ func (s *testSuite) TestShowStatsHasNullValue(c *C) {
tk.MustExec("create table t (a int, index idx(a))")
tk.MustExec("insert into t values(NULL)")
tk.MustExec("analyze table t")
tk.MustQuery("show stats_buckets").Check(testkit.Rows("test t idx 1 0 1 1 NULL NULL"))
tk.MustQuery("show stats_buckets").Check(testkit.Rows("test t idx 1 0 1 1 NULL NULL"))
}

func (s *testSuite) TestShowPartitionStats(c *C) {
tk := testkit.NewTestKit(c, s.store)
tk.MustExec("set @@session.tidb_enable_table_partition=1")
tk.MustExec("use test")
tk.MustExec("drop table if exists t")
createTable := `CREATE TABLE t (a int, b int, primary key(a), index idx(b))
PARTITION BY RANGE ( a ) (PARTITION p0 VALUES LESS THAN (6))`
tk.MustExec(createTable)
tk.MustExec(`insert into t values (1, 1)`)
tk.MustExec("analyze table t")

result := tk.MustQuery("show stats_meta")
c.Assert(len(result.Rows()), Equals, 1)
c.Assert(result.Rows()[0][0], Equals, "test")
c.Assert(result.Rows()[0][1], Equals, "t")
c.Assert(result.Rows()[0][2], Equals, "p0")

result = tk.MustQuery("show stats_histograms").Sort()
c.Assert(len(result.Rows()), Equals, 2)
c.Assert(result.Rows()[0][2], Equals, "p0")
c.Assert(result.Rows()[0][3], Equals, "a")
c.Assert(result.Rows()[1][2], Equals, "p0")
c.Assert(result.Rows()[1][3], Equals, "idx")

result = tk.MustQuery("show stats_buckets").Sort()
result.Check(testkit.Rows("test t p0 a 0 0 1 1 1 1", "test t p0 idx 1 0 1 1 1 1"))

result = tk.MustQuery("show stats_healthy")
result.Check(testkit.Rows("test t p0 100"))
}
16 changes: 8 additions & 8 deletions planner/core/planbuilder.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -1633,20 +1633,20 @@ func buildShowSchema(s *ast.ShowStmt) (schema *expression.Schema) {
ftypes = []byte{mysql.TypeLonglong, mysql.TypeVarchar, mysql.TypeVarchar,
mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeLong, mysql.TypeVarchar, mysql.TypeString, mysql.TypeLonglong}
case ast.ShowStatsMeta:
names = []string{"Db_name", "Table_name", "Update_time", "Modify_count", "Row_count"}
ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeDatetime, mysql.TypeLonglong, mysql.TypeLonglong}
names = []string{"Db_name", "Table_name", "Partition_name", "Update_time", "Modify_count", "Row_count"}
ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeDatetime, mysql.TypeLonglong, mysql.TypeLonglong}
case ast.ShowStatsHistograms:
names = []string{"Db_name", "Table_name", "Column_name", "Is_index", "Update_time", "Distinct_count", "Null_count", "Avg_col_size"}
ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeTiny, mysql.TypeDatetime,
names = []string{"Db_name", "Table_name", "Partition_name", "Column_name", "Is_index", "Update_time", "Distinct_count", "Null_count", "Avg_col_size"}
ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeTiny, mysql.TypeDatetime,
mysql.TypeLonglong, mysql.TypeLonglong, mysql.TypeDouble}
case ast.ShowStatsBuckets:
names = []string{"Db_name", "Table_name", "Column_name", "Is_index", "Bucket_id", "Count",
names = []string{"Db_name", "Table_name", "Partition_name", "Column_name", "Is_index", "Bucket_id", "Count",
"Repeats", "Lower_Bound", "Upper_Bound"}
ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeTiny, mysql.TypeLonglong,
ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeTiny, mysql.TypeLonglong,
mysql.TypeLonglong, mysql.TypeLonglong, mysql.TypeVarchar, mysql.TypeVarchar}
case ast.ShowStatsHealthy:
names = []string{"Db_name", "Table_name", "Healthy"}
ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeLonglong}
names = []string{"Db_name", "Table_name", "Partition_name", "Healthy"}
ftypes = []byte{mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeVarchar, mysql.TypeLonglong}
case ast.ShowProfiles: // ShowProfiles is deprecated.
names = []string{"Query_ID", "Duration", "Query"}
ftypes = []byte{mysql.TypeLong, mysql.TypeDouble, mysql.TypeVarchar}
Expand Down
2 changes: 1 addition & 1 deletion server/statistics_handler_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -153,7 +153,7 @@ func (ds *testDumpStatsSuite) checkData(c *C, path string) {
var dbName, tableName string
var modifyCount, count int64
var other interface{}
err = rows.Scan(&dbName, &tableName, &other, &modifyCount, &count)
err = rows.Scan(&dbName, &tableName, &other, &other, &modifyCount, &count)
dbt.Check(err, IsNil)
dbt.Check(dbName, Equals, "tidb")
dbt.Check(tableName, Equals, "test")
Expand Down

0 comments on commit cc1da16

Please sign in to comment.