Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

[release-18.0] Improve performance for BaseShowTablesWithSizes query. (#15713) #15793

Merged
merged 1 commit into from
Apr 25, 2024
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
28 changes: 6 additions & 22 deletions go/mysql/flavor_mysql.go
Original file line number Diff line number Diff line change
Expand Up @@ -302,41 +302,25 @@ GROUP BY t.table_name, t.table_type, t.create_time, t.table_comment`

// TablesWithSize80 is a query to select table along with size for mysql 8.0
//
// We join with a subquery that materializes the data from `information_schema.innodb_sys_tablespaces`
// early for performance reasons. This effectively causes only a single read of `information_schema.innodb_tablespaces`
// per query.
// Note the following:
// - We use UNION ALL to deal differently with partitioned tables vs. non-partitioned tables.
// Originally, the query handled both, but that introduced "WHERE ... OR" conditions that led to poor query
// optimization. By separating to UNION ALL we remove all "OR" conditions.
// - We use a single query to fetch both partitioned and non-partitioned tables. This is because
// accessing `information_schema.innodb_tablespaces` is expensive on servers with many tablespaces,
// and every query that loads the table needs to perform full table scans on it. Doing a single
// table scan is more efficient than doing more than one.
// - We utilize `INFORMATION_SCHEMA`.`TABLES`.`CREATE_OPTIONS` column to do early pruning before the JOIN.
// - `TABLES`.`TABLE_NAME` has `utf8mb4_0900_ai_ci` collation. `INNODB_TABLESPACES`.`NAME` has `utf8mb3_general_ci`.
// We normalize the collation to get better query performance (we force the casting at the time of our choosing)
// - `create_options` is NULL for views, and therefore we need an additional UNION ALL to include views
const TablesWithSize80 = `SELECT t.table_name,
t.table_type,
UNIX_TIMESTAMP(t.create_time),
t.table_comment,
i.file_size,
i.allocated_size
FROM information_schema.tables t
LEFT JOIN information_schema.innodb_tablespaces i
ON i.name = CONCAT(t.table_schema, '/', t.table_name) COLLATE utf8mb3_general_ci
WHERE
t.table_schema = database() AND not t.create_options <=> 'partitioned'
UNION ALL
SELECT
t.table_name,
t.table_type,
UNIX_TIMESTAMP(t.create_time),
t.table_comment,
SUM(i.file_size),
SUM(i.allocated_size)
FROM information_schema.tables t
LEFT JOIN information_schema.innodb_tablespaces i
ON i.name LIKE (CONCAT(t.table_schema, '/', t.table_name, '#p#%') COLLATE utf8mb3_general_ci )
ON i.name LIKE CONCAT(t.table_schema, '/', t.table_name, IF(t.create_options <=> 'partitioned', '#p#%', '')) COLLATE utf8mb3_general_ci
WHERE
t.table_schema = database() AND t.create_options <=> 'partitioned'
t.table_schema = database()
GROUP BY
t.table_schema, t.table_name, t.table_type, t.create_time, t.table_comment
`
Expand Down
87 changes: 75 additions & 12 deletions go/vt/vttablet/endtoend/misc_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -970,28 +970,91 @@ func TestShowTablesWithSizes(t *testing.T) {
_, err := conn.ExecuteFetch(query, 1, false)
require.NoError(t, err)
}
expectTables := map[string]([]string){ // TABLE_TYPE, TABLE_COMMENT
"show_tables_with_sizes_t1": {"BASE TABLE", ""},
"show_tables_with_sizes_v1": {"VIEW", "VIEW"},
"show_tables_with_sizes_employees": {"BASE TABLE", ""},

expectedTables := []string{
"show_tables_with_sizes_t1",
"show_tables_with_sizes_v1",
"show_tables_with_sizes_employees",
}
actualTables := []string{}

rs, err := conn.ExecuteFetch(conn.BaseShowTablesWithSizes(), math.MaxInt, false)
require.NoError(t, err)
require.NotEmpty(t, rs.Rows)

assert.GreaterOrEqual(t, len(rs.Rows), len(expectTables))
matchedTables := map[string]bool{}
assert.GreaterOrEqual(t, len(rs.Rows), len(expectedTables))

for _, row := range rs.Rows {
assert.Equal(t, 6, len(row))

tableName := row[0].ToString()
vals, ok := expectTables[tableName]
if ok {
assert.Equal(t, vals[0], row[1].ToString()) // TABLE_TYPE
assert.Equal(t, vals[1], row[3].ToString()) // TABLE_COMMENT
matchedTables[tableName] = true
if tableName == "show_tables_with_sizes_t1" {
// TABLE_TYPE
assert.Equal(t, "BASE TABLE", row[1].ToString())

assert.True(t, row[2].IsIntegral())
createTime, err := row[2].ToCastInt64()
assert.NoError(t, err)
assert.Greater(t, createTime, int64(0))

// TABLE_COMMENT
assert.Equal(t, "", row[3].ToString())

assert.True(t, row[4].IsDecimal())
fileSize, err := row[4].ToCastInt64()
assert.NoError(t, err)
assert.Greater(t, fileSize, int64(0))

assert.True(t, row[4].IsDecimal())
allocatedSize, err := row[5].ToCastInt64()
assert.NoError(t, err)
assert.Greater(t, allocatedSize, int64(0))

actualTables = append(actualTables, tableName)
} else if tableName == "show_tables_with_sizes_v1" {
// TABLE_TYPE
assert.Equal(t, "VIEW", row[1].ToString())

assert.True(t, row[2].IsIntegral())
createTime, err := row[2].ToCastInt64()
assert.NoError(t, err)
assert.Greater(t, createTime, int64(0))

// TABLE_COMMENT
assert.Equal(t, "VIEW", row[3].ToString())

assert.True(t, row[4].IsNull())
assert.True(t, row[5].IsNull())

actualTables = append(actualTables, tableName)
} else if tableName == "show_tables_with_sizes_employees" {
// TABLE_TYPE
assert.Equal(t, "BASE TABLE", row[1].ToString())

assert.True(t, row[2].IsIntegral())
createTime, err := row[2].ToCastInt64()
assert.NoError(t, err)
assert.Greater(t, createTime, int64(0))

// TABLE_COMMENT
assert.Equal(t, "", row[3].ToString())

assert.True(t, row[4].IsDecimal())
fileSize, err := row[4].ToCastInt64()
assert.NoError(t, err)
assert.Greater(t, fileSize, int64(0))

assert.True(t, row[5].IsDecimal())
allocatedSize, err := row[5].ToCastInt64()
assert.NoError(t, err)
assert.Greater(t, allocatedSize, int64(0))

actualTables = append(actualTables, tableName)
}
}
assert.Equalf(t, len(expectTables), len(matchedTables), "%v", matchedTables)

assert.Equal(t, len(expectedTables), len(actualTables))
assert.ElementsMatch(t, expectedTables, actualTables)
}

// TestTuple tests that bind variables having tuple values work with vttablet.
Expand Down
Loading