From 4e012e54ce89fab9551e7ff58ebcffe57590244a Mon Sep 17 00:00:00 2001 From: Ehco Date: Fri, 6 Aug 2021 12:31:12 +0800 Subject: [PATCH] *: use `show full tables` in ListAllDatabasesTables (#325) --- .github/workflows/go.yml | 101 +++++++++++++++++- Makefile | 2 +- .../quote-database-schema-create-mysql57.sql | 2 + ...te-database.quote-table-schema-mysql57.sql | 7 ++ ...database.quote-table.000000000-mysql57.sql | 13 +++ tests/quote/run.sh | 18 +++- tests/run.sh | 35 ++++-- tests/views/run.sh | 3 +- v4/export/config.go | 13 +++ v4/export/config_test.go | 21 +++- v4/export/dump.go | 25 +++-- v4/export/dump_test.go | 28 +++++ v4/export/prepare_test.go | 84 +++++++++++++-- v4/export/sql.go | 55 ++++++++-- 14 files changed, 360 insertions(+), 47 deletions(-) create mode 100755 tests/quote/data/quote-database-schema-create-mysql57.sql create mode 100755 tests/quote/data/quote-database.quote-table-schema-mysql57.sql create mode 100755 tests/quote/data/quote-database.quote-table.000000000-mysql57.sql diff --git a/.github/workflows/go.yml b/.github/workflows/go.yml index c57b33b8ac2fa..eaf83d324aa43 100644 --- a/.github/workflows/go.yml +++ b/.github/workflows/go.yml @@ -29,22 +29,113 @@ jobs: run: make test WITH_RACE=1 - uses: codecov/codecov-action@v1 - integration-test: + integration-test-mysql-5735: runs-on: ubuntu-latest timeout-minutes: 15 strategy: fail-fast: true + services: + mysql: + image: mysql:5.7.35 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 steps: - uses: actions/checkout@v2 + - name: Shutdown Ubuntu MySQL (SUDO) + run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it - name: Set up Go 1.16 uses: actions/setup-go@v2 with: go-version: 1.16 + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Get dependencies + run: go mod download + - name: Download dependencies + run: sh install.sh + - name: Integration test + run: make integration_test + - name: Set up tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + + integration-test-mysql-8026: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: true + services: + mysql: + image: mysql:8.0.26 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + steps: + - uses: actions/checkout@v2 + - name: Shutdown Ubuntu MySQL (SUDO) + run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: 1.16 + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Get dependencies + run: go mod download + - name: Download dependencies + run: sh install.sh + - name: Integration test + run: make integration_test + - name: Set up tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 + + integration-test-mysql-8022: + runs-on: ubuntu-latest + timeout-minutes: 15 + strategy: + fail-fast: true + services: + mysql: + image: mysql:8.0.22 + env: + MYSQL_ALLOW_EMPTY_PASSWORD: yes + ports: + - 3306:3306 + options: --health-cmd="mysqladmin ping" --health-interval=10s --health-timeout=5s --health-retries=3 + steps: + - uses: actions/checkout@v2 + - name: Shutdown Ubuntu MySQL (SUDO) + run: sudo service mysql stop # Shutdown the Default MySQL, "sudo" is necessary, please not remove it + - name: Set up Go 1.16 + uses: actions/setup-go@v2 + with: + go-version: 1.16 + - uses: actions/cache@v2 + with: + path: ~/go/pkg/mod + key: ${{ runner.os }}-go-${{ hashFiles('**/go.sum') }} + restore-keys: | + ${{ runner.os }}-go- + - name: Get dependencies + run: go mod download - name: Download dependencies run: sh install.sh - - name: Start MySQL - run: | - sudo systemctl start mysql.service - mysqladmin -uroot -proot password '' - name: Integration test run: make integration_test + - name: Set up tmate session + if: ${{ failure() }} + uses: mxschmitt/action-tmate@v3 diff --git a/Makefile b/Makefile index 108c118f1f65f..6067ae82c7e89 100644 --- a/Makefile +++ b/Makefile @@ -29,7 +29,7 @@ test: failpoint-enable integration_test: bins failpoint-enable bin/dumpling @make failpoint-disable - ./tests/run.sh + ./tests/run.sh $(CASE) tools: @echo "install tools..." diff --git a/tests/quote/data/quote-database-schema-create-mysql57.sql b/tests/quote/data/quote-database-schema-create-mysql57.sql new file mode 100755 index 0000000000000..a5df1c26b6c8b --- /dev/null +++ b/tests/quote/data/quote-database-schema-create-mysql57.sql @@ -0,0 +1,2 @@ +/*!40101 SET NAMES binary*/; +CREATE DATABASE `quo``te/database` /*!40100 DEFAULT CHARACTER SET latin1 */; diff --git a/tests/quote/data/quote-database.quote-table-schema-mysql57.sql b/tests/quote/data/quote-database.quote-table-schema-mysql57.sql new file mode 100755 index 0000000000000..b3c55dee26330 --- /dev/null +++ b/tests/quote/data/quote-database.quote-table-schema-mysql57.sql @@ -0,0 +1,7 @@ +/*!40101 SET NAMES binary*/; +CREATE TABLE `quo``te/table` ( + `quo``te/col` int(11) NOT NULL, + `a` int(11) DEFAULT NULL, + `gen``id` int(11) GENERATED ALWAYS AS (`quo``te/col`) VIRTUAL, + PRIMARY KEY (`quo``te/col`) +) ENGINE=InnoDB DEFAULT CHARSET=latin1; diff --git a/tests/quote/data/quote-database.quote-table.000000000-mysql57.sql b/tests/quote/data/quote-database.quote-table.000000000-mysql57.sql new file mode 100755 index 0000000000000..5cee6b7b4a67d --- /dev/null +++ b/tests/quote/data/quote-database.quote-table.000000000-mysql57.sql @@ -0,0 +1,13 @@ +/*!40101 SET NAMES binary*/; +INSERT INTO `quo``te/table` (`quo``te/col`,`a`) VALUES +(0,10), +(1,9), +(2,8), +(3,7), +(4,6), +(5,5), +(6,4), +(7,3), +(8,2), +(9,1), +(10,0); diff --git a/tests/quote/run.sh b/tests/quote/run.sh index 232d153fefc6f..eafe61ed51caf 100644 --- a/tests/quote/run.sh +++ b/tests/quote/run.sh @@ -1,13 +1,23 @@ -#!/bin/sh +#!/bin/bash # # Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. set -eu mkdir -p "$DUMPLING_OUTPUT_DIR"/data -cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table.000000000.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable.000000000.sql" -cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table-schema.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable-schema.sql" -cp "$DUMPLING_BASE_NAME/data/quote-database-schema-create.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase-schema-create.sql" + +mysql_version=$(echo "select version()" | mysql -uroot -h127.0.0.1 -P3306 | awk 'NR==2' | awk '{print $1}') +echo "current user mysql version is $mysql_version" +if [[ $mysql_version = 5* ]]; then + # there is a bug in mysql 5.x, see https://bugs.mysql.com/bug.php?id=96994, so we use different create db/table sql + cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table.000000000-mysql57.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable.000000000.sql" + cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table-schema-mysql57.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable-schema.sql" + cp "$DUMPLING_BASE_NAME/data/quote-database-schema-create-mysql57.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase-schema-create.sql" +else + cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table.000000000.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable.000000000.sql" + cp "$DUMPLING_BASE_NAME/data/quote-database.quote-table-schema.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase.quo\`te%2Ftable-schema.sql" + cp "$DUMPLING_BASE_NAME/data/quote-database-schema-create.sql" "$DUMPLING_OUTPUT_DIR/data/quo\`te%2Fdatabase-schema-create.sql" +fi db="quo\`te/database" run_sql "drop database if exists \`quo\`\`te/database\`" diff --git a/tests/run.sh b/tests/run.sh index 407a1af6aa72a..bc535c2f8ea73 100755 --- a/tests/run.sh +++ b/tests/run.sh @@ -1,4 +1,4 @@ -#!/bin/sh +#!/bin/bash # # Copyright 2020 PingCAP, Inc. Licensed under Apache-2.0. @@ -17,7 +17,6 @@ mkdir -p "$DUMPLING_TEST_DIR" PATH="tests/_utils:$PATH" . "tests/_utils/run_services" - file_should_exist bin/tidb-server file_should_exist bin/tidb-lightning file_should_exist bin/dumpling @@ -26,8 +25,9 @@ file_should_exist bin/sync_diff_inspector trap stop_services EXIT start_services -for script in tests/*/run.sh; do - echo "****************** Running test $script..." +run_case_by_fullpath() { + script="$1" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>> Running test $script..." DUMPLING_BASE_NAME="$(dirname "$script")" export DUMPLING_BASE_NAME TEST_NAME="$(basename "$(dirname "$script")")" @@ -35,11 +35,24 @@ for script in tests/*/run.sh; do export DUMPLING_OUTPUT_DIR PATH="tests/_utils:$PATH" \ - sh "$script" - - echo "Cleaning up test output dir: $DUMPLING_OUTPUT_DIR" + bash "$script" + echo ">>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>>TEST: $script Passed Cleaning up test output dir: $DUMPLING_OUTPUT_DIR" rm -rf "$DUMPLING_OUTPUT_DIR" - -done - -echo "Passed integration tests." +} + +if [ "$#" -ge 1 ]; then + test_case="$@" +else + test_case="*" +fi + +if [ "$test_case" == "*" ]; then + for script in tests/*/run.sh; do + run_case_by_fullpath "$script" + done +else + script="tests/$test_case/run.sh" + run_case_by_fullpath "$script" +fi + +echo "Passed integration tests." \ No newline at end of file diff --git a/tests/views/run.sh b/tests/views/run.sh index c9462e6c95cb3..b97513543db5c 100644 --- a/tests/views/run.sh +++ b/tests/views/run.sh @@ -9,7 +9,8 @@ run_sql_file "$DUMPLING_BASE_NAME/data/views-schema-create.sql" export DUMPLING_TEST_DATABASE="views" run_sql "create table t (a bigint, b varchar(255))" -run_sql "set session collation_connection='utf8mb4_general_ci'; create definer = 'root'@'localhost' view v as select * from t;" +run_sql_file "$DUMPLING_BASE_NAME/data/views.v-schema-view.sql" + # insert 20 records to `t`. run_sql "insert into t values $(seq -s, 20 | sed 's/,*$//g' | sed 's/[0-9]*/(\0,"\0")/g')" diff --git a/v4/export/config.go b/v4/export/config.go index 6efea6c98a35e..54933ba2e487c 100644 --- a/v4/export/config.go +++ b/v4/export/config.go @@ -733,3 +733,16 @@ func adjustFileFormat(conf *Config) error { } return nil } + +func matchMysqlBugversion(info ServerInfo) bool { + // if 8.0.3 <= mysql8 version < 8.0.23 + // FLUSH TABLES WITH READ LOCK could block other sessions from executing SHOW TABLE STATUS. + // see more in https://dev.mysql.com/doc/relnotes/mysql/8.0/en/news-8-0-23.html + if info.ServerType != ServerTypeMySQL { + return false + } + currentVersion := info.ServerVersion + bugVersionStart := semver.New("8.0.2") + bugVersionEnd := semver.New("8.0.23") + return bugVersionStart.LessThan(*currentVersion) && currentVersion.LessThan(*bugVersionEnd) +} diff --git a/v4/export/config_test.go b/v4/export/config_test.go index 466492d58542b..d76245fa651ad 100644 --- a/v4/export/config_test.go +++ b/v4/export/config_test.go @@ -3,7 +3,7 @@ package export import ( - "context" + tcontext "github.com/pingcap/dumpling/v4/context" . "github.com/pingcap/check" ) @@ -14,7 +14,24 @@ type testConfigSuite struct{} func (s *testConfigSuite) TestCreateExternalStorage(c *C) { mockConfig := defaultConfigForTest(c) - loc, err := mockConfig.createExternalStorage(context.Background()) + loc, err := mockConfig.createExternalStorage(tcontext.Background()) c.Assert(err, IsNil) c.Assert(loc.URI(), Matches, "file:.*") } + +func (s *testConfigSuite) TestMatchMysqlBugversion(c *C) { + cases := []struct { + serverInfo ServerInfo + expected bool + }{ + {ParseServerInfo(tcontext.Background(), "5.7.25-TiDB-3.0.6"), false}, + {ParseServerInfo(tcontext.Background(), "8.0.2"), false}, + {ParseServerInfo(tcontext.Background(), "8.0.3"), true}, + {ParseServerInfo(tcontext.Background(), "8.0.22"), true}, + {ParseServerInfo(tcontext.Background(), "8.0.23"), false}, + } + for _, x := range cases { + cmt := Commentf("server info %s", x.serverInfo) + c.Assert(x.expected, Equals, matchMysqlBugversion(x.serverInfo), cmt) + } +} diff --git a/v4/export/dump.go b/v4/export/dump.go index cd9a86b6f1bb2..51a96fb2fff12 100755 --- a/v4/export/dump.go +++ b/v4/export/dump.go @@ -851,6 +851,19 @@ func extractTiDBRowIDFromDecodedKey(indexField, key string) (string, error) { return "", errors.Errorf("decoded key %s doesn't have %s field", key, indexField) } +func getListTableTypeByConf(conf *Config) listTableType { + // use listTableByShowTableStatus by default because it has better performance + listType := listTableByShowTableStatus + if conf.Consistency == consistencyTypeLock { + // for consistency lock, we need to build the tables to dump as soon as possible + listType = listTableByInfoSchema + } else if conf.Consistency == consistencyTypeFlush && matchMysqlBugversion(conf.ServerInfo) { + // For some buggy versions of mysql, we need a workaround to get a list of table names. + listType = listTableByShowFullTables + } + return listType +} + func prepareTableListToDump(tctx *tcontext.Context, conf *Config, db *sql.Conn) error { databases, err := prepareDumpingDatabases(conf, db) if err != nil { @@ -861,9 +874,7 @@ func prepareTableListToDump(tctx *tcontext.Context, conf *Config, db *sql.Conn) if !conf.NoViews { tableTypes = append(tableTypes, TableTypeView) } - // for consistency lock, we need to build the tables to dump as soon as possible - asap := conf.Consistency == consistencyTypeLock - conf.Tables, err = ListAllDatabasesTables(tctx, db, databases, asap, tableTypes...) + conf.Tables, err = ListAllDatabasesTables(tctx, db, databases, getListTableTypeByConf(conf), tableTypes...) if err != nil { return err } @@ -1045,16 +1056,16 @@ func detectServerInfo(d *Dumper) error { // resolveAutoConsistency is an initialization step of Dumper. func resolveAutoConsistency(d *Dumper) error { conf := d.conf - if conf.Consistency != "auto" { + if conf.Consistency != consistencyTypeAuto { return nil } switch conf.ServerInfo.ServerType { case ServerTypeTiDB: - conf.Consistency = "snapshot" + conf.Consistency = consistencyTypeSnapshot case ServerTypeMySQL, ServerTypeMariaDB: - conf.Consistency = "flush" + conf.Consistency = consistencyTypeFlush default: - conf.Consistency = "none" + conf.Consistency = consistencyTypeNone } return nil } diff --git a/v4/export/dump_test.go b/v4/export/dump_test.go index 8b2fbc92c7167..da55debc6c29d 100644 --- a/v4/export/dump_test.go +++ b/v4/export/dump_test.go @@ -88,3 +88,31 @@ func (s *testSQLSuite) TestDumpTableMeta(c *C) { c.Assert(meta.HasImplicitRowID(), Equals, hasImplicitRowID) } } + +func (s *testSQLSuite) TestGetListTableTypeByConf(c *C) { + conf := defaultConfigForTest(c) + tctx := tcontext.Background().WithLogger(appLogger) + cases := []struct { + serverInfo ServerInfo + consistency string + expected listTableType + }{ + {ParseServerInfo(tctx, "5.7.25-TiDB-3.0.6"), consistencyTypeSnapshot, listTableByShowTableStatus}, + // no bug version + {ParseServerInfo(tctx, "8.0.2"), consistencyTypeLock, listTableByInfoSchema}, + {ParseServerInfo(tctx, "8.0.2"), consistencyTypeFlush, listTableByShowTableStatus}, + {ParseServerInfo(tctx, "8.0.23"), consistencyTypeNone, listTableByShowTableStatus}, + + // bug version + {ParseServerInfo(tctx, "8.0.3"), consistencyTypeLock, listTableByInfoSchema}, + {ParseServerInfo(tctx, "8.0.3"), consistencyTypeFlush, listTableByShowFullTables}, + {ParseServerInfo(tctx, "8.0.3"), consistencyTypeNone, listTableByShowTableStatus}, + } + + for _, x := range cases { + conf.Consistency = x.consistency + conf.ServerInfo = x.serverInfo + cmt := Commentf("server info %s consistency %s", x.serverInfo, x.consistency) + c.Assert(getListTableTypeByConf(conf), Equals, x.expected, cmt) + } +} diff --git a/v4/export/prepare_test.go b/v4/export/prepare_test.go index acf0691a83322..04b806d151763 100644 --- a/v4/export/prepare_test.go +++ b/v4/export/prepare_test.go @@ -5,6 +5,7 @@ package export import ( "context" "fmt" + "strings" tcontext "github.com/pingcap/dumpling/v4/context" @@ -89,7 +90,7 @@ func (s *testPrepareSuite) TestListAllTables(c *C) { query := "SELECT TABLE_SCHEMA,TABLE_NAME,TABLE_TYPE,AVG_ROW_LENGTH FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE'" mock.ExpectQuery(query).WillReturnRows(rows) - tables, err := ListAllDatabasesTables(tctx, conn, dbNames, true, TableTypeBase) + tables, err := ListAllDatabasesTables(tctx, conn, dbNames, listTableByInfoSchema, TableTypeBase) c.Assert(err, IsNil) for d, t := range tables { @@ -108,7 +109,7 @@ func (s *testPrepareSuite) TestListAllTables(c *C) { query = "SELECT TABLE_SCHEMA,TABLE_NAME,TABLE_TYPE,AVG_ROW_LENGTH FROM INFORMATION_SCHEMA.TABLES WHERE TABLE_TYPE='BASE TABLE' OR TABLE_TYPE='VIEW'" mock.ExpectQuery(query).WillReturnRows(sqlmock.NewRows([]string{"TABLE_SCHEMA", "TABLE_NAME", "TABLE_TYPE", "AVG_ROW_LENGTH"}). AddRow("db", "t1", TableTypeBaseStr, 1).AddRow("db", "t2", TableTypeViewStr, nil)) - tables, err = ListAllDatabasesTables(tctx, conn, []string{"db"}, true, TableTypeBase, TableTypeView) + tables, err = ListAllDatabasesTables(tctx, conn, []string{"db"}, listTableByInfoSchema, TableTypeBase, TableTypeView) c.Assert(err, IsNil) c.Assert(len(tables), Equals, 1) c.Assert(len(tables["db"]), Equals, 2) @@ -145,13 +146,13 @@ func (s *testPrepareSuite) TestListAllTablesByTableStatus(c *C) { if tbInfo.Type == TableTypeBase { rows.AddRow(tbInfo.Name, "InnoDB", 10, "Dynamic", 0, 0, 16384, 0, 0, 0, nil, "2021-07-08 03:04:07", nil, nil, "latin1_swedish_ci", nil, "", "") } else { - rows.AddRow(tbInfo.Name, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, "VIEW") + rows.AddRow(tbInfo.Name, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, TableTypeView.String()) } } mock.ExpectQuery(fmt.Sprintf(query, dbName)).WillReturnRows(rows) } - tables, err := ListAllDatabasesTables(tctx, conn, dbNames, false, TableTypeBase) + tables, err := ListAllDatabasesTables(tctx, conn, dbNames, listTableByShowTableStatus, TableTypeBase) c.Assert(err, IsNil) for d, t := range tables { @@ -169,8 +170,79 @@ func (s *testPrepareSuite) TestListAllTablesByTableStatus(c *C) { AppendViews("db", "t2") mock.ExpectQuery(fmt.Sprintf(query, "db")).WillReturnRows(sqlmock.NewRows(showTableStatusColumnNames). AddRow("t1", "InnoDB", 10, "Dynamic", 0, 1, 16384, 0, 0, 0, nil, "2021-07-08 03:04:07", nil, nil, "latin1_swedish_ci", nil, "", ""). - AddRow("t2", nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, "VIEW")) - tables, err = ListAllDatabasesTables(tctx, conn, []string{"db"}, false, TableTypeBase, TableTypeView) + AddRow("t2", nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, nil, TableTypeView.String())) + tables, err = ListAllDatabasesTables(tctx, conn, []string{"db"}, listTableByShowTableStatus, TableTypeBase, TableTypeView) + c.Assert(err, IsNil) + c.Assert(len(tables), Equals, 1) + c.Assert(len(tables["db"]), Equals, 2) + for i := 0; i < len(tables["db"]); i++ { + cmt := Commentf("%v mismatch: %v", tables["db"][i], data["db"][i]) + c.Assert(tables["db"][i].Equals(data["db"][i]), IsTrue, cmt) + } + + c.Assert(mock.ExpectationsWereMet(), IsNil) +} + +func (s *testPrepareSuite) TestListAllTablesByShowFullTables(c *C) { + db, mock, err := sqlmock.New() + c.Assert(err, IsNil) + defer db.Close() + conn, err := db.Conn(context.Background()) + c.Assert(err, IsNil) + tctx := tcontext.Background().WithLogger(appLogger) + + // Test list all tables and skipping views. + data := NewDatabaseTables(). + AppendTables("db1", []string{"t1", "t2"}, []uint64{1, 2}). + AppendTables("db2", []string{"t3", "t4", "t5"}, []uint64{3, 4, 5}). + AppendViews("db3", "t6", "t7", "t8") + + query := "SHOW FULL TABLES FROM `%s` WHERE TABLE_TYPE='BASE TABLE'" + dbNames := make([]databaseName, 0, len(data)) + for dbName, tableInfos := range data { + dbNames = append(dbNames, dbName) + columnNames := []string{strings.ToUpper(fmt.Sprintf("Tables_in_%s", dbName)), "TABLE_TYPE"} + rows := sqlmock.NewRows(columnNames) + for _, tbInfo := range tableInfos { + if tbInfo.Type == TableTypeBase { + rows.AddRow(tbInfo.Name, TableTypeBase.String()) + } else { + rows.AddRow(tbInfo.Name, TableTypeView.String()) + } + } + mock.ExpectQuery(fmt.Sprintf(query, dbName)).WillReturnRows(rows) + } + + tables, err := ListAllDatabasesTables(tctx, conn, dbNames, listTableByShowFullTables, TableTypeBase) + c.Assert(err, IsNil) + + for d, t := range tables { + expectedTbs, ok := data[d] + c.Assert(ok, IsTrue) + for i := 0; i < len(t); i++ { + cmt := Commentf("%v mismatch: %v", t[i], expectedTbs[i]) + c.Assert(t[i].Equals(expectedTbs[i]), IsTrue, cmt) + } + } + + // Test list all tables and not skipping views. + query = "SHOW FULL TABLES FROM `%s` WHERE TABLE_TYPE='BASE TABLE' OR TABLE_TYPE='VIEW'" + data = NewDatabaseTables(). + AppendTables("db", []string{"t1"}, []uint64{1}). + AppendViews("db", "t2") + for dbName, tableInfos := range data { + columnNames := []string{strings.ToUpper(fmt.Sprintf("Tables_in_%s", dbName)), "TABLE_TYPE"} + rows := sqlmock.NewRows(columnNames) + for _, tbInfo := range tableInfos { + if tbInfo.Type == TableTypeBase { + rows.AddRow(tbInfo.Name, TableTypeBase.String()) + } else { + rows.AddRow(tbInfo.Name, TableTypeView.String()) + } + } + mock.ExpectQuery(fmt.Sprintf(query, dbName)).WillReturnRows(rows) + } + tables, err = ListAllDatabasesTables(tctx, conn, []string{"db"}, listTableByShowFullTables, TableTypeBase, TableTypeView) c.Assert(err, IsNil) c.Assert(len(tables), Equals, 1) c.Assert(len(tables["db"]), Equals, 2) diff --git a/v4/export/sql.go b/v4/export/sql.go index b09a659f7d135..76c8edb277ace 100644 --- a/v4/export/sql.go +++ b/v4/export/sql.go @@ -22,7 +22,17 @@ import ( "go.uber.org/zap" ) -const orderByTiDBRowID = "ORDER BY `_tidb_rowid`" +const ( + orderByTiDBRowID = "ORDER BY `_tidb_rowid`" +) + +type listTableType int + +const ( + listTableByInfoSchema listTableType = iota + listTableByShowFullTables + listTableByShowTableStatus +) // ShowDatabases shows the databases of a database server. func ShowDatabases(db *sql.Conn) ([]string, error) { @@ -152,9 +162,11 @@ func RestoreCharset(w io.StringWriter) { } // ListAllDatabasesTables lists all the databases and tables from the database -// if asap is true, will use information_schema to get table info in one query -// if asap is false, will use show table status for each database because it has better performance according to our tests -func ListAllDatabasesTables(tctx *tcontext.Context, db *sql.Conn, databaseNames []string, asap bool, tableTypes ...TableType) (DatabaseTables, error) { // revive:disable-line:flag-parameter +// listTableByInfoSchema list tables by table information_schema in MySQL +// listTableByShowTableStatus has better performance than listTableByInfoSchema +// listTableByShowFullTables is used in mysql8 version [8.0.3,8.0.23), more details can be found in the comments of func matchMysqlBugversion +func ListAllDatabasesTables(tctx *tcontext.Context, db *sql.Conn, databaseNames []string, + listType listTableType, tableTypes ...TableType) (DatabaseTables, error) { // revive:disable-line:flag-parameter dbTables := DatabaseTables{} var ( schema, table, tableTypeStr string @@ -162,11 +174,13 @@ func ListAllDatabasesTables(tctx *tcontext.Context, db *sql.Conn, databaseNames avgRowLength uint64 err error ) - if asap { - tableTypeConditions := make([]string, len(tableTypes)) - for i, tableType := range tableTypes { - tableTypeConditions[i] = fmt.Sprintf("TABLE_TYPE='%s'", tableType) - } + + tableTypeConditions := make([]string, len(tableTypes)) + for i, tableType := range tableTypes { + tableTypeConditions[i] = fmt.Sprintf("TABLE_TYPE='%s'", tableType) + } + switch listType { + case listTableByInfoSchema: query := fmt.Sprintf("SELECT TABLE_SCHEMA,TABLE_NAME,TABLE_TYPE,AVG_ROW_LENGTH FROM INFORMATION_SCHEMA.TABLES WHERE %s", strings.Join(tableTypeConditions, " OR ")) for _, schema := range databaseNames { dbTables[schema] = make([]*TableInfo, 0) @@ -197,7 +211,28 @@ func ListAllDatabasesTables(tctx *tcontext.Context, db *sql.Conn, databaseNames }, query); err != nil { return nil, errors.Annotatef(err, "sql: %s", query) } - } else { + case listTableByShowFullTables: + for _, schema = range databaseNames { + dbTables[schema] = make([]*TableInfo, 0) + query := fmt.Sprintf("SHOW FULL TABLES FROM `%s` WHERE %s", + escapeString(schema), strings.Join(tableTypeConditions, " OR ")) + if err = simpleQueryWithArgs(db, func(rows *sql.Rows) error { + var err2 error + if err2 = rows.Scan(&table, &tableTypeStr); err != nil { + return errors.Trace(err2) + } + tableType, err2 = ParseTableType(tableTypeStr) + if err2 != nil { + return errors.Trace(err2) + } + avgRowLength = 0 // can't get avgRowLength from the result of `show full tables` so hardcode to 0 here + dbTables[schema] = append(dbTables[schema], &TableInfo{table, avgRowLength, tableType}) + return nil + }, query); err != nil { + return nil, errors.Annotatef(err, "sql: %s", query) + } + } + default: queryTemplate := "SHOW TABLE STATUS FROM `%s`" selectedTableType := make(map[TableType]struct{}) for _, tableType = range tableTypes {