Skip to content

Commit

Permalink
Merge branch 'master' into fix_gc_test
Browse files Browse the repository at this point in the history
  • Loading branch information
@ti-chi-bot
ti-chi-bot committed Oct 18, 2022
2 parents 6918c7f + 727c04c commit 0693b69
Show file tree
Hide file tree
Showing 7 changed files with 252 additions and 97 deletions.
20 changes: 16 additions & 4 deletions dm/pkg/checker/table_structure.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -209,7 +209,13 @@ func (c *TablesChecker) checkTable(ctx context.Context) error {

ctStmt, err := getCreateTableStmt(p, statement)
if err != nil {
return err
opt := &incompatibilityOption{
state: StateWarning,
tableID: dbutil.TableName(table.Schema, table.Name),
errMessage: err.Error(),
}
c.optCh <- opt
return nil
}
opts := c.checkAST(ctStmt)
for _, opt := range opts {
Expand Down Expand Up @@ -376,7 +382,7 @@ func (c *ShardingTablesChecker) Check(ctx context.Context) *Result {

c.firstCreateTableStmtNode, err = getCreateTableStmt(p, statement)
if err != nil {
markCheckError(r, err)
markCheckErrorFromParser(r, err)
return r
}

Expand Down Expand Up @@ -438,7 +444,10 @@ func (c *ShardingTablesChecker) checkShardingTable(ctx context.Context, r *Resul

ctStmt, err := getCreateTableStmt(p, statement)
if err != nil {
return err
c.reMu.Lock()
markCheckErrorFromParser(r, err)
c.reMu.Unlock()
continue
}

if has := hasAutoIncrementKey(ctStmt); has {
Expand Down Expand Up @@ -668,7 +677,10 @@ func (c *OptimisticShardingTablesChecker) checkTable(ctx context.Context, r *Res

ctStmt, err := getCreateTableStmt(p, statement)
if err != nil {
return err
c.reMu.Lock()
markCheckErrorFromParser(r, err)
c.reMu.Unlock()
continue
}

if has := hasAutoIncrementKey(ctStmt); has {
Expand Down
228 changes: 168 additions & 60 deletions dm/pkg/checker/table_structure_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -16,17 +16,16 @@ package checker
import (
"context"
"database/sql"
"encoding/json"
"fmt"
"testing"

"github.com/DATA-DOG/go-sqlmock"
tc "github.com/pingcap/check"
"github.com/pingcap/tidb/util/filter"
"github.com/stretchr/testify/require"
)

func (t *testCheckSuite) TestShardingTablesChecker(c *tc.C) {
func TestShardingTablesChecker(t *testing.T) {
db, mock, err := sqlmock.New()
c.Assert(err, tc.IsNil)
require.NoError(t, err)
ctx := context.Background()

// 1. test a success check
Expand All @@ -47,8 +46,8 @@ func (t *testCheckSuite) TestShardingTablesChecker(c *tc.C) {
false,
1)
result := checker.Check(ctx)
c.Assert(result.State, tc.Equals, StateSuccess)
c.Assert(mock.ExpectationsWereMet(), tc.IsNil)
require.Equal(t, StateSuccess, result.State)
require.NoError(t, mock.ExpectationsWereMet())

// 2. check different column number
checker = NewShardingTablesChecker("test-name",
Expand All @@ -69,9 +68,9 @@ func (t *testCheckSuite) TestShardingTablesChecker(c *tc.C) {
mock.ExpectQuery("SHOW CREATE TABLE `test-db`.`test-table-2`").WillReturnRows(createTableRow2)

result = checker.Check(ctx)
c.Assert(result.State, tc.Equals, StateFailure)
c.Assert(result.Errors, tc.HasLen, 1)
c.Assert(mock.ExpectationsWereMet(), tc.IsNil)
require.Equal(t, StateFailure, result.State)
require.Len(t, result.Errors, 1)
require.NoError(t, mock.ExpectationsWereMet())

// 3. check different column def
checker = NewShardingTablesChecker("test-name",
Expand All @@ -91,9 +90,9 @@ func (t *testCheckSuite) TestShardingTablesChecker(c *tc.C) {
mock.ExpectQuery("SHOW CREATE TABLE `test-db`.`test-table-2`").WillReturnRows(createTableRow2)

result = checker.Check(ctx)
c.Assert(result.State, tc.Equals, StateFailure)
c.Assert(result.Errors, tc.HasLen, 1)
c.Assert(mock.ExpectationsWereMet(), tc.IsNil)
require.Equal(t, StateFailure, result.State)
require.Len(t, result.Errors, 1)
require.NoError(t, mock.ExpectationsWereMet())

// 4. test tiflow#5759
checker = NewShardingTablesChecker("test-name",
Expand Down Expand Up @@ -131,13 +130,13 @@ func (t *testCheckSuite) TestShardingTablesChecker(c *tc.C) {

// in tiflow#5759, this function will enter deadlock
result = checker.Check(ctx)
c.Assert(result.State, tc.Equals, StateFailure)
c.Assert(result.Errors, tc.HasLen, 3)
require.Equal(t, StateFailure, result.State)
require.Len(t, result.Errors, 3)
}

func (t *testCheckSuite) TestTablesChecker(c *tc.C) {
func TestTablesChecker(t *testing.T) {
db, mock, err := sqlmock.New()
c.Assert(err, tc.IsNil)
require.NoError(t, err)
ctx := context.Background()

// 1. test a success check
Expand All @@ -161,8 +160,8 @@ func (t *testCheckSuite) TestTablesChecker(c *tc.C) {
}},
1)
result := checker.Check(ctx)
c.Assert(result.State, tc.Equals, StateSuccess)
c.Assert(mock.ExpectationsWereMet(), tc.IsNil)
require.Equal(t, StateSuccess, result.State)
require.NoError(t, mock.ExpectationsWereMet())

// 2. check many errors
maxConnectionsRow = sqlmock.NewRows([]string{"Variable_name", "Value"}).
Expand All @@ -185,34 +184,9 @@ func (t *testCheckSuite) TestTablesChecker(c *tc.C) {
}},
1)
result = checker.Check(ctx)
c.Assert(result.State, tc.Equals, StateWarning)
c.Assert(result.Errors, tc.HasLen, 2) // no PK/UK + has FK
c.Assert(mock.ExpectationsWereMet(), tc.IsNil)

// 3. unsupported charset
maxConnectionsRow = sqlmock.NewRows([]string{"Variable_name", "Value"}).
AddRow("max_connections", "2")
mock.ExpectQuery("SHOW VARIABLES LIKE 'max_connections'").WillReturnRows(maxConnectionsRow)
sqlModeRow = sqlmock.NewRows([]string{"Variable_name", "Value"}).
AddRow("sql_mode", "ANSI_QUOTES")
mock.ExpectQuery("SHOW VARIABLES LIKE 'sql_mode'").WillReturnRows(sqlModeRow)
createTableRow = sqlmock.NewRows([]string{"Table", "Create Table"}).
AddRow("test-table-1", `CREATE TABLE "test-table-1" (
"c" int(11) NOT NULL,
PRIMARY KEY ("c")
) ENGINE=InnoDB DEFAULT CHARSET=ucs2`)
mock.ExpectQuery("SHOW CREATE TABLE `test-db`.`test-table-1`").WillReturnRows(createTableRow)

checker = NewTablesChecker(
map[string]*sql.DB{"test-source": db},
map[string][]*filter.Table{"test-source": {
{Schema: "test-db", Name: "test-table-1"},
}},
1)
result = checker.Check(ctx)
c.Assert(result.State, tc.Equals, StateFailure)
c.Assert(result.Errors, tc.HasLen, 1)
c.Assert(mock.ExpectationsWereMet(), tc.IsNil)
require.Equal(t, StateWarning, result.State)
require.Len(t, result.Errors, 2)
require.NoError(t, mock.ExpectationsWereMet())

// test #5759
maxConnectionsRow = sqlmock.NewRows([]string{"Variable_name", "Value"}).
Expand Down Expand Up @@ -246,20 +220,15 @@ func (t *testCheckSuite) TestTablesChecker(c *tc.C) {
}},
1)
result = checker.Check(ctx)
c.Assert(result.State, tc.Equals, StateWarning)
c.Assert(result.Errors, tc.HasLen, 3)
require.Equal(t, StateWarning, result.State)
require.Len(t, result.Errors, 3)
}

func (t *testCheckSuite) TestOptimisticShardingTablesChecker(c *tc.C) {
func TestOptimisticShardingTablesChecker(t *testing.T) {
db, mock, err := sqlmock.New()
c.Assert(err, tc.IsNil)
require.NoError(t, err)
ctx := context.Background()

printJSON := func(r *Result) {
rawResult, _ := json.MarshalIndent(r, "", "\t")
fmt.Println("\n" + string(rawResult))
}

cases := []struct {
createTable1SQL string
createTable2SQL string
Expand Down Expand Up @@ -364,13 +333,152 @@ func (t *testCheckSuite) TestOptimisticShardingTablesChecker(c *tc.C) {
}},
0)
result := checker.Check(ctx)
printJSON(result)
c.Assert(result.State, tc.Equals, cs.expectState)
c.Assert(len(result.Errors), tc.Equals, cs.errLen)
c.Assert(mock.ExpectationsWereMet(), tc.IsNil)
require.Equal(t, cs.expectState, result.State)
require.Len(t, result.Errors, cs.errLen)
require.NoError(t, mock.ExpectationsWereMet())
}
}

func TestUnknownCharsetCollation(t *testing.T) {
db, mock, err := sqlmock.New()
require.NoError(t, err)
ctx := context.Background()

// 1. test TablesChecker

maxConnectionsRow := sqlmock.NewRows([]string{"Variable_name", "Value"}).
AddRow("max_connections", "2")
mock.ExpectQuery("SHOW VARIABLES LIKE 'max_connections'").WillReturnRows(maxConnectionsRow)
sqlModeRow := sqlmock.NewRows([]string{"Variable_name", "Value"}).
AddRow("sql_mode", "ANSI_QUOTES")
mock.ExpectQuery("SHOW VARIABLES LIKE 'sql_mode'").WillReturnRows(sqlModeRow)
createTableRow := sqlmock.NewRows([]string{"Table", "Create Table"}).
AddRow("test-table-1", `CREATE TABLE "test-table-1" (
"c" int(11) NOT NULL,
PRIMARY KEY ("c")
) ENGINE=InnoDB DEFAULT CHARSET=utf32`)
mock.ExpectQuery("SHOW CREATE TABLE `test-db`.`test-table-1`").WillReturnRows(createTableRow)

checker := NewTablesChecker(
map[string]*sql.DB{"test-source": db},
map[string][]*filter.Table{"test-source": {
{Schema: "test-db", Name: "test-table-1"},
}},
1)
result := checker.Check(ctx)
require.Equal(t, StateWarning, result.State)
require.Len(t, result.Errors, 1)
require.Contains(t, result.Errors[0].ShortErr, "Unknown character set: 'utf32'")
require.NoError(t, mock.ExpectationsWereMet())

// 2. test ShardingTablesChecker
// 2.1 the first table has unknown charset

sqlModeRow = sqlmock.NewRows([]string{"Variable_name", "Value"}).
AddRow("sql_mode", "ANSI_QUOTES")
mock.ExpectQuery("SHOW VARIABLES LIKE 'sql_mode'").WillReturnRows(sqlModeRow)
createTableRow = sqlmock.NewRows([]string{"Table", "Create Table"}).
AddRow("test-table-1", `CREATE TABLE "test-table-1" (
"c" int(11) NOT NULL,
PRIMARY KEY ("c")
) ENGINE=InnoDB DEFAULT CHARSET=utf16`)
mock.ExpectQuery("SHOW CREATE TABLE `test-db`.`test-table-1`").WillReturnRows(createTableRow)

checker = NewShardingTablesChecker("test-name",
map[string]*sql.DB{"test-source": db},
map[string][]*filter.Table{"test-source": {
{Schema: "test-db", Name: "test-table-1"},
{Schema: "test-db", Name: "test-table-2"},
}},
false,
1)
result = checker.Check(ctx)
require.Equal(t, StateWarning, result.State)
require.Len(t, result.Errors, 1)
require.Contains(t, result.Errors[0].ShortErr, "Unknown character set: 'utf16'")
require.NoError(t, mock.ExpectationsWereMet())

// 2.2 not the first table has unknown charset

mock = initShardingMock(mock)
createTableRow2 := sqlmock.NewRows([]string{"Table", "Create Table"}).
AddRow("test-table-2", `CREATE TABLE "test-table-2" (
"c" int(11) NOT NULL,
PRIMARY KEY ("c")
) ENGINE=InnoDB DEFAULT CHARSET=utf16`)
mock.ExpectQuery("SHOW CREATE TABLE `test-db`.`test-table-2`").WillReturnRows(createTableRow2)

checker = NewShardingTablesChecker("test-name",
map[string]*sql.DB{"test-source": db},
map[string][]*filter.Table{"test-source": {
{Schema: "test-db", Name: "test-table-1"},
{Schema: "test-db", Name: "test-table-2"},
}},
false,
1)
result = checker.Check(ctx)
require.Equal(t, StateWarning, result.State)
require.Len(t, result.Errors, 1)
require.Contains(t, result.Errors[0].ShortErr, "Unknown character set: 'utf16'")
require.NoError(t, mock.ExpectationsWereMet())

// 2.3 not the first table has unknown collation

mock = initShardingMock(mock)
createTableRow2 = sqlmock.NewRows([]string{"Table", "Create Table"}).
AddRow("test-table-2", `CREATE TABLE "test-table-2" (
"c" int(11) NOT NULL,
PRIMARY KEY ("c")
) ENGINE=InnoDB DEFAULT CHARSET=latin1 COLLATE=latin1_german1_ci`)
mock.ExpectQuery("SHOW CREATE TABLE `test-db`.`test-table-2`").WillReturnRows(createTableRow2)

checker = NewShardingTablesChecker("test-name",
map[string]*sql.DB{"test-source": db},
map[string][]*filter.Table{"test-source": {
{Schema: "test-db", Name: "test-table-1"},
{Schema: "test-db", Name: "test-table-2"},
}},
false,
1)
result = checker.Check(ctx)
// unknown collation will not raise error during parsing
require.Equal(t, StateSuccess, result.State)
require.NoError(t, mock.ExpectationsWereMet())

// 3. test OptimisticShardingTablesChecker

maxConnecionsRow := sqlmock.NewRows([]string{"Variable_name", "Value"}).AddRow("max_connections", "2")
mock.ExpectQuery("SHOW VARIABLES LIKE 'max_connections'").WillReturnRows(maxConnecionsRow)
sqlModeRow = sqlmock.NewRows([]string{"Variable_name", "Value"}).AddRow("sql_mode", "ANSI_QUOTES")
mock.ExpectQuery("SHOW VARIABLES LIKE 'sql_mode'").WillReturnRows(sqlModeRow)
createTableRow = sqlmock.NewRows([]string{"Table", "Create Table"}).AddRow("test-table-1", `
CREATE TABLE "test-table-1" (
"c" int(11) NOT NULL,
PRIMARY KEY ("c")
) ENGINE=InnoDB DEFAULT CHARSET=utf16`)
mock.ExpectQuery("SHOW CREATE TABLE `test-db`.`test-table-1`").WillReturnRows(createTableRow)
createTableRow2 = sqlmock.NewRows([]string{"Table", "Create Table"}).AddRow("test-table-2", `
CREATE TABLE "test-table-2" (
"c" int(11) NOT NULL,
PRIMARY KEY ("c")
) ENGINE=InnoDB DEFAULT CHARSET=utf16`)
mock.ExpectQuery("SHOW CREATE TABLE `test-db`.`test-table-2`").WillReturnRows(createTableRow2)
checker = NewOptimisticShardingTablesChecker(
"test-name",
map[string]*sql.DB{"test-source": db},
map[string][]*filter.Table{"test-source": {
&filter.Table{Schema: "test-db", Name: "test-table-1"},
&filter.Table{Schema: "test-db", Name: "test-table-2"},
}},
0)
result = checker.Check(ctx)
require.Equal(t, StateWarning, result.State)
require.Len(t, result.Errors, 2)
require.Contains(t, result.Errors[0].ShortErr, "Unknown character set: 'utf16'")
require.Contains(t, result.Errors[1].ShortErr, "Unknown character set: 'utf16'")
require.NoError(t, mock.ExpectationsWereMet())
}

func initShardingMock(mock sqlmock.Sqlmock) sqlmock.Sqlmock {
sqlModeRow := sqlmock.NewRows([]string{"Variable_name", "Value"}).
AddRow("sql_mode", "ANSI_QUOTES")
Expand Down
15 changes: 12 additions & 3 deletions dm/pkg/checker/utils.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -127,11 +127,9 @@ func IsTiDBFromVersion(version string) bool {

func markCheckError(result *Result, err error) {
if err != nil {
var state State
state := StateFailure
if utils.OriginError(err) == context.Canceled {
state = StateWarning
} else {
state = StateFailure
}
// `StateWarning` can't cover `StateFailure`.
if result.State != StateFailure {
Expand All @@ -141,6 +139,17 @@ func markCheckError(result *Result, err error) {
}
}

func markCheckErrorFromParser(result *Result, err error) {
if err != nil {
state := StateWarning
// `StateWarning` can't cover `StateFailure`.
if result.State != StateFailure {
result.State = state
}
result.Errors = append(result.Errors, &Error{Severity: state, ShortErr: err.Error()})
}
}

func isMySQLError(err error, code uint16) bool {
err = errors.Cause(err)
e, ok := err.(*mysql.MySQLError)
Expand Down
2 changes: 1 addition & 1 deletion dm/pkg/utils/common.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -333,7 +333,7 @@ var ZeroSessionCtx sessionctx.Context

// NewSessionCtx return a session context with specified session variables.
func NewSessionCtx(vars map[string]string) sessionctx.Context {
variables := variable.NewSessionVars()
variables := variable.NewSessionVars(nil)
for k, v := range vars {
_ = variables.SetSystemVar(k, v)
if strings.EqualFold(k, "time_zone") {
Expand Down
Loading

0 comments on commit 0693b69

Please sign in to comment.