From 83c8bfef7ae93ae05ced32a714dad6b9ee721fff Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 4 Aug 2022 20:12:10 +0530 Subject: [PATCH 1/2] feat: add code to also reset replication parameters in setReplicationSourceLocked when required Signed-off-by: Manan Gupta --- go/vt/mysqlctl/replication.go | 8 ++++++++ go/vt/vttablet/tabletmanager/rpc_replication.go | 10 ++++++++-- 2 files changed, 16 insertions(+), 2 deletions(-) diff --git a/go/vt/mysqlctl/replication.go b/go/vt/mysqlctl/replication.go index d0ea4a3109d..3d293c08ff0 100644 --- a/go/vt/mysqlctl/replication.go +++ b/go/vt/mysqlctl/replication.go @@ -396,6 +396,14 @@ func (mysqld *Mysqld) SetReplicationSource(ctx context.Context, host string, por if replicationStopBefore { cmds = append(cmds, conn.StopReplicationCommand()) } + // Reset replication parameters commands makes the instance forget the source host port + // This is required because sometimes MySQL gets stuck due to improper initialization of + // master info structure or related failures and throws errors like + // ERROR 1201 (HY000): Could not initialize master info structure; more error messages can be found in the MySQL error log + // These errors can only be resolved by resetting the replication parameters, otherwise START SLAVE fails. + // Therefore, we have elected to always reset the replication parameters whenever we try to set the source host port + // Since there is no real overhead, but it makes this function robust enough to also handle failures like these. + cmds = append(cmds, conn.ResetReplicationParametersCommands()...) // If flag value is same as default, check deprecated flag value if *replicationConnectRetry == 10*time.Second && *masterConnectRetry != *replicationConnectRetry { *replicationConnectRetry = *masterConnectRetry diff --git a/go/vt/vttablet/tabletmanager/rpc_replication.go b/go/vt/vttablet/tabletmanager/rpc_replication.go index 5d1555d2886..739a638cf71 100644 --- a/go/vt/vttablet/tabletmanager/rpc_replication.go +++ b/go/vt/vttablet/tabletmanager/rpc_replication.go @@ -763,8 +763,14 @@ func (tm *TabletManager) setReplicationSourceLocked(ctx context.Context, parentA } host := parent.Tablet.MysqlHostname port := int(parent.Tablet.MysqlPort) - if status.SourceHost != host || status.SourcePort != port { - // This handles both changing the address and starting replication. + // We want to reset the replication parameters and set replication source again when forceStartReplication is provided + // because sometimes MySQL gets stuck due to improper initialization of master info structure or related failures and throws errors like + // ERROR 1201 (HY000): Could not initialize master info structure; more error messages can be found in the MySQL error log + // These errors can only be resolved by resetting the replication parameters, otherwise START SLAVE fails. So when this RPC + // gets called from VTOrc or replication manager to fix the replication in these cases with forceStartReplication, we should also + // reset the replication parameters and set the source port information again. + if status.SourceHost != host || status.SourcePort != port || forceStartReplication { + // This handles reseting the replication parameters, changing the address and then starting the replication. if err := tm.MysqlDaemon.SetReplicationSource(ctx, host, port, wasReplicating, shouldbeReplicating); err != nil { if err := tm.handleRelayLogError(err); err != nil { return err From 2f6cd6d55488177ae9093574d0f548658dbf5094 Mon Sep 17 00:00:00 2001 From: Manan Gupta Date: Thu, 4 Aug 2022 21:45:46 +0530 Subject: [PATCH 2/2] test: fix tests to reflect the change Signed-off-by: Manan Gupta --- .../fakemysqldaemon/fakemysqldaemon.go | 1 + .../endtoend/init_shard_primary_test.go | 6 ++ go/vt/vttablet/tabletmanager/tm_init_test.go | 1 + go/vt/wrangler/testlib/backup_test.go | 25 +++++-- .../testlib/copy_schema_shard_test.go | 3 +- .../testlib/emergency_reparent_shard_test.go | 13 +++- .../testlib/external_reparent_test.go | 14 +++- go/vt/wrangler/testlib/permissions_test.go | 3 +- .../testlib/planned_reparent_shard_test.go | 68 +++++++++++++++---- go/vt/wrangler/testlib/reparent_utils_test.go | 7 +- go/vt/wrangler/testlib/version_test.go | 3 +- 11 files changed, 114 insertions(+), 30 deletions(-) diff --git a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go index 5831d817d40..5402be5e540 100644 --- a/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go +++ b/go/vt/mysqlctl/fakemysqldaemon/fakemysqldaemon.go @@ -431,6 +431,7 @@ func (fmd *FakeMysqlDaemon) SetReplicationSource(ctx context.Context, host strin if stopReplicationBefore { cmds = append(cmds, "STOP SLAVE") } + cmds = append(cmds, "RESET SLAVE ALL") cmds = append(cmds, "FAKE SET MASTER") if startReplicationAfter { cmds = append(cmds, "START SLAVE") diff --git a/go/vt/vtctl/grpcvtctldserver/endtoend/init_shard_primary_test.go b/go/vt/vtctl/grpcvtctldserver/endtoend/init_shard_primary_test.go index 45d13a4e6d7..19a580c451c 100644 --- a/go/vt/vtctl/grpcvtctldserver/endtoend/init_shard_primary_test.go +++ b/go/vt/vtctl/grpcvtctldserver/endtoend/init_shard_primary_test.go @@ -63,21 +63,25 @@ func TestInitShardPrimary(t *testing.T) { tablet2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ // These come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", // These come from InitShardPrimary "FAKE RESET ALL REPLICATION", "FAKE SET SLAVE POSITION", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } tablet2.FakeMysqlDaemon.SetReplicationSourceInputs = append(tablet2.FakeMysqlDaemon.SetReplicationSourceInputs, fmt.Sprintf("%v:%v", tablet1.Tablet.Hostname, tablet1.Tablet.MysqlPort)) tablet3.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "FAKE RESET ALL REPLICATION", "FAKE SET SLAVE POSITION", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -127,6 +131,7 @@ func TestInitShardPrimaryNoFormerPrimary(t *testing.T) { tablet2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE RESET ALL REPLICATION", "FAKE SET SLAVE POSITION", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -135,6 +140,7 @@ func TestInitShardPrimaryNoFormerPrimary(t *testing.T) { tablet3.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "FAKE RESET ALL REPLICATION", "FAKE SET SLAVE POSITION", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } diff --git a/go/vt/vttablet/tabletmanager/tm_init_test.go b/go/vt/vttablet/tabletmanager/tm_init_test.go index a3ca8971e61..8b41931e7a8 100644 --- a/go/vt/vttablet/tabletmanager/tm_init_test.go +++ b/go/vt/vttablet/tabletmanager/tm_init_test.go @@ -382,6 +382,7 @@ func TestCheckPrimaryShip(t *testing.T) { fakeMysql := tm.MysqlDaemon.(*fakemysqldaemon.FakeMysqlDaemon) fakeMysql.SetReplicationSourceInputs = append(fakeMysql.SetReplicationSourceInputs, fmt.Sprintf("%v:%v", otherTablet.MysqlHostname, otherTablet.MysqlPort)) fakeMysql.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } diff --git a/go/vt/wrangler/testlib/backup_test.go b/go/vt/wrangler/testlib/backup_test.go index b0b95c7021c..406400f5c5e 100644 --- a/go/vt/wrangler/testlib/backup_test.go +++ b/go/vt/wrangler/testlib/backup_test.go @@ -191,7 +191,8 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { }, } sourceTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", // This first set of STOP and START commands come from @@ -202,6 +203,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { // These commands come from SetReplicationSource RPC called // to set the correct primary and semi-sync after Backup has concluded "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -238,12 +240,14 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { }, } destTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "STOP SLAVE", "RESET SLAVE ALL", "FAKE SET SLAVE POSITION", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -296,6 +300,7 @@ func testBackupRestore(t *testing.T, cDetails *compressionDetails) error { "STOP SLAVE", "RESET SLAVE ALL", "FAKE SET SLAVE POSITION", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -423,7 +428,8 @@ func TestBackupRestoreLagged(t *testing.T) { } sourceTablet.FakeMysqlDaemon.SetReplicationSourceInputs = []string{fmt.Sprintf("%s:%d", primary.Tablet.MysqlHostname, primary.Tablet.MysqlPort)} sourceTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", // This first set of STOP and START commands come from @@ -434,6 +440,7 @@ func TestBackupRestoreLagged(t *testing.T) { // These commands come from SetReplicationSource RPC called // to set the correct primary and semi-sync after Backup has concluded "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -491,12 +498,14 @@ func TestBackupRestoreLagged(t *testing.T) { }, } destTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "STOP SLAVE", "RESET SLAVE ALL", "FAKE SET SLAVE POSITION", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -642,7 +651,8 @@ func TestRestoreUnreachablePrimary(t *testing.T) { } sourceTablet.FakeMysqlDaemon.SetReplicationSourceInputs = []string{fmt.Sprintf("%s:%d", primary.Tablet.MysqlHostname, primary.Tablet.MysqlPort)} sourceTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", // This first set of STOP and START commands come from @@ -653,6 +663,7 @@ func TestRestoreUnreachablePrimary(t *testing.T) { // These commands come from SetReplicationSource RPC called // to set the correct primary and semi-sync after Backup has concluded "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -682,12 +693,14 @@ func TestRestoreUnreachablePrimary(t *testing.T) { }, } destTablet.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "STOP SLAVE", "RESET SLAVE ALL", "FAKE SET SLAVE POSITION", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } diff --git a/go/vt/wrangler/testlib/copy_schema_shard_test.go b/go/vt/wrangler/testlib/copy_schema_shard_test.go index e20b21d1bee..e23e687e272 100644 --- a/go/vt/wrangler/testlib/copy_schema_shard_test.go +++ b/go/vt/wrangler/testlib/copy_schema_shard_test.go @@ -74,7 +74,8 @@ func copySchema(t *testing.T, useShardAsSource bool) { sourceRdonly := NewFakeTablet(t, wr, "cell1", 1, topodatapb.TabletType_RDONLY, sourceRdonlyDb, TabletKeyspaceShard(t, "ks", "-80")) sourceRdonly.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } diff --git a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go index 511a7d7ccb7..9c1f0282bc6 100644 --- a/go/vt/wrangler/testlib/emergency_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/emergency_reparent_shard_test.go @@ -137,11 +137,13 @@ func TestEmergencyReparentShard(t *testing.T) { goodReplica1.FakeMysqlDaemon.WaitPrimaryPositions = append(goodReplica1.FakeMysqlDaemon.WaitPrimaryPositions, goodReplica1.FakeMysqlDaemon.CurrentSourceFilePosition) goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "STOP SLAVE IO_THREAD", "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -167,9 +169,11 @@ func TestEmergencyReparentShard(t *testing.T) { goodReplica2.FakeMysqlDaemon.WaitPrimaryPositions = append(goodReplica2.FakeMysqlDaemon.WaitPrimaryPositions, goodReplica2.FakeMysqlDaemon.CurrentSourceFilePosition) goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", } goodReplica2.StartActionLoop(t, wr) @@ -232,6 +236,7 @@ func TestEmergencyReparentShardPrimaryElectNotBest(t *testing.T) { newPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE IO_THREAD", "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "CREATE DATABASE IF NOT EXISTS _vt", @@ -267,11 +272,13 @@ func TestEmergencyReparentShardPrimaryElectNotBest(t *testing.T) { moreAdvancedReplica.FakeMysqlDaemon.WaitPrimaryPositions = append(moreAdvancedReplica.FakeMysqlDaemon.WaitPrimaryPositions, moreAdvancedReplica.FakeMysqlDaemon.CurrentSourceFilePosition) newPrimary.FakeMysqlDaemon.WaitPrimaryPositions = append(newPrimary.FakeMysqlDaemon.WaitPrimaryPositions, moreAdvancedReplica.FakeMysqlDaemon.CurrentPrimaryPosition) moreAdvancedReplica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "STOP SLAVE IO_THREAD", "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } diff --git a/go/vt/wrangler/testlib/external_reparent_test.go b/go/vt/wrangler/testlib/external_reparent_test.go index edacaffd5e9..a0c065261a0 100644 --- a/go/vt/wrangler/testlib/external_reparent_test.go +++ b/go/vt/wrangler/testlib/external_reparent_test.go @@ -91,6 +91,7 @@ func TestTabletExternallyReparentedBasic(t *testing.T) { oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START Replica", } @@ -170,6 +171,7 @@ func TestTabletExternallyReparentedToReplica(t *testing.T) { // primary is still good to go. oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START Replica", } @@ -248,6 +250,7 @@ func TestTabletExternallyReparentedWithDifferentMysqlPort(t *testing.T) { oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START Replica", } @@ -260,7 +263,8 @@ func TestTabletExternallyReparentedWithDifferentMysqlPort(t *testing.T) { // TabletActionReplicaWasRestarted and point to the new mysql port goodReplica.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -335,6 +339,7 @@ func TestTabletExternallyReparentedContinueOnUnexpectedPrimary(t *testing.T) { oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START Replica", } @@ -347,7 +352,8 @@ func TestTabletExternallyReparentedContinueOnUnexpectedPrimary(t *testing.T) { // TabletActionReplicaWasRestarted and point to a bad host goodReplica.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -418,6 +424,7 @@ func TestTabletExternallyReparentedRerun(t *testing.T) { oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START Replica", } @@ -430,7 +437,8 @@ func TestTabletExternallyReparentedRerun(t *testing.T) { // On the good replica, we will respond to // TabletActionReplicaWasRestarted. goodReplica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } diff --git a/go/vt/wrangler/testlib/permissions_test.go b/go/vt/wrangler/testlib/permissions_test.go index 6783e8a2e72..1a023355d99 100644 --- a/go/vt/wrangler/testlib/permissions_test.go +++ b/go/vt/wrangler/testlib/permissions_test.go @@ -564,7 +564,8 @@ func TestPermissions(t *testing.T) { } replica.FakeMysqlDaemon.SetReplicationSourceInputs = append(replica.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet)) replica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } diff --git a/go/vt/wrangler/testlib/planned_reparent_shard_test.go b/go/vt/wrangler/testlib/planned_reparent_shard_test.go index 151a0b52294..4bf6141428f 100644 --- a/go/vt/wrangler/testlib/planned_reparent_shard_test.go +++ b/go/vt/wrangler/testlib/planned_reparent_shard_test.go @@ -79,6 +79,7 @@ func TestPlannedReparentShardNoPrimaryProvided(t *testing.T) { } newPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "CREATE DATABASE IF NOT EXISTS _vt", @@ -96,9 +97,11 @@ func TestPlannedReparentShardNoPrimaryProvided(t *testing.T) { oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0] oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", // we end up calling SetReplicationSource twice on the old primary + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -114,10 +117,12 @@ func TestPlannedReparentShardNoPrimaryProvided(t *testing.T) { goodReplica1.FakeMysqlDaemon.Replicating = true goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -194,6 +199,7 @@ func TestPlannedReparentShardNoError(t *testing.T) { } newPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "CREATE DATABASE IF NOT EXISTS _vt", @@ -211,9 +217,11 @@ func TestPlannedReparentShardNoError(t *testing.T) { oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0] oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", // we end up calling SetReplicationSource twice on the old primary + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -229,10 +237,12 @@ func TestPlannedReparentShardNoError(t *testing.T) { goodReplica1.FakeMysqlDaemon.Replicating = true goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -243,9 +253,11 @@ func TestPlannedReparentShardNoError(t *testing.T) { goodReplica2.FakeMysqlDaemon.ReadOnly = true goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", } goodReplica2.StartActionLoop(t, wr) @@ -332,6 +344,7 @@ func TestPlannedReparentInitialization(t *testing.T) { goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -344,6 +357,7 @@ func TestPlannedReparentInitialization(t *testing.T) { goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) goodReplica2.StartActionLoop(t, wr) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", } defer goodReplica2.StopActionLoop(t) @@ -414,6 +428,7 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { } newPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "CREATE DATABASE IF NOT EXISTS _vt", @@ -431,6 +446,7 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.PromoteResult oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -445,10 +461,12 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { goodReplica1.FakeMysqlDaemon.Replicating = true goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -459,9 +477,11 @@ func TestPlannedReparentShardWaitForPositionFail(t *testing.T) { goodReplica2.FakeMysqlDaemon.ReadOnly = true goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", } goodReplica2.StartActionLoop(t, wr) @@ -522,6 +542,7 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { } newPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "CREATE DATABASE IF NOT EXISTS _vt", @@ -538,6 +559,7 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0] oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -552,10 +574,12 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { goodReplica1.FakeMysqlDaemon.Replicating = true goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -566,9 +590,11 @@ func TestPlannedReparentShardWaitForPositionTimeout(t *testing.T) { goodReplica2.FakeMysqlDaemon.ReadOnly = true goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", } goodReplica2.StartActionLoop(t, wr) @@ -629,7 +655,8 @@ func TestPlannedReparentShardRelayLogError(t *testing.T) { goodReplica1.FakeMysqlDaemon.Replicating = true goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", // simulate error that will trigger a call to RestartReplication @@ -713,7 +740,8 @@ func TestPlannedReparentShardRelayLogErrorStartReplication(t *testing.T) { goodReplica1.FakeMysqlDaemon.CurrentSourcePort = int(primary.Tablet.MysqlPort) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ // simulate error that will trigger a call to RestartReplication - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", // In SetReplicationSource, we find that the source host and port was already set correctly, @@ -792,6 +820,7 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { } newPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "CREATE DATABASE IF NOT EXISTS _vt", @@ -809,6 +838,7 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { oldPrimary.FakeMysqlDaemon.CurrentPrimaryPosition = newPrimary.FakeMysqlDaemon.WaitPrimaryPositions[0] oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs = append(oldPrimary.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet)) oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -823,10 +853,12 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { goodReplica1.FakeMysqlDaemon.Replicating = true goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -837,9 +869,11 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { goodReplica2.FakeMysqlDaemon.ReadOnly = true goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(newPrimary.Tablet), topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", } goodReplica2.StartActionLoop(t, wr) @@ -860,10 +894,12 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { newPrimary.FakeMysqlDaemon.PromoteError = nil newPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", // extra commands because of retry "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "CREATE DATABASE IF NOT EXISTS _vt", @@ -872,9 +908,11 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) { "SUBINSERT INTO _vt.reparent_journal (time_created_ns, action_name, primary_alias, replication_position) VALUES", } oldPrimary.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", // extra commands because of retry + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -936,10 +974,12 @@ func TestPlannedReparentShardSamePrimary(t *testing.T) { goodReplica1.FakeMysqlDaemon.Replicating = true goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica1.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica1.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -950,9 +990,11 @@ func TestPlannedReparentShardSamePrimary(t *testing.T) { goodReplica2.FakeMysqlDaemon.ReadOnly = true goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs = append(goodReplica2.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(oldPrimary.Tablet)) goodReplica2.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", } goodReplica2.StartActionLoop(t, wr) diff --git a/go/vt/wrangler/testlib/reparent_utils_test.go b/go/vt/wrangler/testlib/reparent_utils_test.go index 09dc4544a62..315ebce9638 100644 --- a/go/vt/wrangler/testlib/reparent_utils_test.go +++ b/go/vt/wrangler/testlib/reparent_utils_test.go @@ -91,7 +91,8 @@ func TestShardReplicationStatuses(t *testing.T) { replica.FakeMysqlDaemon.CurrentSourcePort = int(primary.Tablet.MysqlPort) replica.FakeMysqlDaemon.SetReplicationSourceInputs = append(replica.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet)) replica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } @@ -159,10 +160,12 @@ func TestReparentTablet(t *testing.T) { replica.FakeMysqlDaemon.IOThreadRunning = true replica.FakeMysqlDaemon.SetReplicationSourceInputs = append(replica.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(primary.Tablet)) replica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", "STOP SLAVE", + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", } diff --git a/go/vt/wrangler/testlib/version_test.go b/go/vt/wrangler/testlib/version_test.go index f06a98aa688..fbd8610e437 100644 --- a/go/vt/wrangler/testlib/version_test.go +++ b/go/vt/wrangler/testlib/version_test.go @@ -93,7 +93,8 @@ func TestVersion(t *testing.T) { sourceReplicaGitRev := "fake git rev" sourceReplica.FakeMysqlDaemon.SetReplicationSourceInputs = append(sourceReplica.FakeMysqlDaemon.SetReplicationSourceInputs, topoproto.MysqlAddr(sourcePrimary.Tablet)) sourceReplica.FakeMysqlDaemon.ExpectedExecuteSuperQueryList = []string{ - // These 2 statements come from tablet startup + // These 3 statements come from tablet startup + "RESET SLAVE ALL", "FAKE SET MASTER", "START SLAVE", }