Skip to content

Commit

Permalink
Flaky tests: Fix wrangler tests (#13568)
Browse files Browse the repository at this point in the history
Signed-off-by: Manan Gupta <[email protected]>
  • Loading branch information
GuptaManan100 authored Jul 20, 2023
1 parent 0e7b441 commit 138d540
Show file tree
Hide file tree
Showing 4 changed files with 64 additions and 20 deletions.
4 changes: 4 additions & 0 deletions go/vt/wrangler/testlib/copy_schema_shard_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -155,6 +155,10 @@ func copySchema(t *testing.T, useShardAsSource bool) {
if useShardAsSource {
source = "ks/-80"
}

// PrimaryAlias in the shard record is updated asynchronously, so we should wait for it to succeed.
waitForShardPrimary(t, wr, destinationPrimary.Tablet)

if err := vp.Run([]string{"CopySchemaShard", "--include-views", source, "ks/-40"}); err != nil {
t.Fatalf("CopySchemaShard failed: %v", err)
}
Expand Down
3 changes: 3 additions & 0 deletions go/vt/wrangler/testlib/external_reparent_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -550,6 +550,9 @@ func TestRPCTabletExternallyReparentedDemotesPrimaryToConfiguredTabletType(t *te
}
}

// PrimaryAlias in the shard record is updated asynchronously, so we should wait for it to succeed.
waitForShardPrimary(t, wr, newPrimary.Tablet)

shardInfo, err := ts.GetShard(context.Background(), newPrimary.Tablet.Keyspace, newPrimary.Tablet.Shard)
assert.NoError(t, err)

Expand Down
20 changes: 0 additions & 20 deletions go/vt/wrangler/testlib/planned_reparent_shard_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -878,26 +878,6 @@ func TestPlannedReparentShardPromoteReplicaFail(t *testing.T) {
assert.True(t, oldPrimary.FakeMysqlDaemon.ReadOnly, "oldPrimary.FakeMysqlDaemon.ReadOnly")
}

// waitForTabletType waits for the given tablet type to be reached.
func waitForTabletType(t *testing.T, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias, tabletType topodatapb.TabletType) {
timeout := time.After(15 * time.Second)
for {
tablet, err := wr.TopoServer().GetTablet(context.Background(), tabletAlias)
require.NoError(t, err)
if tablet.Type == tabletType {
return
}

select {
case <-timeout:
t.Fatalf("%s didn't reach the tablet type %v", topoproto.TabletAliasString(tabletAlias), tabletType.String())
return
default:
time.Sleep(100 * time.Millisecond)
}
}
}

// TestPlannedReparentShardSamePrimary tests PRS with oldPrimary works correctly
// Simulate failure of previous PRS and oldPrimary is ReadOnly
// Verify that primary correctly gets set to ReadWrite
Expand Down
57 changes: 57 additions & 0 deletions go/vt/wrangler/testlib/utils.go
Original file line number Diff line number Diff line change
@@ -0,0 +1,57 @@
package testlib

import (
"context"
"testing"
"time"

"github.com/stretchr/testify/require"

topodatapb "vitess.io/vitess/go/vt/proto/topodata"
"vitess.io/vitess/go/vt/topo/topoproto"
"vitess.io/vitess/go/vt/wrangler"
)

// waitForTabletType waits for the given tablet type to be reached.
func waitForTabletType(t *testing.T, wr *wrangler.Wrangler, tabletAlias *topodatapb.TabletAlias, tabletType topodatapb.TabletType) {
timeout := time.After(15 * time.Second)
for {
tablet, err := wr.TopoServer().GetTablet(context.Background(), tabletAlias)
require.NoError(t, err)
if tablet.Type == tabletType {
return
}

select {
case <-timeout:
t.Fatalf("%s didn't reach the tablet type %v", topoproto.TabletAliasString(tabletAlias), tabletType.String())
return
default:
time.Sleep(100 * time.Millisecond)
}
}
}

// waitForShardPrimary waits for the shard record to be upto date such that it has the given primary.
func waitForShardPrimary(t *testing.T, wr *wrangler.Wrangler, primaryTablet *topodatapb.Tablet) {
timeout := time.After(15 * time.Second)
for {
si, err := wr.TopoServer().GetShard(context.Background(), primaryTablet.Keyspace, primaryTablet.Shard)
require.NoError(t, err)
if topoproto.TabletAliasEqual(si.PrimaryAlias, primaryTablet.Alias) {
return
}

select {
case <-timeout:
t.Fatalf("%s/%s didn't see the tablet %v become the primary, instead it is %v",
primaryTablet.Keyspace, primaryTablet.Shard,
topoproto.TabletAliasString(primaryTablet.Alias),
topoproto.TabletAliasString(si.PrimaryAlias),
)
return
default:
time.Sleep(100 * time.Millisecond)
}
}
}

0 comments on commit 138d540

Please sign in to comment.