TestRandomKillTSOServer

Signed-off-by: lhy1024 <[email protected]>

tests/compatibility/forward_test.go

@@ -16,6 +16,7 @@ package compatibility_test
 
 import (
 	"context"
+	"math/rand"
 	"os"
 	"strconv"
 	"testing"
@@ -143,6 +144,47 @@ func (suite *APIServerForwardTestSuite) TestForwardTSOWhenPrimaryChanged() {
 	suite.checkAvailableTSO()
 }
 
+func (suite *APIServerForwardTestSuite) TestRandomKillTSOServer() {
+	serverMap := make(map[string]bs.Server)
+	for i := 0; i < 3; i++ {
+		s, cleanup := startSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc())
+		defer cleanup()
+		serverMap[s.GetAddr()] = s
+	}
+
+	ticker := time.NewTicker(500 * time.Millisecond)
+	for j := 0; j < 20; j++ {
+		select {
+		case <-suite.ctx.Done():
+			return
+		case <-ticker.C:
+			r := rand.Intn(2)
+			if r == 0 && len(serverMap) > 1 {
+				expectedPrimary := waitForPrimaryServing(suite.Require(), serverMap)
+				primary, exist := suite.pdLeader.GetServer().GetServicePrimaryAddr(suite.ctx, "tso")
+				suite.True(exist)
+				suite.Equal(expectedPrimary, primary)
+				serverMap[primary].Close()
+				delete(serverMap, primary)
+			} else {
+				s, cleanup := startSingleTSOTestServer(suite.ctx, suite.Require(), suite.backendEndpoints, tempurl.Alloc())
+				defer cleanup()
+				serverMap[s.GetAddr()] = s
+			}
+			waitForPrimaryServing(suite.Require(), serverMap)
+			var err error
+			for i := 0; i < 3; i++ { // try 3 times
+				_, _, err = suite.pdClient.GetTS(suite.ctx)
+				if err == nil {
+					break
+				}
+				time.Sleep(100 * time.Millisecond)
+			}
+			suite.NoError(err)
+		}
+	}
+}
+
 func (suite *APIServerForwardTestSuite) TestForwardTSOUnexpectedToFollower() {
 	serverMap := make(map[string]bs.Server)
 	for i := 0; i < 3; i++ {
@@ -237,15 +279,18 @@ func (suite *APIServerForwardTestSuite) checkAvailableTSO() {
 }
 
 // waitForPrimaryServing waits for one of servers being elected to be the primary/leader
-func waitForPrimaryServing(re *require.Assertions, serverMap map[string]bs.Server) {
+func waitForPrimaryServing(re *require.Assertions, serverMap map[string]bs.Server) string {
+	var primary string
 	testutil.Eventually(re, func() bool {
-		for _, s := range serverMap {
+		for name, s := range serverMap {
 			if s.IsServing() {
+				primary = name
 				return true
 			}
 		}
 		return false
 	}, testutil.WithWaitFor(5*time.Second), testutil.WithTickInterval(50*time.Millisecond))
+	return primary
 }
 
 // startSingleTSOTestServer creates and starts a tso server with default config for testing.