mcs: make scheduling server test stable

pkg/utils/testutil/api_check.go

@@ -114,6 +114,21 @@
 	return checkResp(resp, checkOpts...)
 }
 
+// CheckGetUntilStatusCode is used to do get request and do check options.
+func CheckGetUntilStatusCode(re *require.Assertions, client *http.Client, url string, code int) error {
+	var err error
+	Eventually(re, func() bool {
+		resp, err2 := apiutil.GetJSON(client, url, nil)
+		if err2 != nil {
+			err = err2
+			return true
+		}
+		defer resp.Body.Close()
+		return resp.StatusCode == code
+	})
+	return err
+}
+
 // CheckPatchJSON is used to do patch request and do check options.
 func CheckPatchJSON(client *http.Client, url string, data []byte, checkOpts ...func([]byte, int, http.Header)) error {
 	resp, err := apiutil.PatchJSON(client, url, data)

server/api/diagnostic_test.go

@@ -17,6 +17,7 @@ package api
 import (
 	"encoding/json"
 	"fmt"
+	"net/http"
 	"testing"
 	"time"
 
@@ -63,6 +64,8 @@ func (suite *diagnosticTestSuite) TearDownSuite() {
 
 func (suite *diagnosticTestSuite) checkStatus(status string, url string) {
 	re := suite.Require()
+	err := tu.CheckGetUntilStatusCode(re, testDialClient, url, http.StatusOK)
+	suite.NoError(err)
 	suite.Eventually(func() bool {
 		result := &schedulers.DiagnosticResult{}
 		err := tu.ReadGetJSON(re, testDialClient, url, result)

tests/pdctl/config/config_test.go

@@ -431,14 +431,16 @@ func (suite *configTestSuite) checkPlacementRuleGroups(cluster *tests.TestCluste
 
 	// show all
 	var groups []placement.RuleGroup
-	output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show")
-	re.NoError(err)
-	re.NoError(json.Unmarshal(output, &groups))
-	re.Equal([]placement.RuleGroup{
-		{ID: "pd", Index: 42, Override: true},
-		{ID: "group2", Index: 100, Override: false},
-		{ID: "group3", Index: 200, Override: false},
-	}, groups)
+	testutil.Eventually(re, func() bool { // wait for the config to be synced to the scheduling server
+		output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "show")
+		re.NoError(err)
+		re.NoError(json.Unmarshal(output, &groups))
+		return reflect.DeepEqual([]placement.RuleGroup{
+			{ID: "pd", Index: 42, Override: true},
+			{ID: "group2", Index: 100, Override: false},
+			{ID: "group3", Index: 200, Override: false},
+		}, groups)
+	})
 
 	// delete
 	output, err = pdctl.ExecuteCommand(cmd, "-u", pdAddr, "config", "placement-rules", "rule-group", "delete", "group2")

tests/server/api/rule_test.go

@@ -210,8 +210,10 @@ func (suite *ruleTestSuite) checkGet(cluster *tests.TestCluster) {
 		var resp placement.Rule
 		url := fmt.Sprintf("%s/rule/%s/%s", urlPrefix, testCase.rule.GroupID, testCase.rule.ID)
 		if testCase.found {
-			err = tu.ReadGetJSON(re, testDialClient, url, &resp)
-			suite.compareRule(&resp, &testCase.rule)
+			tu.Eventually(suite.Require(), func() bool {
+				err = tu.ReadGetJSON(re, testDialClient, url, &resp)
+				return suite.compareRule(&resp, &testCase.rule)
+			})
 		} else {
 			err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, testCase.code))
 		}
@@ -421,13 +423,17 @@ func (suite *ruleTestSuite) checkGetAllByGroup(cluster *tests.TestCluster) {
 		suite.T().Log(testCase.name)
 		var resp []*placement.Rule
 		url := fmt.Sprintf("%s/rules/group/%s", urlPrefix, testCase.groupID)
-		err = tu.ReadGetJSON(re, testDialClient, url, &resp)
-		suite.NoError(err)
-		suite.Len(resp, testCase.count)
-		if testCase.count == 2 {
-			suite.compareRule(resp[0], &rule)
-			suite.compareRule(resp[1], &rule1)
-		}
+		tu.Eventually(re, func() bool {
+			err = tu.ReadGetJSON(re, testDialClient, url, &resp)
+			suite.NoError(err)
+			if len(resp) != testCase.count {
+				return false
+			}
+			if testCase.count == 2 {
+				return suite.compareRule(resp[0], &rule) && suite.compareRule(resp[1], &rule1)
+			}
+			return true
+		})
 	}
 }
 
@@ -487,12 +493,15 @@ func (suite *ruleTestSuite) checkGetAllByRegion(cluster *tests.TestCluster) {
 		url := fmt.Sprintf("%s/rules/region/%s", urlPrefix, testCase.regionID)
 
 		if testCase.success {
-			err = tu.ReadGetJSON(re, testDialClient, url, &resp)
-			for _, r := range resp {
-				if r.GroupID == "e" {
-					suite.compareRule(r, &rule)
+			tu.Eventually(suite.Require(), func() bool {
+				err = tu.ReadGetJSON(re, testDialClient, url, &resp)
+				for _, r := range resp {
+					if r.GroupID == "e" {
+						return suite.compareRule(r, &rule)
+					}
 				}
-			}
+				return true
+			})
 		} else {
 			err = tu.CheckGetJSON(testDialClient, url, nil, tu.Status(re, testCase.code))
 		}
@@ -956,22 +965,26 @@ func (suite *ruleTestSuite) checkBundleBadRequest(cluster *tests.TestCluster) {
 }
 
 func (suite *ruleTestSuite) compareBundle(b1, b2 placement.GroupBundle) {
-	suite.Equal(b2.ID, b1.ID)
-	suite.Equal(b2.Index, b1.Index)
-	suite.Equal(b2.Override, b1.Override)
-	suite.Len(b2.Rules, len(b1.Rules))
-	for i := range b1.Rules {
-		suite.compareRule(b1.Rules[i], b2.Rules[i])
-	}
+	tu.Eventually(suite.Require(), func() bool {
+		if b2.ID != b1.ID || b2.Index != b1.Index || b2.Override != b1.Override || len(b2.Rules) != len(b1.Rules) {
+			return false
+		}
+		for i := range b1.Rules {
+			if !suite.compareRule(b1.Rules[i], b2.Rules[i]) {
+				return false
+			}
+		}
+		return true
+	})
 }
 
-func (suite *ruleTestSuite) compareRule(r1 *placement.Rule, r2 *placement.Rule) {
-	suite.Equal(r2.GroupID, r1.GroupID)
-	suite.Equal(r2.ID, r1.ID)
-	suite.Equal(r2.StartKeyHex, r1.StartKeyHex)
-	suite.Equal(r2.EndKeyHex, r1.EndKeyHex)
-	suite.Equal(r2.Role, r1.Role)
-	suite.Equal(r2.Count, r1.Count)
+func (suite *ruleTestSuite) compareRule(r1 *placement.Rule, r2 *placement.Rule) bool {
+	return r2.GroupID == r1.GroupID &&
+		r2.ID == r1.ID &&
+		r2.StartKeyHex == r1.StartKeyHex &&
+		r2.EndKeyHex == r1.EndKeyHex &&
+		r2.Role == r1.Role &&
+		r2.Count == r1.Count
 }
 
 type regionRuleTestSuite struct {