wip: test server restart with/without leave_on_terminate

acceptance/tests/basic/basic_test.go

@@ -5,13 +5,16 @@ package basic
 
 import (
 	"context"
+	"encoding/json"
 	"fmt"
 	"strconv"
 	"strings"
 	"testing"
+	"time"
 
 	"github.com/hashicorp/consul-k8s/acceptance/framework/consul"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/helpers"
+	"github.com/hashicorp/consul-k8s/acceptance/framework/k8s"
 	"github.com/hashicorp/consul-k8s/acceptance/framework/logger"
 	"github.com/hashicorp/consul/api"
 	"github.com/stretchr/testify/require"
@@ -96,3 +99,73 @@ func TestBasicInstallation(t *testing.T) {
 		})
 	}
 }
+
+// Test that when servers are restarted, they don't lose quorum.
+func TestServerRestart(t *testing.T) {
+	cfg := suite.Config()
+	ctx := suite.Environment().DefaultContext(t)
+	if cfg.EnableCNI {
+		t.Skipf("skipping because -enable-cni is set and installing CNI is not a basic installation")
+	}
+	replicas := 3
+	releaseName := helpers.RandomName()
+	helmValues := map[string]string{
+		"global.enabled":        "false",
+		"connectInject.enabled": "false",
+		"server.enabled":        "true",
+		"server.replicas":       fmt.Sprintf("%d", replicas),
+		"server.extraConfig":    "\"{\\\"leave_on_terminate\\\": true}\"",
+	}
+	consulCluster := consul.NewHelmCluster(t, helmValues, suite.Environment().DefaultContext(t), suite.Config(), releaseName)
+
+	consulCluster.Create(t)
+
+	// Start a separate goroutine to check if at any point quorum is lost.
+	// Use number of ready replicas as proxy for whether there is quorum because
+	// replicas are marked unready if they don't think there's a leader so if
+	// >n/2 replicas are unready then there's no quorum.
+	noQuorumCount := 0
+	done := make(chan bool)
+	go func() {
+		for {
+			select {
+			case <-done:
+				return
+			default:
+				out, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "get", fmt.Sprintf("statefulset/%s-consul-server", releaseName),
+					"-o", "jsonpath={.status}")
+				if err != nil {
+					logger.Logf(t, "kubectl err: %s: %s", err, out)
+					break
+				}
+				type statefulsetOut struct {
+					ReadyReplicas *int `json:"readyReplicas,omitempty"`
+				}
+				jsonOut := statefulsetOut{}
+				err = json.Unmarshal([]byte(out), &jsonOut)
+				if err != nil {
+					logger.Logf(t, "err unmarshalling: %s: '%s'", err, out)
+				} else if jsonOut.ReadyReplicas == nil || *jsonOut.ReadyReplicas < replicas-1 {
+					// note: if readyReplicas is 0 it's not included in the json output so
+					// that's why we're checking if it's nil.
+					logger.Logf(t, "ready replicas: %v", jsonOut.ReadyReplicas)
+					noQuorumCount++
+				}
+				time.Sleep(1 * time.Second)
+			}
+		}
+	}()
+
+	// Restart servers
+	out, err := k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "rollout", "restart", fmt.Sprintf("statefulset/%s-consul-server", releaseName))
+	require.NoError(t, err, out)
+
+	// Wait for restart to finish.
+	start := time.Now()
+	out, err = k8s.RunKubectlAndGetOutputE(t, ctx.KubectlOptions(t), "rollout", "status", "--watch", fmt.Sprintf("statefulset/%s-consul-server", releaseName))
+	require.NoError(t, err, out)
+	close(done)
+
+	logger.Logf(t, "restart took %s, there were %d seconds without quorum", time.Now().Sub(start), noQuorumCount)
+	require.Equal(t, 0, noQuorumCount)
+}