Merge pull request #452 from zalando-incubator/bugfix-single-shard-pe…

…r-node-scaledown

Bugfix: Allow scale-down when at one-shard-per-node

cmd/e2e/test_utils.go

@@ -274,10 +274,12 @@ func waitForSTSCondition(t *testing.T, stsName string, conditions ...func(sts *a
 
 func createEDS(name string, spec zv1.ElasticsearchDataSetSpec) error {
 	myspec := spec.DeepCopy()
-	myspec.Template.Spec.Containers[0].Env = append(myspec.Template.Spec.Containers[0].Env, v1.EnvVar{
-		Name:  "node.attr.group",
-		Value: name,
-	})
+	for i, env := range myspec.Template.Spec.Containers[0].Env {
+		if env.Name == "node.attr.group" {
+			myspec.Template.Spec.Containers[0].Env[i].Value = name
+			break
+		}
+	}
 	eds := &zv1.ElasticsearchDataSet{
 		ObjectMeta: metav1.ObjectMeta{
 			Name:      name,

e2e.sh

@@ -2,7 +2,7 @@
 
 # Simple script for running the e2e test from your local development machine
 # against "some" cluster.
-# You need to run `kubectl proxy` in a different terminat before running the
+# You need to run `kubectl proxy` in a different terminal before running the
 # script.
 
 NAMESPACE="${NAMESPACE:-"es-operator-e2e-$(date +%s)"}"

operator/autoscaler.go

@@ -382,10 +382,8 @@ func shardToNodeRatio(shards, nodes int32) float64 {
 }
 
 func calculateNodesWithSameShardToNodeRatio(currentDesiredNodeReplicas, currentTotalShards, newTotalShards int32) int32 {
-	currentShardToNodeRatio := shardToNodeRatio(currentTotalShards, currentDesiredNodeReplicas)
-	if currentShardToNodeRatio <= 1 {
-		return currentDesiredNodeReplicas
-	}
+	// reconcile shardToNodeRatio to not become below 1
+	currentShardToNodeRatio := math.Max(shardToNodeRatio(currentTotalShards, currentDesiredNodeReplicas), 1)
 	return int32(math.Ceil(float64(newTotalShards) / float64(currentShardToNodeRatio)))
 }
 

operator/autoscaler_test.go

@@ -304,6 +304,31 @@ func TestScaleDownByRemovingIndexReplica(t *testing.T) {
 	require.Equal(t, DOWN, actual.ScalingDirection, actual.Description)
 }
 
+func TestScaleDownByRemovingIndexReplicaWhenHavingOneShardPerNodeBeforeScaleDown(t *testing.T) {
+	eds := edsTestFixture(30)
+	eds.Spec.Scaling.MinReplicas = 5
+	eds.Spec.Scaling.MaxReplicas = 30
+	eds.Spec.Scaling.MinShardsPerNode = 1
+	eds.Spec.Scaling.MaxShardsPerNode = 6
+	eds.Spec.Scaling.MaxIndexReplicas = 4
+	eds.Spec.Scaling.MinIndexReplicas = 2
+
+	esNodes := make([]ESNode, 0)
+
+	// scale down by removing an index replica
+	esIndices := map[string]ESIndex{
+		"ad1": {Replicas: 4, Primaries: 6, Index: "ad1"},
+	}
+	scalingHint := DOWN
+
+	as := systemUnderTest(eds, nil, nil)
+
+	actual := as.calculateScalingOperation(esIndices, esNodes, scalingHint)
+	require.Equal(t, int32(24), *actual.NodeReplicas, actual.Description)
+	require.Equal(t, int32(3), actual.IndexReplicas[0].Replicas, actual.Description)
+	require.Equal(t, DOWN, actual.ScalingDirection, actual.Description)
+}
+
 func TestAtMaxIndexReplicas(t *testing.T) {
 	eds := edsTestFixture(4)
 	esNodes := make([]ESNode, 0)
@@ -354,14 +379,16 @@ func TestScaleUpCausedByShardToNodeRatioLessThanOne(t *testing.T) {
 		"ad1": {Replicas: 1, Primaries: 5, Index: "ad1"},
 	}
 	// calculated ShardToNode ratio is 10/11 ~ 0.9
+	// this should get reconciled to ShardToNode ratio of 1.
 	eds.Spec.Scaling.MinShardsPerNode = 1
 	// scaling independent of desired scaling direction
 	scalingHint := UP
 
 	as := systemUnderTest(eds, nil, nil)
 
 	actual := as.calculateScalingOperation(esIndices, esNodes, scalingHint)
-	require.Equal(t, int32(11), *actual.NodeReplicas, actual.Description)
+	require.Contains(t, actual.IndexReplicas, ESIndex{Index: "ad1", Primaries: 5, Replicas: 2})
+	require.Equal(t, int32(15), *actual.NodeReplicas, actual.Description)
 	require.Equal(t, 1, len(actual.IndexReplicas), actual.Description)
 	require.Equal(t, UP, actual.ScalingDirection, actual.Description)
 }
@@ -610,8 +637,8 @@ func TestCalculateDecreaseNodes(t *testing.T) {
 }
 
 func TestCalculateNodesWithSameShardToNodeRatio(t *testing.T) {
-	require.Equal(t, 16, int(calculateNodesWithSameShardToNodeRatio(16, 4, 4)))
-	require.Equal(t, 16, int(calculateNodesWithSameShardToNodeRatio(16, 4, 6)))
+	require.Equal(t, 4, int(calculateNodesWithSameShardToNodeRatio(16, 4, 4)))
+	require.Equal(t, 6, int(calculateNodesWithSameShardToNodeRatio(16, 4, 6)))
 	require.Equal(t, 12, int(calculateNodesWithSameShardToNodeRatio(16, 32, 24)))
 	require.Equal(t, 17, int(calculateNodesWithSameShardToNodeRatio(17, 32, 32)))
 }