Availability zone support for k8s

docs/clusterdefinition.md

@@ -490,6 +490,8 @@ A cluster can have 0 to 12 agent pool profiles. Agent Pool Profiles are used for
 | ---------------------------- | -------------------------------------------------------------------- | -------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- |
 | availabilityProfile          | no                                                                   | Supported values are `VirtualMachineScaleSets` (default, except for Kubernetes clusters before version 1.10) and `AvailabilitySet`.                                                                                                                                                                                                                                                                                                                                                                                              |
 | count                        | yes                                                                  | Describes the node count                                                                                                                                                                                                                                                                                                                                                                                                                                                                                                         |
+| availabilityZones                    | no                                       | To protect your cluster from datacenter-level failures, you can provide Availability Zones for each of your agentPool. Only applies to Kubernetes clusters version 1.12+. Supported values are arrays of strings, each representing a supported availability zone in a region for your subscription. e.g. `"availabilityZones": ["1","2"]` represents zone 1 and zone 2 can be used. To get supported zones for a region in your subscription, run `az vm list-skus --location centralus --query "[?name=='Standard_DS2_v2'].[locationInfo, restrictions"] -o table`. You should see values like `'zones': ['2', '3', '1']` appear in the first column. If `NotAvailableForSubscription` appears in the output, then you need to create an Azure support ticket to enable zones for that region. Note: For availability zones, only standard load balancer is supported. ([Availability zone example](../examples/e2e-tests/kubernetes/zones)).                                                                                                                                                                                                                                                  |
+| singlePlacementGroup             | no                                                                   | Supported values are `true` (default) and `false`. Only applies to clusters with availabilityProfile `VirtualMachineScaleSets`. `true`: A VMSS with a single placement group and has a range of 0-100 VMs. `false`: A VMSS with multiple placement groups and has a range of 0-1,000 VMs. For more information, check out [virtual machine scale sets placement groups](https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-placement-groups).                                                                                                                                                                                                                           |
 | scaleSetPriority             | no                                                                   | Supported values are `Regular` (default) and `Low`. Only applies to clusters with availabilityProfile `VirtualMachineScaleSets`. Enables the usage of [Low-priority VMs on Scale Sets](https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-use-low-priority).                                                                                                                                                                                                                           |
 | scaleSetEvictionPolicy       | no                                                                   | Supported values are `Delete` (default) and `Deallocate`. Only applies to clusters with availabilityProfile of `VirtualMachineScaleSets` and scaleSetPriority of `Low`.                                                                                                                                                                                                                                                                                                                                                          |
 | diskSizesGB                  | no                                                                   | Describes an array of up to 4 attached disk sizes. Valid disk size values are between 1 and 1024                                                                                                                                                                                                                                                                                                                                                                                                                                 |

examples/e2e-tests/kubernetes/zones/definition.json

@@ -0,0 +1,39 @@
+{
+    "apiVersion": "vlabs",
+    "properties": {
+      "orchestratorProfile": {
+        "orchestratorType": "Kubernetes",
+        "orchestratorRelease": "1.12"
+      },
+      "masterProfile": {
+        "count": 1,
+        "dnsPrefix": "",
+        "vmSize": "Standard_DS2_v2"
+      },
+      "agentPoolProfiles": [
+        {
+            "name": "agentpool",
+            "count": 4,
+            "vmSize": "Standard_DS2_v2",
+            "availabilityZones": [
+                "1",
+                "2"
+            ]
+        }
+      ],
+      "linuxProfile": {
+        "adminUsername": "azureuser",
+        "ssh": {
+          "publicKeys": [
+            {
+              "keyData": ""
+            }
+          ]
+        }
+      },
+      "servicePrincipalProfile": {
+        "clientId": "",
+        "secret": ""
+      }
+    }
+  }

parts/agentparams.t

@@ -48,6 +48,14 @@
       },
       "type": "string"
     },
+{{if HasAvailabilityZones .}}
+    "{{.Name}}AvailabilityZones": {
+      "metadata": {
+        "description": "Agent availability zones"
+      },
+      "type": "array"
+    },
+{{end}}
     "{{.Name}}osImageName": {
       "defaultValue": "",
       "metadata": {

parts/k8s/kubernetesagentresourcesvmss.t

@@ -26,6 +26,9 @@
       "poolName" : "{{.Name}}"
     },
     "location": "[variables('location')]",
+    {{ if HasAvailabilityZones .}}
+    "zones": "[parameters('{{.Name}}AvailabilityZones')]",
+    {{ end }}
     "name": "[variables('{{.Name}}VMNamePrefix')]",
     {{if UseManagedIdentity}}
     "identity": {
@@ -38,6 +41,7 @@
       "name": "[variables('{{.Name}}VMSize')]"
     },
     "properties": {
+      "singlePlacementGroup": {{UseSinglePlacementGroup .}},
       "overprovision": false,
       "upgradePolicy": {
         "mode": "Manual"

parts/k8s/kubernetesconfigs.sh

@@ -280,7 +280,13 @@ function ensureK8sControlPlane() {
         return
     fi
     wait_for_file 600 1 $KUBECTL || exit $ERR_FILE_WATCH_TIMEOUT
-    retrycmd_if_failure 900 1 20 $KUBECTL 2>/dev/null cluster-info || exit $ERR_K8S_RUNNING_TIMEOUT
+    # workaround for 1.12 bug https://github.com/Azure/acs-engine/issues/3681 will remove once upstream is fixed
+    if [[ "${KUBERNETES_VERSION}" = 1.12.* ]]; then
+        ensureKubelet 
+        retrycmd_if_failure 900 1 20 $KUBECTL 2>/dev/null cluster-info || ensureKubelet && retrycmd_if_failure 900 1 20 $KUBECTL 2>/dev/null cluster-info || exit $ERR_K8S_RUNNING_TIMEOUT
+    else
+        retrycmd_if_failure 900 1 20 $KUBECTL 2>/dev/null cluster-info || exit $ERR_K8S_RUNNING_TIMEOUT
+    fi
     ensurePodSecurityPolicy
 }
 

parts/k8s/kuberneteswinagentresourcesvmss.t

@@ -26,6 +26,9 @@
       "poolName" : "{{.Name}}"
     },
     "location": "[variables('location')]",
+    {{ if HasAvailabilityZones .}}
+    "zones": "[parameters('{{.Name}}AvailabilityZones')]",
+    {{ end }}
     "name": "[variables('{{.Name}}VMNamePrefix')]",
     {{if UseManagedIdentity}}
     "identity": {
@@ -38,6 +41,7 @@
       "name": "[variables('{{.Name}}VMSize')]"
     },
     "properties": {
+      "singlePlacementGroup": {{UseSinglePlacementGroup .}},
       "overprovision": false,
       "upgradePolicy": {
         "mode": "Manual"

pkg/acsengine/defaults.go

@@ -241,6 +241,7 @@ func setPropertiesDefaults(cs *api.ContainerService, isUpgrade, isScale bool) (b
 
 	setStorageDefaults(properties)
 	setExtensionDefaults(properties)
+	setVMSSDefaults(properties)
 
 	certsGenerated, e := setDefaultCerts(properties)
 	if e != nil {
@@ -559,6 +560,28 @@ func setMasterNetworkDefaults(a *api.Properties, isUpgrade bool) {
 	}
 }
 
+// setVMSSDefaults
+func setVMSSDefaults(a *api.Properties) {
+	for _, profile := range a.AgentPoolProfiles {
+		if profile.AvailabilityProfile == api.VirtualMachineScaleSets {
+			if profile.Count > 100 {
+				profile.SinglePlacementGroup = helpers.PointerToBool(false)
+			}
+			if profile.SinglePlacementGroup == nil {
+				profile.SinglePlacementGroup = helpers.PointerToBool(api.DefaultSinglePlacementGroup)
+			}
+			if profile.SinglePlacementGroup == helpers.PointerToBool(false) {
+				profile.StorageProfile = api.ManagedDisks
+			}
+			if profile.HasAvailabilityZones() {
+				a.OrchestratorProfile.KubernetesConfig.LoadBalancerSku = "Standard"
+				a.OrchestratorProfile.KubernetesConfig.ExcludeMasterFromStandardLB = helpers.PointerToBool(api.DefaultExcludeMasterFromStandardLB)
+			}
+		}
+
+	}
+}
+
 // SetAgentNetworkDefaults for agents
 func setAgentNetworkDefaults(a *api.Properties, isUpgrade, isScale bool) {
 	// configure the subnets if not in custom VNET

pkg/acsengine/defaults_test.go

@@ -588,6 +588,48 @@ func TestIsAzureCNINetworkmonitorAddon(t *testing.T) {
 	}
 }
 
+// TestSetVMSSDefaults covers tests for setVMSSDefaults
+func TestSetVMSSDefaults(t *testing.T) {
+	mockCS := getMockBaseContainerService("1.10.3")
+	properties := mockCS.Properties
+	properties.OrchestratorProfile.OrchestratorType = "Kubernetes"
+	properties.AgentPoolProfiles[0].Count = 4
+	setPropertiesDefaults(&mockCS, false, false)
+	if !properties.AgentPoolProfiles[0].IsVirtualMachineScaleSets() {
+		t.Fatalf("AgentPoolProfile[0].AvailabilityProfile did not have the expected configuration, got %s, expected %s",
+			properties.AgentPoolProfiles[0].AvailabilityProfile, api.VirtualMachineScaleSets)
+	}
+
+	if *properties.AgentPoolProfiles[0].SinglePlacementGroup != api.DefaultSinglePlacementGroup {
+		t.Fatalf("AgentPoolProfile[0].SinglePlacementGroup default did not have the expected configuration, got %t, expected %t",
+			*properties.AgentPoolProfiles[0].SinglePlacementGroup, api.DefaultSinglePlacementGroup)
+	}
+
+	if properties.AgentPoolProfiles[0].HasAvailabilityZones() {
+		if properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku != "Standard" {
+			t.Fatalf("OrchestratorProfile.KubernetesConfig.LoadBalancerSku did not have the expected configuration, got %s, expected %s",
+				properties.OrchestratorProfile.KubernetesConfig.LoadBalancerSku, "Standard")
+		}
+		if properties.OrchestratorProfile.KubernetesConfig.ExcludeMasterFromStandardLB != helpers.PointerToBool(api.DefaultExcludeMasterFromStandardLB) {
+			t.Fatalf("OrchestratorProfile.KubernetesConfig.ExcludeMasterFromStandardLB did not have the expected configuration, got %t, expected %t",
+				*properties.OrchestratorProfile.KubernetesConfig.ExcludeMasterFromStandardLB, api.DefaultExcludeMasterFromStandardLB)
+		}
+	}
+
+	properties.AgentPoolProfiles[0].Count = 110
+	setPropertiesDefaults(&mockCS, false, false)
+	if *properties.AgentPoolProfiles[0].SinglePlacementGroup != false {
+		t.Fatalf("AgentPoolProfile[0].SinglePlacementGroup did not have the expected configuration, got %t, expected %t",
+			*properties.AgentPoolProfiles[0].SinglePlacementGroup, false)
+	}
+
+	if *properties.AgentPoolProfiles[0].SinglePlacementGroup == false && properties.AgentPoolProfiles[0].StorageProfile != api.ManagedDisks {
+		t.Fatalf("AgentPoolProfile[0].StorageProfile did not have the expected configuration, got %s, expected %s",
+			properties.AgentPoolProfiles[0].StorageProfile, api.ManagedDisks)
+	}
+
+}
+
 func getMockAddon(name string) api.KubernetesAddon {
 	return api.KubernetesAddon{
 		Name:    name,

pkg/acsengine/params.go

@@ -173,6 +173,9 @@ func getParameters(cs *api.ContainerService, generatorCode string, acsengineVers
 	for _, agentProfile := range properties.AgentPoolProfiles {
 		addValue(parametersMap, fmt.Sprintf("%sCount", agentProfile.Name), agentProfile.Count)
 		addValue(parametersMap, fmt.Sprintf("%sVMSize", agentProfile.Name), agentProfile.VMSize)
+		if agentProfile.HasAvailabilityZones() {
+			addValue(parametersMap, fmt.Sprintf("%sAvailabilityZones", agentProfile.Name), agentProfile.AvailabilityZones)
+		}
 		if agentProfile.IsCustomVNET() {
 			addValue(parametersMap, fmt.Sprintf("%sVnetSubnetID", agentProfile.Name), agentProfile.VnetSubnetID)
 		} else {

pkg/acsengine/template_generator.go

@@ -679,6 +679,12 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat
 		"IsNSeriesSKU": func(profile *api.AgentPoolProfile) bool {
 			return isNSeriesSKU(profile)
 		},
+		"UseSinglePlacementGroup": func(profile *api.AgentPoolProfile) bool {
+			return *profile.SinglePlacementGroup
+		},
+		"HasAvailabilityZones": func(profile *api.AgentPoolProfile) bool {
+			return profile.HasAvailabilityZones()
+		},
 		"HasLinuxSecrets": func() bool {
 			return cs.Properties.LinuxProfile.HasSecrets()
 		},

pkg/api/const.go

@@ -156,6 +156,9 @@ const (
 	NetworkPluginKubenet = "kubenet"
 	// NetworkPluginAzure is thee string expression for Azure CNI plugin.
 	NetworkPluginAzure = "azure"
+	// DefaultSinglePlacementGroup determines the acs-engine provided default for supporting large VMSS
+	// (true = single placement group 0-100 VMs, false = multiple placement group 0-1000 VMs)
+	DefaultSinglePlacementGroup = true
 )
 
 const (

pkg/api/converterfromapi.go

@@ -996,6 +996,8 @@ func convertAgentPoolProfileToVLabs(api *AgentPoolProfile, p *vlabs.AgentPoolPro
 	p.FQDN = api.FQDN
 	p.CustomNodeLabels = map[string]string{}
 	p.AcceleratedNetworkingEnabled = api.AcceleratedNetworkingEnabled
+	p.AvailabilityZones = api.AvailabilityZones
+	p.SinglePlacementGroup = api.SinglePlacementGroup
 
 	for k, v := range api.CustomNodeLabels {
 		p.CustomNodeLabels[k] = v

pkg/api/convertertoapi.go

@@ -1009,6 +1009,8 @@ func convertVLabsAgentPoolProfile(vlabs *vlabs.AgentPoolProfile, api *AgentPoolP
 	api.IPAddressCount = vlabs.IPAddressCount
 	api.FQDN = vlabs.FQDN
 	api.AcceleratedNetworkingEnabled = vlabs.AcceleratedNetworkingEnabled
+	api.AvailabilityZones = vlabs.AvailabilityZones
+	api.SinglePlacementGroup = vlabs.SinglePlacementGroup
 
 	api.CustomNodeLabels = map[string]string{}
 	for k, v := range vlabs.CustomNodeLabels {

pkg/api/types.go

@@ -464,6 +464,8 @@ type AgentPoolProfile struct {
 	MaxCount                     *int                 `json:"maxCount,omitempty"`
 	MinCount                     *int                 `json:"minCount,omitempty"`
 	EnableAutoScaling            *bool                `json:"enableAutoScaling,omitempty"`
+	AvailabilityZones            []string             `json:"availabilityZones,omitempty"`
+	SinglePlacementGroup         *bool                `json:"singlePlacementGroup,omitempty"`
 }
 
 // AgentPoolProfileRole represents an agent role
@@ -785,6 +787,11 @@ func (a *AgentPoolProfile) HasDisks() bool {
 	return len(a.DiskSizesGB) > 0
 }
 
+// HasAvailabilityZones returns true if the agent pool has availability zones
+func (a *AgentPoolProfile) HasAvailabilityZones() bool {
+	return a.AvailabilityZones != nil && len(a.AvailabilityZones) > 0
+}
+
 // HasSecrets returns true if the customer specified secrets to install
 func (w *WindowsProfile) HasSecrets() bool {
 	return len(w.Secrets) > 0

pkg/api/vlabs/types.go

@@ -436,6 +436,8 @@ type AgentPoolProfile struct {
 	CustomNodeLabels      map[string]string `json:"customNodeLabels,omitempty"`
 	PreProvisionExtension *Extension        `json:"preProvisionExtension"`
 	Extensions            []Extension       `json:"extensions"`
+	SinglePlacementGroup  *bool             `json:"singlePlacementGroup,omitempty"`
+	AvailabilityZones     []string          `json:"availabilityZones,omitempty"`
 }
 
 // AgentPoolProfileRole represents an agent role
@@ -496,6 +498,16 @@ func (p *Properties) HasWindows() bool {
 	return false
 }
 
+// HasAvailabilityZones returns true if the cluster contains pools with zones
+func (p *Properties) HasAvailabilityZones() bool {
+	for _, agentPoolProfile := range p.AgentPoolProfiles {
+		if agentPoolProfile.HasAvailabilityZones() {
+			return true
+		}
+	}
+	return false
+}
+
 // IsCustomVNET returns true if the customer brought their own VNET
 func (m *MasterProfile) IsCustomVNET() bool {
 	return len(m.VnetSubnetID) > 0
@@ -596,6 +608,11 @@ func (a *AgentPoolProfile) SetSubnet(subnet string) {
 	a.subnet = subnet
 }
 
+// HasAvailabilityZones returns true if the agent pool has availability zones
+func (a *AgentPoolProfile) HasAvailabilityZones() bool {
+	return a.AvailabilityZones != nil && len(a.AvailabilityZones) > 0
+}
+
 // HasSearchDomain returns true if the customer specified secrets to install
 func (l *LinuxProfile) HasSearchDomain() bool {
 	if l.CustomSearchDomain != nil {

pkg/api/vlabs/types_test.go

@@ -177,3 +177,17 @@ func TestAgentPoolProfile(t *testing.T) {
 		t.Fatalf("unexpectedly detected AgentPoolProfile.AvailabilitySets != VirtualMachineScaleSets after unmarshal")
 	}
 }
+
+func TestContainerServiceProperties(t *testing.T) {
+	// Agent pool with availability zones
+	ContainerServicePropertiesText := `{"orchestratorProfile": {"orchestratorType": "Kubernetes","orchestratorRelease": "1.11"}, "agentPoolProfiles":[{ "name": "linuxpool1", "osType" : "Linux", "count": 1, "vmSize": "Standard_D2_v2", 
+		"availabilityProfile": "VirtualMachineScaleSets", "AvailabilityZones": ["1","2"]}]}`
+	prop := &Properties{}
+	if e := json.Unmarshal([]byte(ContainerServicePropertiesText), prop); e != nil {
+		t.Fatalf("unexpectedly detected unmarshal failure for ContainerServiceProperties, %+v", e)
+	}
+
+	if !prop.HasAvailabilityZones() {
+		t.Fatalf("unexpectedly detected ContainerServiceProperties HasAvailabilityZones returns false  after unmarshal")
+	}
+}