diff --git a/parts/agentparams.t b/parts/agentparams.t index feb3fab594..99a2db54a0 100644 --- a/parts/agentparams.t +++ b/parts/agentparams.t @@ -50,7 +50,7 @@ }, "type": "string" }, -{{if HaveAvailabilityZones .}} +{{if HasAvailabilityZones .}} "{{.Name}}AvailabilityZones": { "metadata": { "description": "Agent availability zones" diff --git a/parts/k8s/kubernetesagentresourcesvmas.t b/parts/k8s/kubernetesagentresourcesvmas.t index 21a07e05b7..3242f82482 100644 --- a/parts/k8s/kubernetesagentresourcesvmas.t +++ b/parts/k8s/kubernetesagentresourcesvmas.t @@ -81,7 +81,6 @@ "type": "Microsoft.Network/networkInterfaces" }, {{if .IsManagedDisks}} -{{if not (HaveAvailabilityZones .)}} { "location": "[variables('location')]", "name": "[variables('{{.Name}}AvailabilitySet')]", @@ -95,7 +94,6 @@ "type": "Microsoft.Compute/availabilitySets" }, -{{end}} {{else if .IsStorageAccount}} { "apiVersion": "[variables('apiVersionStorage')]", @@ -139,7 +137,6 @@ "type": "Microsoft.Storage/storageAccounts" }, {{end}} -{{if not (HaveAvailabilityZones .)}} { "location": "[variables('location')]", "name": "[variables('{{.Name}}AvailabilitySet')]", @@ -147,7 +144,6 @@ "properties": {}, "type": "Microsoft.Compute/availabilitySets" }, -{{end}} {{end}} { {{if .IsManagedDisks}} @@ -168,9 +164,7 @@ {{end}} {{end}} "[concat('Microsoft.Network/networkInterfaces/', variables('{{.Name}}VMNamePrefix'), 'nic-', copyIndex(variables('{{.Name}}Offset')))]", - {{if not (HaveAvailabilityZones .)}} "[concat('Microsoft.Compute/availabilitySets/', variables('{{.Name}}AvailabilitySet'))]" - {{end}} ], "tags": { @@ -181,9 +175,6 @@ "poolName" : "{{.Name}}" }, "location": "[variables('location')]", - {{if HaveAvailabilityZones .}} - "zones": "[split(string(add(mod(copyIndex(),{{GetAvailabilityZoneLength .}}),{{GetMinAvailabilityZone .}})), ',')]", - {{ end }} "name": "[concat(variables('{{.Name}}VMNamePrefix'), copyIndex(variables('{{.Name}}Offset')))]", {{if UseManagedIdentity}} "identity": { @@ -198,11 +189,9 @@ }, {{end}} "properties": { - {{if not (HaveAvailabilityZones .)}} "availabilitySet": { "id": "[resourceId('Microsoft.Compute/availabilitySets',variables('{{.Name}}AvailabilitySet'))]" }, - {{end}} "hardwareProfile": { "vmSize": "[variables('{{.Name}}VMSize')]" }, diff --git a/parts/k8s/kubernetesagentresourcesvmss.t b/parts/k8s/kubernetesagentresourcesvmss.t index 57121206a7..79f67bab75 100644 --- a/parts/k8s/kubernetesagentresourcesvmss.t +++ b/parts/k8s/kubernetesagentresourcesvmss.t @@ -26,8 +26,8 @@ "poolName" : "{{.Name}}" }, "location": "[variables('location')]", - {{ if HaveAvailabilityZones .}} - "zones": "[variables('{{.Name}}AvailabilityZones')]", + {{ if HasAvailabilityZones .}} + "zones": "[parameters('{{.Name}}AvailabilityZones')]", {{ end }} "name": "[variables('{{.Name}}VMNamePrefix')]", {{if UseManagedIdentity}} @@ -41,7 +41,9 @@ "name": "[variables('{{.Name}}VMSize')]" }, "properties": { - {{if not (UseSinglePlacementGroup .)}} + {{if UseSinglePlacementGroup .}} + "singlePlacementGroup": true, + {{else}} "singlePlacementGroup": false, {{end}} "overprovision": false, diff --git a/parts/k8s/kubernetesagentvars.t b/parts/k8s/kubernetesagentvars.t index c165a06255..f59aad16fa 100644 --- a/parts/k8s/kubernetesagentvars.t +++ b/parts/k8s/kubernetesagentvars.t @@ -20,9 +20,6 @@ "{{.Name}}ScaleSetEvictionPolicy": "[parameters('{{.Name}}ScaleSetEvictionPolicy')]", {{end}} {{end}} -{{ if HaveAvailabilityZones .}} - "{{.Name}}AvailabilityZones": "[parameters('{{.Name}}AvailabilityZones')]", -{{end}} {{end}} "{{.Name}}VMSize": "[parameters('{{.Name}}VMSize')]", {{if .IsCustomVNET}} diff --git a/parts/k8s/kubernetescustomscript.sh b/parts/k8s/kubernetescustomscript.sh index e257d36c41..b58d5b5dc4 100644 --- a/parts/k8s/kubernetescustomscript.sh +++ b/parts/k8s/kubernetescustomscript.sh @@ -108,6 +108,7 @@ if [[ ! -z "${MASTER_NODE}" ]]; then writeKubeConfig ensureEtcd ensureK8sControlPlane + ensureKubelet fi if [[ $OS == $UBUNTU_OS_NAME ]]; then diff --git a/parts/k8s/kubernetesmasterresources.t b/parts/k8s/kubernetesmasterresources.t index b325bc0861..1ba232b555 100644 --- a/parts/k8s/kubernetesmasterresources.t +++ b/parts/k8s/kubernetesmasterresources.t @@ -1,4 +1,3 @@ -{{if not HaveMasterAvailabilityZones}} {{if .MasterProfile.IsManagedDisks}} { "apiVersion": "[variables('apiVersionStorageManagedDisks')]", @@ -12,9 +11,7 @@ }, "type": "Microsoft.Compute/availabilitySets" }, -{{end}} {{else if .MasterProfile.IsStorageAccount}} -{{if not HaveMasterAvailabilityZones}} { "apiVersion": "[variables('apiVersionDefault')]", "location": "[variables('location')]", @@ -22,7 +19,6 @@ "properties": {}, "type": "Microsoft.Compute/availabilitySets" }, -{{end}} { "apiVersion": "[variables('apiVersionStorage')]", {{if not IsPrivateCluster}} @@ -145,36 +141,24 @@ {{end}} {{if not IsPrivateCluster}} { - "apiVersion": "[variables('apiVersionNetwork')]", + "apiVersion": "[variables('apiVersionDefault')]", "location": "[variables('location')]", "name": "[variables('masterPublicIPAddressName')]", - {{if HaveMasterAvailabilityZones}} - "sku": { - "name": "Standard" - }, - {{end}} "properties": { "dnsSettings": { "domainNameLabel": "[variables('masterFqdnPrefix')]" - } - {{if HaveMasterAvailabilityZones}} - ,"publicIPAllocationMethod": "Static" - {{end}} + }, + "publicIPAllocationMethod": "Dynamic" }, "type": "Microsoft.Network/publicIPAddresses" }, { - "apiVersion": "[variables('apiVersionNetwork')]", + "apiVersion": "[variables('apiVersionDefault')]", "dependsOn": [ "[concat('Microsoft.Network/publicIPAddresses/', variables('masterPublicIPAddressName'))]" ], "location": "[variables('location')]", "name": "[variables('masterLbName')]", - {{if HaveMasterAvailabilityZones}} - "sku": { - "name": "Standard" - }, - {{end}} "properties": { "backendAddressPools": [ { @@ -526,13 +510,11 @@ }, { "type": "Microsoft.Network/publicIpAddresses", - {{if HaveMasterAvailabilityZones}} "sku": { - "name": "Standard" + "name": "Basic" }, - {{end}} "name": "[variables('jumpboxPublicIpAddressName')]", - "apiVersion": "[variables('apiVersionPublicIP')]", + "apiVersion": "[variables('apiVersionDefault')]", "location": "[variables('location')]", "properties": { "dnsSettings": { @@ -578,7 +560,7 @@ {{end}} {{if gt .MasterProfile.Count 1}} { - "apiVersion": "[variables('apiVersionNetwork')]", + "apiVersion": "[variables('apiVersionDefault')]", "dependsOn": [ {{if .MasterProfile.IsCustomVNET}} "[variables('nsgID')]" @@ -588,11 +570,6 @@ ], "location": "[variables('location')]", "name": "[variables('masterInternalLbName')]", - {{if HaveMasterAvailabilityZones}} - "sku": { - "name": "Standard" - }, - {{end}} "properties": { "backendAddressPools": [ { @@ -663,7 +640,7 @@ "apiVersion": "[variables('apiVersionKeyVault')]", "location": "[variables('location')]", {{ if UseManagedIdentity}} - "dependsOn": + "dependsOn": [ {{$max := .MasterProfile.Count}} {{$c := subtract $max 1}} @@ -696,7 +673,7 @@ } ], {{else}} - "accessPolicies": + "accessPolicies": [ {{$max := .MasterProfile.Count}} {{$c := subtract $max 1}} @@ -754,9 +731,7 @@ }, "dependsOn": [ "[concat('Microsoft.Network/networkInterfaces/', variables('masterVMNamePrefix'), 'nic-', copyIndex(variables('masterOffset')))]" -{{if not HaveMasterAvailabilityZones}} ,"[concat('Microsoft.Compute/availabilitySets/',variables('masterAvailabilitySet'))]" -{{end}} {{if .MasterProfile.IsStorageAccount}} ,"[variables('masterStorageAccountName')]" {{end}} @@ -770,9 +745,6 @@ "poolName" : "master" }, "location": "[variables('location')]", - {{ if HaveMasterAvailabilityZones}} - "zones": "[split(string(add(mod(copyIndex(),{{GetMasterAvailabilityZoneLength}}),{{GetMasterMinAvailabilityZone}})), ',')]", - {{ end }} "name": "[concat(variables('masterVMNamePrefix'), copyIndex(variables('masterOffset')))]", {{if UseManagedIdentity}} "identity": { @@ -787,11 +759,9 @@ }, {{end}} "properties": { - {{if not HaveMasterAvailabilityZones}} "availabilitySet": { "id": "[resourceId('Microsoft.Compute/availabilitySets',variables('masterAvailabilitySet'))]" }, - {{end}} "hardwareProfile": { "vmSize": "[parameters('masterVMSize')]" }, @@ -961,4 +931,4 @@ } } {{end}} - {{WriteLinkedTemplatesForExtensions}} + {{WriteLinkedTemplatesForExtensions}} \ No newline at end of file diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t index 2886111f28..88a9d0ddda 100644 --- a/parts/k8s/kubernetesmastervars.t +++ b/parts/k8s/kubernetesmastervars.t @@ -83,7 +83,6 @@ {{end}} "apiVersionDefault": "2016-03-30", "apiVersionLinkDefault": "2015-01-01", - "apiVersionNetwork": "2018-05-01", "locations": [ "[resourceGroup().location]", "[parameters('location')]" diff --git a/parts/k8s/kuberneteswinagentresourcesvmas.t b/parts/k8s/kuberneteswinagentresourcesvmas.t index 192cdb3186..3d04aff616 100644 --- a/parts/k8s/kuberneteswinagentresourcesvmas.t +++ b/parts/k8s/kuberneteswinagentresourcesvmas.t @@ -29,9 +29,6 @@ {{end}} ], "location": "[variables('location')]", - {{ if HaveAvailabilityZones .}} - "zones": "[variables('{{.Name}}AvailabilityZones')]", - {{ end }} "name": "[concat(variables('{{.Name}}VMNamePrefix'), 'nic-', copyIndex(variables('{{.Name}}Offset')))]", "properties": { {{if .IsCustomVNET}} diff --git a/parts/k8s/kuberneteswinagentresourcesvmss.t b/parts/k8s/kuberneteswinagentresourcesvmss.t index 998ef4533c..c64877ed0d 100644 --- a/parts/k8s/kuberneteswinagentresourcesvmss.t +++ b/parts/k8s/kuberneteswinagentresourcesvmss.t @@ -26,8 +26,8 @@ "poolName" : "{{.Name}}" }, "location": "[variables('location')]", - {{ if HaveAvailabilityZones .}} - "zones": "[variables('{{.Name}}AvailabilityZones')]", + {{ if HasAvailabilityZones .}} + "zones": "[parameters('{{.Name}}AvailabilityZones')]", {{ end }} "name": "[variables('{{.Name}}VMNamePrefix')]", {{if UseManagedIdentity}} @@ -41,6 +41,11 @@ "name": "[variables('{{.Name}}VMSize')]" }, "properties": { + {{if UseSinglePlacementGroup .}} + "singlePlacementGroup": true, + {{else}} + "singlePlacementGroup": false, + {{end}} "overprovision": false, "upgradePolicy": { "mode": "Manual" diff --git a/parts/masterparams.t b/parts/masterparams.t index 1d6295ea42..53791eacb8 100644 --- a/parts/masterparams.t +++ b/parts/masterparams.t @@ -72,14 +72,6 @@ }, "type": "string" }, -{{end}} -{{if HaveMasterAvailabilityZones}} - "masterAvailabilityZones": { - "metadata": { - "description": "Master availability zones" - }, - "type": "array" - }, {{end}} "sshRSAPublicKey": { "metadata": { diff --git a/pkg/acsengine/defaults.go b/pkg/acsengine/defaults.go index 55b0eec5a7..34b4c24d34 100644 --- a/pkg/acsengine/defaults.go +++ b/pkg/acsengine/defaults.go @@ -232,6 +232,7 @@ func setPropertiesDefaults(cs *api.ContainerService, isUpgrade, isScale bool) (b setStorageDefaults(properties) setExtensionDefaults(properties) + setVMSSDefaults(properties) certsGenerated, e := setDefaultCerts(properties) if e != nil { @@ -546,6 +547,33 @@ func setMasterNetworkDefaults(a *api.Properties, isUpgrade bool) { } } +// setVMSSDefaults +// singlePlacementGroup = false, the scale set can be composed of multiple placement groups and has a range of 0-1,000 VMs +// singlePlacementGroup = true,, default value, a scale set is composed of a single placement group, and has a range of 0-100 VMs +// Large scale sets require Azure Managed Disks. +// https://docs.microsoft.com/en-us/azure/virtual-machine-scale-sets/virtual-machine-scale-sets-placement-groups +// For availability zones, only standard load balancer is supported. +func setVMSSDefaults(a *api.Properties) { + for _, profile := range a.AgentPoolProfiles { + if profile.AvailabilityProfile == api.VirtualMachineScaleSets { + if profile.Count > 100 { + profile.SinglePlacementGroup = helpers.PointerToBool(false) + } + if profile.SinglePlacementGroup == helpers.PointerToBool(false) { + profile.StorageProfile = api.ManagedDisks + } + if len(profile.AvailabilityZones) > 0 { + a.OrchestratorProfile.KubernetesConfig.LoadBalancerSku = "Standard" + a.OrchestratorProfile.KubernetesConfig.ExcludeMasterFromStandardLB = helpers.PointerToBool(api.DefaultExcludeMasterFromStandardLB) + } + if profile.SinglePlacementGroup == nil { + profile.SinglePlacementGroup = helpers.PointerToBool(api.DefaultSinglePlacementGroup) + } + } + + } +} + // SetAgentNetworkDefaults for agents func setAgentNetworkDefaults(a *api.Properties, isUpgrade, isScale bool) { // configure the subnets if not in custom VNET diff --git a/pkg/acsengine/template_generator.go b/pkg/acsengine/template_generator.go index 9913a78d1b..342c944dcc 100644 --- a/pkg/acsengine/template_generator.go +++ b/pkg/acsengine/template_generator.go @@ -683,9 +683,9 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat return getGPUDriversInstallScript(profile) }, "UseSinglePlacementGroup": func(profile *api.AgentPoolProfile) bool { - return profile.SinglePlacementGroup + return *profile.SinglePlacementGroup }, - "HaveAvailabilityZones": func(profile *api.AgentPoolProfile) bool { + "HasAvailabilityZones": func(profile *api.AgentPoolProfile) bool { return len(profile.AvailabilityZones) > 0 }, "GetMinAvailabilityZone": func(profile *api.AgentPoolProfile) int { @@ -701,22 +701,6 @@ func (t *TemplateGenerator) getTemplateFuncMap(cs *api.ContainerService) templat "GetAvailabilityZoneLength": func(profile *api.AgentPoolProfile) int { return len(profile.AvailabilityZones) }, - "HaveMasterAvailabilityZones": func() bool { - return len(cs.Properties.MasterProfile.AvailabilityZones) > 0 - }, - "GetMasterMinAvailabilityZone": func() int { - min, _ := strconv.Atoi(cs.Properties.MasterProfile.AvailabilityZones[0]) - for _, z := range cs.Properties.MasterProfile.AvailabilityZones { - i, _ := strconv.Atoi(z) - if min > i { - min = i - } - } - return min - }, - "GetMasterAvailabilityZoneLength": func() int { - return len(cs.Properties.MasterProfile.AvailabilityZones) - }, "HasLinuxSecrets": func() bool { return cs.Properties.LinuxProfile.HasSecrets() }, diff --git a/pkg/api/common/versions.go b/pkg/api/common/versions.go index 8196181c71..1a4aec3304 100644 --- a/pkg/api/common/versions.go +++ b/pkg/api/common/versions.go @@ -74,6 +74,7 @@ var AllKubernetesSupportedVersions = map[string]bool{ "1.11.1": true, "1.11.2": true, "1.12.0-alpha.1": true, + "1.12.0-beta.0": true, } // GetDefaultKubernetesVersion returns the default Kubernetes version, that is the latest patch of the default release diff --git a/pkg/api/const.go b/pkg/api/const.go index 7e3c027627..27a8c5a659 100644 --- a/pkg/api/const.go +++ b/pkg/api/const.go @@ -155,6 +155,9 @@ const ( NetworkPluginKubenet = "kubenet" // NetworkPluginAzure is thee string expression for Azure CNI plugin. NetworkPluginAzure = "azure" + // DefaultSinglePlacementGroup determines the acs-engine provided default for supporting large VMSS + // (true = single placement group 0-100 VMs, false = multiple placement group 0-1000 VMs) + DefaultSinglePlacementGroup = true ) const ( diff --git a/pkg/api/converterfromapi.go b/pkg/api/converterfromapi.go index 330dd55a1f..c326002042 100644 --- a/pkg/api/converterfromapi.go +++ b/pkg/api/converterfromapi.go @@ -895,7 +895,6 @@ func convertMasterProfileToVLabs(api *MasterProfile, vlabsProfile *vlabs.MasterP vlabsProfile.SetSubnet(api.Subnet) vlabsProfile.FQDN = api.FQDN vlabsProfile.StorageProfile = api.StorageProfile - vlabsProfile.AvailabilityZones = api.AvailabilityZones if api.PreprovisionExtension != nil { vlabsExtension := &vlabs.Extension{} convertExtensionToVLabs(api.PreprovisionExtension, vlabsExtension) diff --git a/pkg/api/convertertoapi.go b/pkg/api/convertertoapi.go index aeab24da7d..1267f7cdcd 100644 --- a/pkg/api/convertertoapi.go +++ b/pkg/api/convertertoapi.go @@ -894,7 +894,6 @@ func convertVLabsMasterProfile(vlabs *vlabs.MasterProfile, api *MasterProfile) { api.StorageProfile = vlabs.StorageProfile api.HTTPSourceAddressPrefix = vlabs.HTTPSourceAddressPrefix api.OAuthEnabled = vlabs.OAuthEnabled - api.AvailabilityZones = vlabs.AvailabilityZones // by default vlabs will use managed disks as it has encryption at rest if len(api.StorageProfile) == 0 { api.StorageProfile = ManagedDisks diff --git a/pkg/api/types.go b/pkg/api/types.go index cd059493b6..e9895acf90 100644 --- a/pkg/api/types.go +++ b/pkg/api/types.go @@ -403,7 +403,6 @@ type MasterProfile struct { KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"` ImageRef *ImageReference `json:"imageReference,omitempty"` CustomFiles *[]CustomFile `json:"customFiles,omitempty"` - AvailabilityZones []string `json:"availabilityZones,omitempty"` // Master LB public endpoint/FQDN with port // The format will be FQDN:2376 @@ -466,7 +465,7 @@ type AgentPoolProfile struct { MinCount *int `json:"minCount,omitempty"` EnableAutoScaling *bool `json:"enableAutoScaling,omitempty"` AvailabilityZones []string `json:"availabilityZones,omitempty"` - SinglePlacementGroup bool `json:"singlePlacementGroup,omitempty"` + SinglePlacementGroup *bool `json:"singlePlacementGroup,omitempty"` } // AgentPoolProfileRole represents an agent role diff --git a/pkg/api/vlabs/types.go b/pkg/api/vlabs/types.go index 7143e77d4a..fdebc3261a 100644 --- a/pkg/api/vlabs/types.go +++ b/pkg/api/vlabs/types.go @@ -372,7 +372,6 @@ type MasterProfile struct { KubernetesConfig *KubernetesConfig `json:"kubernetesConfig,omitempty"` ImageRef *ImageReference `json:"imageReference,omitempty"` CustomFiles *[]CustomFile `json:"customFiles,omitempty"` - AvailabilityZones []string `json:"availabilityZones,omitempty"` // subnet is internal subnet string @@ -437,7 +436,7 @@ type AgentPoolProfile struct { CustomNodeLabels map[string]string `json:"customNodeLabels,omitempty"` PreProvisionExtension *Extension `json:"preProvisionExtension"` Extensions []Extension `json:"extensions"` - SinglePlacementGroup bool `json:"singlePlacementGroup,omitempty"` + SinglePlacementGroup *bool `json:"singlePlacementGroup,omitempty"` AvailabilityZones []string `json:"availabilityZones,omitempty"` }