Squashed commit of the following:

commit d0ebb9e Author: chshou <[email protected]> Date: Mon Mar 26 18:06:57 2018 -0700 Fix merge errors (#22) * move sed to kubelet.sh, remove unnecessary exit 0 (Azure#2520) * move sed to kubelet.sh, remove unnecessary exit 0 * circleci bump * circleci bump * fix merge errors and deployment succeeded * added example * sanitize example commit 5639dce Merge: 1294a58 89f4b2e Author: Wenjun Wu <[email protected]> Date: Mon Mar 26 17:51:08 2018 -0700 Merge pull request #21 from yolo3301/mig-merge Merge from upstream commit 89f4b2e Merge: 375d0c0 1294a58 Author: chshou <[email protected]> Date: Mon Mar 26 16:03:30 2018 -0700 fix merge error commit 1294a58 Author: Wenjun Wu <[email protected]> Date: Fri Mar 16 18:13:44 2018 -0700 skip create initial role binding due to bulit-in RBAC support. (#19) commit 375d0c0 Merge: 295461d 7d91a71 Author: chshou <[email protected]> Date: Fri Mar 9 12:09:00 2018 -0800 resolve conflicts commit 7d91a71 Author: Wenjun Wu <[email protected]> Date: Fri Mar 9 11:29:11 2018 -0800 disable heapster config (#18) commit 295461d Author: chshou <[email protected]> Date: Thu Mar 8 18:23:57 2018 -0800 remove more unnecessary commit 87f7746 Author: chshou <[email protected]> Date: Thu Mar 8 17:55:42 2018 -0800 remove unnecessary commit ae0caf8 Author: chshou <[email protected]> Date: Wed Mar 7 17:53:45 2018 -0800 fix 2 more missed error commit efa144e Author: chshou <[email protected]> Date: Wed Mar 7 17:32:09 2018 -0800 a miss commit 8d96a93 Merge: fd2a409 e3587cb Author: chshou <[email protected]> Date: Wed Mar 7 16:54:22 2018 -0800 merged from upstream master commit fd2a409 Author: Jess Frazelle <[email protected]> Date: Tue Jan 16 23:49:48 2018 -0500 k8s/script: allow parallelizing custom script without clear-containers (Azure#2067) Signed-off-by: Jess Frazelle <[email protected]> (cherry picked from commit cdd2832) Signed-off-by: Jess Frazelle <[email protected]> commit 1e10c0d Author: Jess Frazelle <[email protected]> Date: Tue Jan 16 19:09:32 2018 -0500 clear containers (Azure#1945) * clear-containers: add runtime to api and pass through parameters Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: add scripts Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: add example Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: fix variables Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: add docs Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: update install script Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: fix script Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: update example Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: update features docs Signed-off-by: Jess Frazelle <[email protected]> * clear-containers: make test linters happy Signed-off-by: Jess Frazelle <[email protected]> * setKubeletOpts to work better with kubeconfig Signed-off-by: Jess Frazelle <[email protected]> * whitespace cruft * more whitespace fun (cherry picked from commit 8bd7c2c) commit fa3d6ff Author: Wenjun Wu <[email protected]> Date: Mon Feb 12 19:24:35 2018 -0800 Squashed commit of the following: commit 203efbf Author: Jiangtian Li <[email protected]> Date: Fri Jan 19 09:07:14 2018 -0800 Extend windows os drive size when customized OSDiskSizeGB is used (Azure#2097) commit 88ec2fb Author: Robbie Zhang <[email protected]> Date: Thu Jan 11 13:49:44 2018 -0800 Update the kube-dns addon commit 217ad8d Merge: 530bedb d8856c8 Author: Wenjun Wu <[email protected]> Date: Mon Jan 8 16:22:56 2018 -0800 Merge remote-tracking branch 'origin/migration' into migration commit d8856c8 Author: Robbie Zhang <[email protected]> Date: Fri Jan 5 15:39:28 2018 -0800 Remove the Allow SSH and RDP Rules from NSG commit 530bedb Merge: f3389a6 5070934 Author: Wenjun Wu <[email protected]> Date: Fri Jan 5 15:38:54 2018 -0800 Merge tag 'v0.9.4' into migration commit f3389a6 Author: Wenjun Wu <[email protected]> Date: Fri Dec 15 11:11:13 2017 -0800 remove agent customscript and service file (#13) * remove agent specific custom script and service file. * remove cloud provider from windows start ps1 commit c2eda57 Merge: 8ef4f2b 004145c Author: Wenjun Wu <[email protected]> Date: Tue Dec 12 18:05:13 2017 -0800 Merge commit '004145cba163' into migration commit 004145c Author: Wenjun Wu <[email protected]> Date: Tue Dec 12 18:03:36 2017 -0800 fix merge error: azure storage classes yaml commit 8ef4f2b Merge: adbc1cf bd006fc Author: Wenjun Wu <[email protected]> Date: Mon Nov 27 18:24:06 2017 -0800 Merge tag 'v0.9.3' into migration commit adbc1cf Merge: f8da501 7957245 Author: Wenjun Wu <[email protected]> Date: Wed Oct 25 14:36:24 2017 -0700 Merge tag 'v0.8.0' into migration commit f8da501 Author: Robbie Zhang <[email protected]> Date: Fri Sep 1 16:38:00 2017 -0700 Disable Windows Update commit ac83868 Author: Robbie Zhang <[email protected]> Date: Fri Sep 1 16:37:36 2017 -0700 Use kubelet v1.6.6.1 for Windows agent commit 5424f14 Author: Robbie Zhang <[email protected]> Date: Fri Sep 1 16:36:47 2017 -0700 Set master AvailabilitySet FaultDomainCount and UpdateDomainCount to 1 commit 5b1fbb0 Author: Robbie Zhang <[email protected]> Date: Tue Aug 15 12:23:41 2017 -0700 Enable StorageAccount Encryption and Enforce HTTPS commit 12fd01d Author: Harry He <[email protected]> Date: Fri Jul 7 10:16:03 2017 -0700 Remove Resource Requests from kube-proxy (#5) Previously kube-proxy requested 100m CPU. It prevented containers requesting 1 CPU from being deployed onto nodes with 1 CPU, because there is only 900m CPU left. This change remove resource requests from kube-proxy. commit 5241639 Author: Robbie Zhang <[email protected]> Date: Fri Jul 7 14:23:32 2017 -0700 Set the default CloudProvider backoff values commit 549a4c2 Merge: 0506730 8a47cbd Author: Robbie Zhang <[email protected]> Date: Fri Jul 7 16:14:12 2017 -0700 Merge with v0.3.0 commit 0506730 Author: Robbie Zhang <[email protected]> Date: Fri Jul 7 13:01:18 2017 -0700 Disable Automatic Windows Update commit 8eb8afe Merge: 639e36a fb09cdf Author: Robbie Zhang <[email protected]> Date: Fri Jul 7 12:07:03 2017 -0700 Merge from upstream release v0.2.0 commit 639e36a Author: Robbie Zhang <[email protected]> Date: Mon Jul 3 11:05:10 2017 -0700 Remove azure.json from Windows Agent commit c9d0704 Merge: bae0a8b 579e8b8 Author: Robbie Zhang <[email protected]> Date: Mon Jun 19 10:13:37 2017 -0700 Merge tag 'v0.1.2' into migration commit bae0a8b Author: Raghu Shantha [MSFT] <[email protected]> Date: Thu Jun 15 11:36:03 2017 -0700 Enable Firewall on Node, Add Windows Firewall rules for required ports (#2) * Enable Firewall on Node, Add Windows Firewall rules for required ports * Added comments for firewall rules * Allow all traffic; lockdown kubectl Node ports to Master only * Remove & and single quote in comment section resource group deployment parser does not like these chars in the comment section commit af24ad6 Author: Robbie Zhang <[email protected]> Date: Tue Jun 6 18:20:40 2017 -0700 Enable RBAC on APIServer commit e648d3d Merge: 380bc58 cc95f47 Author: Robbie Zhang <[email protected]> Date: Wed May 24 11:01:11 2017 -0700 Merge branch 'master' into migration commit 380bc58 Author: Robbie Zhang <[email protected]> Date: Mon May 15 11:39:43 2017 -0700 Fix: add the size map for F1 commit e64b446 Merge: 87c56c3 253dd41 Author: Wenjun Wu <[email protected]> Date: Sun May 14 15:47:20 2017 -0700 Merge branch 'master' into migration commit 87c56c3 Author: Robbie Zhang <[email protected]> Date: Fri Apr 14 12:55:21 2017 -0700 Private Commit for Azure Console Shell Remove SPN secrets from agent node Remove the Kube Dashboard and Heapster Addons Add agentpool label on the agent nodes Use static IP address for system and agentpool1 commit 9fa6a69 Author: Jack Francis <[email protected]> Date: Wed Jan 31 17:07:00 2018 -0800 for loop and --retry-connrefused not avail commit 0dda4bb Author: Jack Francis <[email protected]> Date: Wed Jan 31 16:44:47 2018 -0800 retry etcd download commit bb4b9bc Author: Jack Francis <[email protected]> Date: Wed Jan 31 15:37:34 2018 -0800 addresses etcd startup race condition commit 2f1bfe6 Author: Jack Francis <[email protected]> Date: Wed Jan 31 15:25:33 2018 -0800 cloud-init does not respect {1..5} expression commit 069d9e4 Author: CecileRobertMichon <[email protected]> Date: Mon Jan 22 11:41:38 2018 -0800 Add fix to upgrade backwards compatibility commit 030e5dc Author: Jack Francis <[email protected]> Date: Mon Jan 22 10:42:35 2018 -0800 add support for Kubernetes v1.8.7 commit 7d19218 Author: Jack Francis <[email protected]> Date: Fri Jan 19 12:31:12 2018 -0800 lint commit 9ed1610 Author: Jack Francis <[email protected]> Date: Fri Jan 19 12:26:58 2018 -0800 restore properties to KubernetesConfig commit 93589b4 Author: Jack Francis <[email protected]> Date: Thu Jan 18 11:48:11 2018 -0800 re-enable read-only port on kubelet fixes heapster connection issues
yangl900 · Mar 27, 2018 · 4e1a300 · wenwu449 · Mar 27, 2018 · 4e1a300
1 parent e3587cb
commit 4e1a300
Show file tree

Hide file tree

Showing 14 changed files with 245 additions and 120 deletions.
diff --git a/examples/k8s-upgrade/v1.7.9-hybrid.json.env b/examples/k8s-upgrade/v1.7.9-hybrid.json.env
@@ -1,2 +1,2 @@
 ACSE_POSTDEPLOY=examples/k8s-upgrade/k8s-upgrade.sh
-EXPECTED_ORCHESTRATOR_VERSION=1.8.8
+EXPECTED_ORCHESTRATOR_VERSION=1.8.8
diff --git a/examples/kubernetes-aci.json b/examples/kubernetes-aci.json
@@ -0,0 +1,90 @@
+{
+  "apiVersion": "vlabs",
+  "plan": {},
+  "properties": {
+    "provisioningState": "",
+    "orchestratorProfile": {
+      "orchestratorType": "Kubernetes",
+      "orchestratorVersion": "1.8.2",
+      "kubernetesConfig": {
+        "networkPolicy": "none",
+        "kubeletConfig": {
+          "--cloud-provider": "",
+          "--cloud-config": "",
+          "--azure-container-registry-config": ""
+        },
+        "addons": [
+          {
+            "name": "tiller",
+            "enabled" : false
+          },
+          {
+            "name": "kubernetes-dashboard",
+            "enabled" : false
+          }
+        ]
+      }
+    },
+    "masterProfile": {
+      "count": 1,
+      "dnsPrefix": "caas-test-eastus-linux-03",
+      "vmSize": "Standard_D2_v2",
+      "firstConsecutiveStaticIP": "10.240.255.5"
+    },
+    "agentPoolProfiles": [
+      {
+        "name": "system",
+        "count": 2,
+        "vmSize": "Standard_F1",
+        "availabilityProfile": "AvailabilitySet",
+        "storageProfile": "StorageAccount",
+        "osType": "Linux"
+      },
+      {
+        "name": "agentpool1",
+        "count": 2,
+        "vmSize": "Standard_F2",
+        "availabilityProfile": "AvailabilitySet",
+        "storageProfile": "StorageAccount",
+        "osType": "Linux"
+      },
+      {
+        "name": "agentpool2",
+        "count": 3,
+        "vmSize": "Standard_F1",
+        "availabilityProfile": "AvailabilitySet",
+        "storageProfile": "StorageAccount",
+        "osType": "Linux",
+        "osDiskSizeGB": 50
+      },
+      {
+        "name": "agentpool3",
+        "count": 3,
+        "vmSize": "Standard_F1",
+        "availabilityProfile": "AvailabilitySet",
+        "storageProfile": "StorageAccount",
+        "osType": "Linux",
+        "osDiskSizeGB": 50
+      }
+    ],
+    "linuxProfile": {
+      "adminUsername": "azureuser",
+      "ssh": {
+        "publicKeys": [
+          {
+            "keyData": ""
+          }
+        ]
+      }
+    },
+    "windowsProfile": {
+      "adminUsername": "",
+      "adminPassword": ""
+    },
+    "servicePrincipalProfile": {
+      "clientId": "",
+      "secret": ""
+    },
+    "certificateProfile": {}
+  }
+}
diff --git a/parts/k8s/addons/kubernetesmasteraddons-kube-dns-deployment.yaml b/parts/k8s/addons/kubernetesmasteraddons-kube-dns-deployment.yaml
@@ -43,6 +43,11 @@ spec:
     matchLabels:
       k8s-app: kube-dns
       version: v20
+  strategy:
+    rollingUpdate:
+      maxSurge: 1
+      maxUnavailable: 1
+    type: RollingUpdate
   template:
     metadata:
       annotations:
@@ -168,3 +173,4 @@ spec:
       serviceAccountName: kube-dns
       nodeSelector:
         beta.kubernetes.io/os: linux
+        agentpool: system
diff --git a/parts/k8s/addons/kubernetesmasteraddons-kube-proxy-daemonset.yaml b/parts/k8s/addons/kubernetesmasteraddons-kube-proxy-daemonset.yaml
@@ -28,9 +28,6 @@ spec:
         - "--feature-gates=ExperimentalCriticalPodAnnotation=true"
         image: "<kubernetesHyperkubeSpec>"
         name: kube-proxy
-        resources:
-          requests:
-            cpu: 100m
         securityContext:
           privileged: true
         volumeMounts:

diff --git a/parts/k8s/kubernetesagentcustomdata.yml b/parts/k8s/kubernetesagentcustomdata.yml
@@ -129,8 +129,9 @@ AGENT_ARTIFACTS_CONFIG_PLACEHOLDER
     # SNAT outbound traffic from pods to destinations outside of VNET.
     iptables -t nat -A POSTROUTING -m iprange ! --dst-range 168.63.129.16 -m addrtype ! --dst-type local ! -d {{WrapAsVariable "vnetCidr"}} -j MASQUERADE
 {{end}}
-
-    exit 0
+{{if not EnablePodSecurityPolicy}}
+    sed -i "s|apparmor_parser|d|g" "/etc/systemd/system/kubelet.service"
+{{end}}
 
 - path: "/opt/azure/containers/provision.sh"
   permissions: "0744"
@@ -168,9 +169,6 @@ coreos:
         [Service]
         ExecStart=/opt/azure/containers/provision-setup.sh
 {{else}}
-{{if not EnablePodSecurityPolicy}}
-    sed -i "s|apparmor_parser|d|g" "/etc/systemd/system/kubelet.service"
-{{end}}
 runcmd:
 - echo `date`,`hostname`, startruncmd>>/opt/m
 # the first arg is the number of retries, the second arg is the wait duration between two retries and the rest of the args are the cmd to run

diff --git a/parts/k8s/kubernetesagentresourcesvmas.t b/parts/k8s/kubernetesagentresourcesvmas.t
@@ -27,7 +27,15 @@
               {{if eq $seq 1}}
               "primary": true,
               {{end}}
+              {{if eq $.Name "system"}}
+              "privateIPAddress": "[concat(variables('masterFirstAddrPrefix'), copyIndex(add(50, int(variables('masterFirstAddrOctet4')))))]",
+              "privateIPAllocationMethod": "Static",
+              {{else if eq $.Name "agentpool1"}}
+              "privateIPAddress": "[concat(variables('masterFirstAddrPrefix'), copyIndex(add(100, int(variables('masterFirstAddrOctet4')))))]",
+              "privateIPAllocationMethod": "Static",
+              {{else}}
               "privateIPAllocationMethod": "Dynamic",
+              {{end}}
               "subnet": {
                 "id": "[variables('{{$.Name}}VnetSubnetID')]"
               }
@@ -71,10 +79,25 @@
           ],
         {{end}}
       {{end}}
+      "kind": "Storage",
       "location": "[variables('location')]",
       "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}AccountName'))]",
       "properties": {
-        "accountType": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]"
+        "encryption": {
+          "keySource": "Microsoft.Storage",
+          "services": {
+            "blob": {
+              "enabled": true
+            },
+            "file": {
+              "enabled": true
+            }
+          }
+        },
+        "supportsHttpsTrafficOnly": true
+      },
+      "sku": {
+        "name": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]"
       },
       "type": "Microsoft.Storage/storageAccounts"
     },
@@ -92,10 +115,25 @@
           ],
         {{end}}
       {{end}}
+      "kind": "Storage",
       "location": "[variables('location')]",
       "name": "[concat(variables('storageAccountPrefixes')[mod(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('storageAccountPrefixes')[div(add(copyIndex(variables('dataStorageAccountPrefixSeed')),variables('{{.Name}}StorageAccountOffset')),variables('storageAccountPrefixesCount'))],variables('{{.Name}}DataAccountName'))]",
       "properties": {
-        "accountType": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]"
+        "encryption": {
+          "keySource": "Microsoft.Storage",
+          "services": {
+            "blob": {
+              "enabled": true
+            },
+            "file": {
+              "enabled": true
+            }
+          }
+        },
+        "supportsHttpsTrafficOnly": true
+      },
+      "sku": {
+        "name": "[variables('vmSizesMap')[variables('{{.Name}}VMSize')].storageAccountType]"
       },
       "type": "Microsoft.Storage/storageAccounts"
     },

diff --git a/parts/k8s/kubernetesmastercustomdata.yml b/parts/k8s/kubernetesmastercustomdata.yml
@@ -188,7 +188,7 @@ MASTER_ARTIFACTS_CONFIG_PLACEHOLDER
     sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g" "/etc/kubernetes/manifests/kube-scheduler.yaml"
     sed -i "s|<kubernetesHyperkubeSpec>|{{WrapAsVariable "kubernetesHyperkubeSpec"}}|g; s|<kubeClusterCidr>|{{WrapAsVariable "kubeClusterCidr"}}|g" "/etc/kubernetes/addons/kube-proxy-daemonset.yaml"
     sed -i "s|<kubernetesKubeDNSSpec>|{{WrapAsVariable "kubernetesKubeDNSSpec"}}|g; s|<kubernetesDNSMasqSpec>|{{WrapAsVariable "kubernetesDNSMasqSpec"}}|g; s|<kubernetesExecHealthzSpec>|{{WrapAsVariable "kubernetesExecHealthzSpec"}}|g; s|<kubernetesKubeletClusterDomain>|{{WrapAsVariable "kubernetesKubeletClusterDomain"}}|g; s|<kubeDNSServiceIP>|{{WrapAsVariable "kubeDNSServiceIP"}}|g" "/etc/kubernetes/addons/kube-dns-deployment.yaml"
-    sed -i "s|<kubernetesHeapsterSpec>|{{WrapAsVariable "kubernetesHeapsterSpec"}}|g; s|<kubernetesAddonResizerSpec>|{{WrapAsVariable "kubernetesAddonResizerSpec"}}|g" "/etc/kubernetes/addons/kube-heapster-deployment.yaml"
+    # sed -i "s|<kubernetesHeapsterSpec>|{{WrapAsVariable "kubernetesHeapsterSpec"}}|g; s|<kubernetesAddonResizerSpec>|{{WrapAsVariable "kubernetesAddonResizerSpec"}}|g" "/etc/kubernetes/addons/kube-heapster-deployment.yaml"
 
 {{if .OrchestratorProfile.KubernetesConfig.IsDashboardEnabled}}
     sed -i "s|<kubernetesDashboardSpec>|{{WrapAsVariable "kubernetesDashboardSpec"}}|g" "/etc/kubernetes/addons/kubernetes-dashboard-deployment.yaml"

diff --git a/parts/k8s/kubernetesmastercustomscript.sh b/parts/k8s/kubernetesmastercustomscript.sh
@@ -56,11 +56,7 @@ echo `date`,`hostname`, startscript>>/opt/m
 # A delay to start the kubernetes processes is necessary
 # if a reboot is required.  Otherwise, the agents will encounter issue:
 # https://github.com/kubernetes/kubernetes/issues/41185
-if [ -f /var/run/reboot-required ]; then
-    REBOOTREQUIRED=true
-else
-    REBOOTREQUIRED=false
-fi
+REBOOTREQUIRED=false
 
 if [[ ! -z "${MASTER_NODE}" ]]; then
     echo "executing master node provision operations"
@@ -138,15 +134,21 @@ touch "${APISERVER_PUBLIC_KEY_PATH}"
 chmod 0644 "${APISERVER_PUBLIC_KEY_PATH}"
 chown root:root "${APISERVER_PUBLIC_KEY_PATH}"
 
-AZURE_JSON_PATH="/etc/kubernetes/azure.json"
-touch "${AZURE_JSON_PATH}"
-chmod 0600 "${AZURE_JSON_PATH}"
-chown root:root "${AZURE_JSON_PATH}"
-
 set +x
 echo "${KUBELET_PRIVATE_KEY}" | base64 --decode > "${KUBELET_PRIVATE_KEY_PATH}"
 echo "${APISERVER_PUBLIC_KEY}" | base64 --decode > "${APISERVER_PUBLIC_KEY_PATH}"
-cat << EOF > "${AZURE_JSON_PATH}"
+set -x
+
+if [[ ! -z "${MASTER_NODE}" ]]; then
+    echo "MASTER_NODE is non-empty, master node, configure azure json."
+
+    AZURE_JSON_PATH="/etc/kubernetes/azure.json"
+    touch "${AZURE_JSON_PATH}"
+    chmod 0600 "${AZURE_JSON_PATH}"
+    chown root:root "${AZURE_JSON_PATH}"
+
+    set +x
+    cat << EOF > "${AZURE_JSON_PATH}"
 {
     "cloud":"${TARGET_ENVIRONMENT}",
     "tenantId": "${TENANT_ID}",
@@ -173,6 +175,9 @@ cat << EOF > "${AZURE_JSON_PATH}"
     "useInstanceMetadata": ${USE_INSTANCE_METADATA}
 }
 EOF
+else
+    echo "MASTER_NODE is empty, worker node, skip azure json."
+fi
 
 ###########################################################
 # END OF SECRET DATA
@@ -754,6 +759,12 @@ fi
 
 echo "Install complete successfully"
 
+if [ -f /var/run/reboot-required ]; then
+    REBOOTREQUIRED=true
+else
+    REBOOTREQUIRED=false
+fi
+
 if $REBOOTREQUIRED; then
   # wait 1 minute to restart node, so that the custom script extension can complete
   echo 'reboot required, rebooting node in 1 minute'

diff --git a/parts/k8s/kubernetesmasterresources.t b/parts/k8s/kubernetesmasterresources.t
@@ -5,8 +5,8 @@
       "name": "[variables('masterAvailabilitySet')]",
       "properties":
         {
-            "platformFaultDomainCount": "2",
-            "platformUpdateDomainCount": "3",
+            "platformFaultDomainCount": "1",
+            "platformUpdateDomainCount": "1",
 		        "managed" : "true"
         },
       "type": "Microsoft.Compute/availabilitySets"
@@ -26,10 +26,25 @@
         "[concat('Microsoft.Network/publicIPAddresses/', variables('masterPublicIPAddressName'))]"
       ],
 {{end}}
+      "kind": "Storage",
       "location": "[variables('location')]",
       "name": "[variables('masterStorageAccountName')]",
       "properties": {
-        "accountType": "[variables('vmSizesMap')[variables('masterVMSize')].storageAccountType]"
+        "encryption": {
+          "keySource": "Microsoft.Storage",
+          "services": {
+            "blob": {
+              "enabled": true
+            },
+            "file": {
+              "enabled": true
+            }
+          }
+        },
+        "supportsHttpsTrafficOnly": true
+      },
+      "sku": {
+        "name": "[variables('vmSizesMap')[variables('masterVMSize')].storageAccountType]"
       },
       "type": "Microsoft.Storage/storageAccounts"
     },
@@ -79,36 +94,6 @@
       "name": "[variables('nsgName')]",
       "properties": {
         "securityRules": [
-{{if .HasWindows}}
-          {
-            "name": "allow_rdp", 
-            "properties": {
-              "access": "Allow", 
-              "description": "Allow RDP traffic to master", 
-              "destinationAddressPrefix": "*", 
-              "destinationPortRange": "3389-3389", 
-              "direction": "Inbound", 
-              "priority": 102, 
-              "protocol": "Tcp", 
-              "sourceAddressPrefix": "*", 
-              "sourcePortRange": "*"
-            }
-          },
-{{end}}       
-          {
-            "name": "allow_ssh",
-            "properties": {
-              "access": "Allow",
-              "description": "Allow SSH traffic to master",
-              "destinationAddressPrefix": "*",
-              "destinationPortRange": "22-22",
-              "direction": "Inbound",
-              "priority": 101,
-              "protocol": "Tcp",
-              "sourceAddressPrefix": "*",
-              "sourcePortRange": "*"
-            }
-          },
           {
             "name": "allow_kube_tls",
             "properties": {

diff --git a/parts/k8s/kubernetesmastervars.t b/parts/k8s/kubernetesmastervars.t
@@ -188,7 +188,7 @@
     "sshKeyPath": "[concat('/home/',variables('username'),'/.ssh/authorized_keys')]",
 
 {{if .HasStorageAccountDisks}}
-    "apiVersionStorage": "2015-06-15",
+    "apiVersionStorage": "2016-12-01",
     "maxVMsPerStorageAccount": 20,
     "maxStorageAccountsPerAgent": "[div(variables('maxVMsPerPool'),variables('maxVMsPerStorageAccount'))]",
     "dataStorageAccountPrefixSeed": 97,
@@ -211,10 +211,10 @@
 {{end}}
     "provisionScript": "{{GetKubernetesB64Provision}}",
     "mountetcdScript": "{{GetKubernetesB64Mountetcd}}",
-    "provisionScriptParametersCommon": "[concat('TENANT_ID=',variables('tenantID'),' APISERVER_PUBLIC_KEY=',variables('apiserverCertificate'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('singleQuote'),variables('servicePrincipalClientSecret'),variables('singleQuote'),' KUBELET_PRIVATE_KEY=',variables('clientPrivateKey'),' TARGET_ENVIRONMENT=',variables('targetEnvironment'),' NETWORK_POLICY=',variables('networkPolicy'),' FQDNSuffix=',variables('fqdnEndpointSuffix'),' VNET_CNI_PLUGINS_URL=',variables('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',variables('cniPluginsURL'),' MAX_PODS=',variables('maxPods'),' CLOUDPROVIDER_BACKOFF=',variables('cloudProviderBackoff'),' CLOUDPROVIDER_BACKOFF_RETRIES=',variables('cloudProviderBackoffRetries'),' CLOUDPROVIDER_BACKOFF_EXPONENT=',variables('cloudProviderBackoffExponent'),' CLOUDPROVIDER_BACKOFF_DURATION=',variables('cloudProviderBackoffDuration'),' CLOUDPROVIDER_BACKOFF_JITTER=',variables('cloudProviderBackoffJitter'),' CLOUDPROVIDER_RATELIMIT=',variables('cloudProviderRatelimit'),' CLOUDPROVIDER_RATELIMIT_QPS=',variables('cloudProviderRatelimitQPS'),' CLOUDPROVIDER_RATELIMIT_BUCKET=',variables('cloudProviderRatelimitBucket'),' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' CONTAINER_RUNTIME=',variables('containerRuntime'),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]",
+    "provisionScriptParametersCommon": "[concat('KUBELET_PRIVATE_KEY=',variables('clientPrivateKey'),' NETWORK_POLICY=',variables('networkPolicy'),' APISERVER_PUBLIC_KEY=',variables('apiserverCertificate'),' MAX_PODS=',variables('maxPods'),' CONTAINER_RUNTIME=',variables('containerRuntime'))]",
 
 {{if not IsHostedMaster}}
-    "provisionScriptParametersMaster": "[concat('MASTER_NODE=true TOTAL_NODES=',variables('totalNodes'),' APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',variables('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',variables('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',variables('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',variables('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ADMINUSER=',variables('username'))]",
+    "provisionScriptParametersMaster": "[concat('MASTER_NODE=true TOTAL_NODES=',variables('totalNodes'),' TENANT_ID=',variables('tenantID'),' SUBSCRIPTION_ID=',variables('subscriptionId'),' RESOURCE_GROUP=',variables('resourceGroup'),' LOCATION=',variables('location'),' SUBNET=',variables('subnetName'),' NETWORK_SECURITY_GROUP=',variables('nsgName'),' VIRTUAL_NETWORK=',variables('virtualNetworkName'),' VIRTUAL_NETWORK_RESOURCE_GROUP=',variables('virtualNetworkResourceGroupName'),' ROUTE_TABLE=',variables('routeTableName'),' PRIMARY_AVAILABILITY_SET=',variables('primaryAvailabilitySetName'),' SERVICE_PRINCIPAL_CLIENT_ID=',variables('servicePrincipalClientId'),' SERVICE_PRINCIPAL_CLIENT_SECRET=',variables('servicePrincipalClientSecret'),' TARGET_ENVIRONMENT=',variables('targetEnvironment'),' FQDNSuffix=',variables('fqdnEndpointSuffix'),' VNET_CNI_PLUGINS_URL=',variables('vnetCniLinuxPluginsURL'),' CNI_PLUGINS_URL=',variables('cniPluginsURL'),' CLOUDPROVIDER_BACKOFF=',variables('cloudProviderBackoff'),' CLOUDPROVIDER_BACKOFF_RETRIES=',variables('cloudProviderBackoffRetries'),' CLOUDPROVIDER_BACKOFF_EXPONENT=',variables('cloudProviderBackoffExponent'),' CLOUDPROVIDER_BACKOFF_DURATION=',variables('cloudProviderBackoffDuration'),' CLOUDPROVIDER_BACKOFF_JITTER=',variables('cloudProviderBackoffJitter'),' CLOUDPROVIDER_RATELIMIT=',variables('cloudProviderRatelimit'),' CLOUDPROVIDER_RATELIMIT_QPS=',variables('cloudProviderRatelimitQPS'),' CLOUDPROVIDER_RATELIMIT_BUCKET=',variables('cloudProviderRatelimitBucket'),' USE_MANAGED_IDENTITY_EXTENSION=',variables('useManagedIdentityExtension'),' USE_INSTANCE_METADATA=',variables('useInstanceMetadata'),' APISERVER_PRIVATE_KEY=',variables('apiServerPrivateKey'),' CA_CERTIFICATE=',variables('caCertificate'),' CA_PRIVATE_KEY=',variables('caPrivateKey'),' MASTER_FQDN=',variables('masterFqdnPrefix'),' KUBECONFIG_CERTIFICATE=',variables('kubeConfigCertificate'),' KUBECONFIG_KEY=',variables('kubeConfigPrivateKey'),' ETCD_SERVER_CERTIFICATE=',variables('etcdServerCertificate'),' ETCD_CLIENT_CERTIFICATE=',variables('etcdClientCertificate'),' ETCD_SERVER_PRIVATE_KEY=',variables('etcdServerPrivateKey'),' ETCD_CLIENT_PRIVATE_KEY=',variables('etcdClientPrivateKey'),' ETCD_PEER_CERTIFICATES=',string(variables('etcdPeerCertificates')),' ETCD_PEER_PRIVATE_KEYS=',string(variables('etcdPeerPrivateKeys')),' ADMINUSER=',variables('username'),' KUBECONFIG_SERVER=',variables('kubeconfigServer'))]",
 {{end}}
     "generateProxyCertsScript": "{{GetKubernetesB64GenerateProxyCerts}}",
     "orchestratorNameVersionTag": "{{.OrchestratorProfile.OrchestratorType}}:{{.OrchestratorProfile.OrchestratorVersion}}",
@@ -269,7 +269,7 @@
     "nsgName": "[concat(variables('agentNamePrefix'), 'nsg')]",
 {{end}}
     "nsgID": "[resourceId('Microsoft.Network/networkSecurityGroups',variables('nsgName'))]",
-    "primaryAvailabilitySetName": "[concat('{{ (index .AgentPoolProfiles 0).Name }}-availabilitySet-',variables('nameSuffix'))]",
+    "primaryAvailabilitySetName": "[concat('{{ (index .AgentPoolProfiles 1).Name }}-availabilitySet-',variables('nameSuffix'))]",
 {{if not IsHostedMaster }}
     {{if IsPrivateCluster}}
         "kubeconfigServer": "[concat('https://', variables('kubernetesAPIServerIP'), ':443')]",