- Add etcdClusterSpec Image & Version in bootstrap data for Master nodes

- Reuse execWithTee fn for ETCD Command (tee & mkfifo in different path for newer image versions)

cmd/kops/integration_test.go

@@ -511,7 +511,7 @@ func runTestCloudformation(t *testing.T, clusterName string, srcDir string, vers
 				t.Logf("actual terraform output in %s", actualPath)
 			}
 
-			t.Fatalf("cloudformation output differed from expected")
+			t.Fatalf("cloudformation output differed from expected. Test file: %s", path.Join(srcDir, expectedCfPath))
 		}
 
 		expectedExtracted, err := ioutil.ReadFile(path.Join(srcDir, expectedCfPath+".extracted.yaml"))
@@ -542,7 +542,7 @@ func runTestCloudformation(t *testing.T, clusterName string, srcDir string, vers
 
 				diffString := diff.FormatDiff(expectedValue, extractedValueTrimmed)
 				t.Logf("diff for key %s:\n%s\n\n\n\n\n\n", key, diffString)
-				t.Fatalf("cloudformation output differed from expected")
+				t.Fatalf("cloudformation output differed from expected. Test file: %s", path.Join(srcDir, expectedCfPath+".extracted.yaml"))
 			}
 		}
 	}

hack/.packages

@@ -155,6 +155,7 @@ k8s.io/kops/upup/pkg/fi/utils
 k8s.io/kops/upup/pkg/kutil
 k8s.io/kops/upup/tools/generators/fitask
 k8s.io/kops/upup/tools/generators/pkg/codegen
+k8s.io/kops/util/pkg/exec
 k8s.io/kops/util/pkg/hashing
 k8s.io/kops/util/pkg/slice
 k8s.io/kops/util/pkg/tables

nodeup/pkg/model/convenience.go

@@ -21,7 +21,6 @@ import (
 	"path/filepath"
 	"sort"
 	"strconv"
-	"strings"
 
 	"github.com/golang/glog"
 	"k8s.io/api/core/v1"
@@ -187,19 +186,3 @@ func addHostPathMapping(pod *v1.Pod, container *v1.Container, name, path string)
 func convEtcdSettingsToMs(dur *metav1.Duration) string {
 	return strconv.FormatInt(dur.Nanoseconds()/1000000, 10)
 }
-
-// execWithTee returns the command to run the command while piping output to both the log file and stdout/stderr
-func execWithTee(cmd string, args []string, logfile string) []string {
-	// exec so we don't have a shell that doesn't pass signals to the real process
-	execCmd := "exec " + cmd + " " + strings.Join(args, " ")
-
-	// NOTE: tee & mkfifo is in /usr/bin in the kube-proxy image, but /bin in other images
-
-	// Bash supports something like this, but dash and other limited shells don't
-	//shCmd := "exec &> >(/usr/bin/tee -a " + logfile + "); " + execCmd
-	// Instead we create the pipe manually and wire up the tee:
-	shCmd := "mkfifo /tmp/pipe; (tee -a " + logfile + " < /tmp/pipe & ) ; " + execCmd + " > /tmp/pipe 2>&1"
-
-	// Execute shell command
-	return []string{"/bin/sh", "-c", shCmd}
-}

nodeup/pkg/model/kube_apiserver.go

@@ -26,6 +26,7 @@ import (
 	"k8s.io/kops/pkg/kubeconfig"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
+	"k8s.io/kops/util/pkg/exec"
 
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -247,7 +248,7 @@ func (b *KubeAPIServerBuilder) buildPod() (*v1.Pod, error) {
 	container := &v1.Container{
 		Name:  "kube-apiserver",
 		Image: b.Cluster.Spec.KubeAPIServer.Image,
-		Command: execWithTee(
+		Command: exec.WithTee(
 			"/usr/local/bin/kube-apiserver",
 			sortedStrings(flags),
 			"/var/log/kube-apiserver.log"),

nodeup/pkg/model/kube_controller_manager.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
+	"k8s.io/kops/util/pkg/exec"
 
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -167,7 +168,7 @@ func (b *KubeControllerManagerBuilder) buildPod() (*v1.Pod, error) {
 	container := &v1.Container{
 		Name:  "kube-controller-manager",
 		Image: b.Cluster.Spec.KubeControllerManager.Image,
-		Command: execWithTee(
+		Command: exec.WithTee(
 			"/usr/local/bin/kube-controller-manager",
 			sortedStrings(flags),
 			"/var/log/kube-controller-manager.log"),

nodeup/pkg/model/kube_proxy.go

@@ -23,6 +23,7 @@ import (
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
+	"k8s.io/kops/util/pkg/exec"
 
 	"github.com/golang/glog"
 	"k8s.io/api/core/v1"
@@ -143,7 +144,7 @@ func (b *KubeProxyBuilder) buildPod() (*v1.Pod, error) {
 	container := &v1.Container{
 		Name:  "kube-proxy",
 		Image: image,
-		Command: execWithTee(
+		Command: exec.WithTee(
 			"/usr/local/bin/kube-proxy",
 			sortedStrings(flags),
 			"/var/log/kube-proxy.log"),

nodeup/pkg/model/kube_scheduler.go

@@ -22,6 +22,7 @@ import (
 	"k8s.io/kops/pkg/flagbuilder"
 	"k8s.io/kops/upup/pkg/fi"
 	"k8s.io/kops/upup/pkg/fi/nodeup/nodetasks"
+	"k8s.io/kops/util/pkg/exec"
 
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/api/resource"
@@ -124,7 +125,7 @@ func (b *KubeSchedulerBuilder) buildPod() (*v1.Pod, error) {
 	container := &v1.Container{
 		Name:  "kube-scheduler",
 		Image: c.Image,
-		Command: execWithTee(
+		Command: exec.WithTee(
 			"/usr/local/bin/kube-scheduler",
 			sortedStrings(flags),
 			"/var/log/kube-scheduler.log"),

pkg/model/bootstrapscript.go

@@ -111,6 +111,14 @@ func (b *BootstrapScript) ResourceNodeUp(ig *kops.InstanceGroup, cs *kops.Cluste
 				spec["kubeControllerManager"] = cs.KubeControllerManager
 				spec["kubeScheduler"] = cs.KubeScheduler
 				spec["masterKubelet"] = cs.MasterKubelet
+				spec["etcdClusters"] = make(map[string]kops.EtcdClusterSpec, 0)
+
+				for _, etcdCluster := range cs.EtcdClusters {
+					spec["etcdClusters"].(map[string]kops.EtcdClusterSpec)[etcdCluster.Name] = kops.EtcdClusterSpec{
+						Image:   etcdCluster.Image,
+						Version: etcdCluster.Version,
+					}
+				}
 			}
 
 			hooks, err := b.getRelevantHooks(cs.Hooks, ig.Spec.Role)

pkg/model/bootstrapscript_test.go

@@ -173,6 +173,18 @@ func makeTestCluster(hookSpecRoles []kops.InstanceGroupRole, fileAssetSpecRoles
 							InstanceGroup: s("ig-1"),
 						},
 					},
+					Version: "3.1.11",
+				},
+				{
+					Name: "events",
+					Members: []*kops.EtcdMemberSpec{
+						{
+							Name:          "test",
+							InstanceGroup: s("ig-1"),
+						},
+					},
+					Version: "3.1.11",
+					Image:   "gcr.io/etcd-development/etcd:v3.1.11",
 				},
 			},
 			NetworkCIDR: "10.79.0.0/24",

pkg/model/tests/data/bootstrapscript_0.txt

@@ -159,6 +159,12 @@ cloudConfig:
 docker:
   logLevel: INFO
 encryptionConfig: null
+etcdClusters:
+  events:
+    image: gcr.io/etcd-development/etcd:v3.1.11
+    version: 3.1.11
+  main:
+    version: 3.1.11
 kubeAPIServer:
   image: CoreOS
 kubeControllerManager:

pkg/model/tests/data/bootstrapscript_1.txt

@@ -159,6 +159,12 @@ cloudConfig:
 docker:
   logLevel: INFO
 encryptionConfig: null
+etcdClusters:
+  events:
+    image: gcr.io/etcd-development/etcd:v3.1.11
+    version: 3.1.11
+  main:
+    version: 3.1.11
 fileAssets:
 - content: E1oeAbrnQsSldrIP1BpoP2SDykM= (fingerprint)
   name: iptables-restore

pkg/model/tests/data/bootstrapscript_2.txt

@@ -159,6 +159,12 @@ cloudConfig:
 docker:
   logLevel: INFO
 encryptionConfig: null
+etcdClusters:
+  events:
+    image: gcr.io/etcd-development/etcd:v3.1.11
+    version: 3.1.11
+  main:
+    version: 3.1.11
 fileAssets:
 - content: E1oeAbrnQsSldrIP1BpoP2SDykM= (fingerprint)
   name: iptables-restore

protokube/pkg/protokube/etcd_manifest.go

@@ -24,6 +24,7 @@ import (
 	"k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/util/intstr"
 	"k8s.io/kops/pkg/kubemanifest"
+	"k8s.io/kops/util/pkg/exec"
 )
 
 // BuildEtcdManifest creates the pod spec, based on the etcd cluster
@@ -44,12 +45,7 @@ func BuildEtcdManifest(c *EtcdCluster) *v1.Pod {
 					v1.ResourceCPU: c.CPURequest,
 				},
 			},
-			Command: []string{
-				"/bin/sh", "-c",
-				"/bin/mkfifo /tmp/pipe; " +
-					"(/bin/tee -a /var/log/etcd.log < /tmp/pipe & ); " +
-					"exec /usr/local/bin/etcd > /tmp/pipe 2>&1",
-			},
+			Command: exec.WithTee("/usr/local/bin/etcd", []string{}, "/var/log/etcd.log"),
 		}
 		// build the environment variables for etcd service
 		container.Env = buildEtcdEnvironmentOptions(c)

protokube/tests/integration/build_etcd_manifest/main/etcd_env_vars.yaml

@@ -37,8 +37,8 @@ spec:
   - command:
     - /bin/sh
     - -c
-    - /bin/mkfifo /tmp/pipe; (/bin/tee -a /var/log/etcd.log < /tmp/pipe & ); exec
-      /usr/local/bin/etcd > /tmp/pipe 2>&1
+    - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /usr/local/bin/etcd  >
+      /tmp/pipe 2>&1
     env:
     - name: ETCD_NAME
       value: node0

protokube/tests/integration/build_etcd_manifest/main/non_tls.yaml

@@ -35,8 +35,8 @@ spec:
   - command:
     - /bin/sh
     - -c
-    - /bin/mkfifo /tmp/pipe; (/bin/tee -a /var/log/etcd.log < /tmp/pipe & ); exec
-      /usr/local/bin/etcd > /tmp/pipe 2>&1
+    - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /usr/local/bin/etcd  >
+      /tmp/pipe 2>&1
     env:
     - name: ETCD_NAME
       value: node0

protokube/tests/integration/build_etcd_manifest/main/tls.yaml

@@ -41,8 +41,8 @@ spec:
   - command:
     - /bin/sh
     - -c
-    - /bin/mkfifo /tmp/pipe; (/bin/tee -a /var/log/etcd.log < /tmp/pipe & ); exec
-      /usr/local/bin/etcd > /tmp/pipe 2>&1
+    - mkfifo /tmp/pipe; (tee -a /var/log/etcd.log < /tmp/pipe & ) ; exec /usr/local/bin/etcd  >
+      /tmp/pipe 2>&1
     env:
     - name: ETCD_NAME
       value: node0

tests/integration/update_cluster/additional_user-data/cloudformation.json.extracted.yaml

@@ -157,6 +157,11 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersadditionaluserda
     storage: overlay,aufs
     version: 1.11.2
   encryptionConfig: null
+  etcdClusters:
+    events:
+      version: 2.2.1
+    main:
+      version: 2.2.1
   kubeAPIServer:
     address: 127.0.0.1
     admissionControl:

tests/integration/update_cluster/minimal-cloudformation/cloudformation.json.extracted.yaml

@@ -148,6 +148,11 @@ Resources.AWSAutoScalingLaunchConfigurationmasterustest1amastersminimalexampleco
     storage: overlay,aufs
     version: 1.11.2
   encryptionConfig: null
+  etcdClusters:
+    events:
+      version: 2.2.1
+    main:
+      version: 2.2.1
   kubeAPIServer:
     address: 127.0.0.1
     admissionControl:

util/pkg/exec/exec.go

@@ -0,0 +1,35 @@
+/*
+Copyright 2018 The Kubernetes Authors.
+
+Licensed under the Apache License, Version 2.0 (the "License");
+you may not use this file except in compliance with the License.
+You may obtain a copy of the License at
+
+    http://www.apache.org/licenses/LICENSE-2.0
+
+Unless required by applicable law or agreed to in writing, software
+distributed under the License is distributed on an "AS IS" BASIS,
+WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+See the License for the specific language governing permissions and
+limitations under the License.
+*/
+
+package exec
+
+import "strings"
+
+// WithTee returns the command to run the command while piping output to both the log file and stdout/stderr
+func WithTee(cmd string, args []string, logfile string) []string {
+	// exec so we don't have a shell that doesn't pass signals to the real process
+	execCmd := "exec " + cmd + " " + strings.Join(args, " ")
+
+	// NOTE: tee & mkfifo is in /usr/bin in the kube-proxy image, but /bin in other images
+
+	// Bash supports something like this, but dash and other limited shells don't
+	//shCmd := "exec &> >(/usr/bin/tee -a " + logfile + "); " + execCmd
+	// Instead we create the pipe manually and wire up the tee:
+	shCmd := "mkfifo /tmp/pipe; (tee -a " + logfile + " < /tmp/pipe & ) ; " + execCmd + " > /tmp/pipe 2>&1"
+
+	// Execute shell command
+	return []string{"/bin/sh", "-c", shCmd}
+}