Kubernetes E2E: test addons if present

test/e2e/dcos/dcos_test.go

@@ -31,7 +31,7 @@ var _ = BeforeSuite(func() {
 
 	engCfg, err := engine.ParseConfig(c.CurrentWorkingDir, c.ClusterDefinition, c.Name)
 	Expect(err).NotTo(HaveOccurred())
-	cs, err := engine.Parse(engCfg.ClusterDefinitionTemplate)
+	cs, err := engine.ParseInput(engCfg.ClusterDefinitionTemplate)
 	Expect(err).NotTo(HaveOccurred())
 	eng = engine.Engine{
 		Config:            engCfg,

test/e2e/engine/template.go

@@ -10,6 +10,7 @@ import (
 	"github.com/Azure/acs-engine/pkg/api"
 	"github.com/Azure/acs-engine/pkg/api/vlabs"
 	"github.com/Azure/acs-engine/pkg/helpers"
+	"github.com/Azure/acs-engine/pkg/i18n"
 	"github.com/Azure/acs-engine/test/e2e/config"
 	"github.com/kelseyhightower/envconfig"
 )
@@ -38,8 +39,9 @@ type Config struct {
 
 // Engine holds necessary information to interact with acs-engine cli
 type Engine struct {
-	Config            *Config
-	ClusterDefinition *api.VlabsARMContainerService // Holds the parsed ClusterDefinition
+	Config             *Config
+	ClusterDefinition  *api.VlabsARMContainerService // Holds the parsed ClusterDefinition
+	ExpandedDefinition *api.ContainerService         // Holds the expanded ClusterDefinition
 }
 
 // ParseConfig will return a new engine config struct taking values from env vars
@@ -69,7 +71,7 @@ func Build(cfg *config.Config, subnetID string) (*Engine, error) {
 		log.Printf("Error while trying to build Engine Configuration:%s\n", err)
 	}
 
-	cs, err := Parse(config.ClusterDefinitionPath)
+	cs, err := ParseInput(config.ClusterDefinitionPath)
 	if err != nil {
 		return nil, err
 	}
@@ -156,6 +158,46 @@ func (e *Engine) HasWindowsAgents() bool {
 	return false
 }
 
+// HasDashboard will return true if kubernetes-dashboard addon is enabled
+func (e *Engine) HasDashboard() bool {
+	for _, addon := range e.ExpandedDefinition.Properties.OrchestratorProfile.KubernetesConfig.Addons {
+		if addon.Name == "kubernetes-dashboard" {
+			return *addon.Enabled
+		}
+	}
+	return false
+}
+
+// HasTiller will return true if tiller addon is enabled
+func (e *Engine) HasTiller() bool {
+	for _, addon := range e.ExpandedDefinition.Properties.OrchestratorProfile.KubernetesConfig.Addons {
+		if addon.Name == "tiller" {
+			return *addon.Enabled
+		}
+	}
+	return false
+}
+
+// HasACIConnector will return true if aci-connector addon is enabled
+func (e *Engine) HasACIConnector() bool {
+	for _, addon := range e.ExpandedDefinition.Properties.OrchestratorProfile.KubernetesConfig.Addons {
+		if addon.Name == "aci-connector" {
+			return *addon.Enabled
+		}
+	}
+	return false
+}
+
+// HasRescheduler will return true if rescheduler addon is enabled
+func (e *Engine) HasRescheduler() bool {
+	for _, addon := range e.ExpandedDefinition.Properties.OrchestratorProfile.KubernetesConfig.Addons {
+		if addon.Name == "rescheduler" {
+			return *addon.Enabled
+		}
+	}
+	return false
+}
+
 // OrchestratorVersion1Dot8AndUp will return true if the orchestrator version is 1.8 and up
 func (e *Engine) OrchestratorVersion1Dot8AndUp() bool {
 	return e.ClusterDefinition.ContainerService.Properties.OrchestratorProfile.OrchestratorVersion >= "1.8"
@@ -172,11 +214,12 @@ func (e *Engine) Write() error {
 	if err != nil {
 		log.Printf("Error while trying to write container service definition to file (%s):%s\n%s\n", e.Config.ClusterDefinitionTemplate, err, string(json))
 	}
+
 	return nil
 }
 
-// Parse takes a template path and will parse that into a api.VlabsARMContainerService
-func Parse(path string) (*api.VlabsARMContainerService, error) {
+// ParseInput takes a template path and will parse that into a api.VlabsARMContainerService
+func ParseInput(path string) (*api.VlabsARMContainerService, error) {
 	contents, err := ioutil.ReadFile(path)
 	if err != nil {
 		log.Printf("Error while trying to read cluster definition at (%s):%s\n", path, err)
@@ -189,3 +232,21 @@ func Parse(path string) (*api.VlabsARMContainerService, error) {
 	}
 	return &cs, nil
 }
+
+// ParseOutput takes the generated api model and will parse that into a api.ContainerService
+func ParseOutput(path string) (*api.ContainerService, error) {
+	locale, err := i18n.LoadTranslations()
+	if err != nil {
+		return nil, fmt.Errorf(fmt.Sprintf("error loading translation files: %s", err.Error()))
+	}
+	apiloader := &api.Apiloader{
+		Translator: &i18n.Translator{
+			Locale: locale,
+		},
+	}
+	containerService, _, err := apiloader.LoadContainerServiceFromFile(path, true, false, nil)
+	if err != nil {
+		return nil, err
+	}
+	return containerService, nil
+}

test/e2e/kubernetes/kubernetes_test.go

@@ -41,11 +41,14 @@ var _ = BeforeSuite(func() {
 
 	engCfg, err := engine.ParseConfig(c.CurrentWorkingDir, c.ClusterDefinition, c.Name)
 	Expect(err).NotTo(HaveOccurred())
-	cs, err := engine.Parse(engCfg.ClusterDefinitionTemplate)
+	csInput, err := engine.ParseInput(engCfg.ClusterDefinitionTemplate)
+	Expect(err).NotTo(HaveOccurred())
+	csGenerated, err := engine.ParseOutput(engCfg.GeneratedDefinitionPath + "/apimodel.json")
 	Expect(err).NotTo(HaveOccurred())
 	eng = engine.Engine{
-		Config:            engCfg,
-		ClusterDefinition: cs,
+		Config:             engCfg,
+		ClusterDefinition:  csInput,
+		ExpandedDefinition: csGenerated,
 	}
 })
 
@@ -121,56 +124,84 @@ var _ = Describe("Azure Container Cluster using the Kubernetes Orchestrator", fu
 		})
 
 		It("should have tiller running", func() {
-			running, err := pod.WaitOnReady("tiller", "kube-system", 3, 30*time.Second, cfg.Timeout)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(running).To(Equal(true))
+			if eng.HasTiller() {
+				running, err := pod.WaitOnReady("tiller", "kube-system", 3, 30*time.Second, cfg.Timeout)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(running).To(Equal(true))
+			} else {
+				Skip("tiller disabled for this cluster, will not test")
+			}
 		})
 
 		It("should be able to access the dashboard from each node", func() {
-			running, err := pod.WaitOnReady("kubernetes-dashboard", "kube-system", 3, 30*time.Second, cfg.Timeout)
-			Expect(err).NotTo(HaveOccurred())
-			Expect(running).To(Equal(true))
-
-			kubeConfig, err := GetConfig()
-			Expect(err).NotTo(HaveOccurred())
-			sshKeyPath := cfg.GetSSHKeyPath()
+			if eng.HasDashboard() {
+				running, err := pod.WaitOnReady("kubernetes-dashboard", "kube-system", 3, 30*time.Second, cfg.Timeout)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(running).To(Equal(true))
 
-			s, err := service.Get("kubernetes-dashboard", "kube-system")
-			Expect(err).NotTo(HaveOccurred())
-			dashboardPort := 80
-			version, err := node.Version()
-			Expect(err).NotTo(HaveOccurred())
+				kubeConfig, err := GetConfig()
+				Expect(err).NotTo(HaveOccurred())
+				sshKeyPath := cfg.GetSSHKeyPath()
 
-			re := regexp.MustCompile("v1.9")
-			if re.FindString(version) != "" {
-				dashboardPort = 443
-			}
-			port := s.GetNodePort(dashboardPort)
+				s, err := service.Get("kubernetes-dashboard", "kube-system")
+				Expect(err).NotTo(HaveOccurred())
+				dashboardPort := 80
+				version, err := node.Version()
+				Expect(err).NotTo(HaveOccurred())
 
-			master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
-			nodeList, err := node.Get()
-			Expect(err).NotTo(HaveOccurred())
+				re := regexp.MustCompile("v1.9")
+				if re.FindString(version) != "" {
+					dashboardPort = 443
+				}
+				port := s.GetNodePort(dashboardPort)
 
-			if !eng.HasWindowsAgents() {
-				for _, node := range nodeList.Nodes {
-					success := false
-					for i := 0; i < 60; i++ {
-						dashboardURL := fmt.Sprintf("http://%s:%v", node.Status.GetAddressByType("InternalIP").Address, port)
-						curlCMD := fmt.Sprintf("curl --max-time 60 %s", dashboardURL)
-						_, err := exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD).CombinedOutput()
-						if err == nil {
-							success = true
-							break
-						}
-						if i > 58 {
-							log.Println(curlCMD)
-							log.Println(err.Error())
-							log.Printf("%#v\n", err)
+				master := fmt.Sprintf("azureuser@%s", kubeConfig.GetServerName())
+				nodeList, err := node.Get()
+				Expect(err).NotTo(HaveOccurred())
+
+				if !eng.HasWindowsAgents() {
+					for _, node := range nodeList.Nodes {
+						success := false
+						for i := 0; i < 60; i++ {
+							dashboardURL := fmt.Sprintf("http://%s:%v", node.Status.GetAddressByType("InternalIP").Address, port)
+							curlCMD := fmt.Sprintf("curl --max-time 60 %s", dashboardURL)
+							_, err := exec.Command("ssh", "-i", sshKeyPath, "-o", "ConnectTimeout=10", "-o", "StrictHostKeyChecking=no", "-o", "UserKnownHostsFile=/dev/null", master, curlCMD).CombinedOutput()
+							if err == nil {
+								success = true
+								break
+							}
+							if i > 58 {
+								log.Println(curlCMD)
+								log.Println(err.Error())
+								log.Printf("%#v\n", err)
+							}
+							time.Sleep(10 * time.Second)
 						}
-						time.Sleep(10 * time.Second)
+						Expect(success).To(BeTrue())
 					}
-					Expect(success).To(BeTrue())
 				}
+			} else {
+				Skip("kubernetes-dashboard disabled for this cluster, will not test")
+			}
+		})
+
+		It("should have aci-connector running", func() {
+			if eng.HasACIConnector() {
+				running, err := pod.WaitOnReady("aci-connector", "kube-system", 3, 30*time.Second, cfg.Timeout)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(running).To(Equal(true))
+			} else {
+				Skip("aci-connector disabled for this cluster, will not test")
+			}
+		})
+
+		It("should have rescheduler running", func() {
+			if eng.HasACIConnector() {
+				running, err := pod.WaitOnReady("rescheduler", "kube-system", 3, 30*time.Second, cfg.Timeout)
+				Expect(err).NotTo(HaveOccurred())
+				Expect(running).To(Equal(true))
+			} else {
+				Skip("rescheduler disabled for this cluster, will not test")
 			}
 		})
 	})

test/e2e/runner.go

@@ -74,7 +74,7 @@ func main() {
 			teardown()
 			log.Fatalf("Error trying to parse Engine config:%s\n", err)
 		}
-		cs, err := engine.Parse(engCfg.ClusterDefinitionTemplate)
+		cs, err := engine.ParseInput(engCfg.ClusterDefinitionTemplate)
 		if err != nil {
 			teardown()
 			log.Fatalf("Error trying to parse engine template into memory:%s\n", err)