clusterctl - Allow for disabling pivoting of cluster

clusterctl/clusterdeployer/clusterdeployer.go

@@ -89,20 +89,23 @@ type ClusterDeployer struct {
 	providerComponents      string
 	addonComponents         string
 	cleanupBootstrapCluster bool
+	pivotCluster            bool
 }
 
 func New(
 	bootstrapProvisioner ClusterProvisioner,
 	clientFactory ClientFactory,
 	providerComponents string,
 	addonComponents string,
-	cleanupBootstrapCluster bool) *ClusterDeployer {
+	cleanupBootstrapCluster bool,
+	pivotCluster bool) *ClusterDeployer {
 	return &ClusterDeployer{
 		bootstrapProvisioner:    bootstrapProvisioner,
 		clientFactory:           clientFactory,
 		providerComponents:      providerComponents,
 		addonComponents:         addonComponents,
 		cleanupBootstrapCluster: cleanupBootstrapCluster,
+		pivotCluster:            pivotCluster,
 	}
 }
 
@@ -149,39 +152,47 @@ func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*cluster
 		return fmt.Errorf("unable to update bootstrap cluster endpoint: %v", err)
 	}
 
-	glog.Info("Creating target cluster")
-	targetClient, err := d.createTargetClusterClient(bootstrapClient, provider, kubeconfigOutput)
+	glog.Info("Creating provisioned cluster client")
+	provisionedClient, err := d.createProvisionedClusterClient(bootstrapClient, provider, kubeconfigOutput)
 	if err != nil {
-		return fmt.Errorf("unable to create target cluster: %v", err)
+		return fmt.Errorf("unable to create provisioned cluster client: %v", err)
 	}
-	defer closeClient(targetClient, "target")
+	defer closeClient(provisionedClient, "provisioned")
 
-	glog.Info("Applying Cluster API stack to target cluster")
-	if err := d.applyClusterAPIStackWithPivoting(targetClient, bootstrapClient); err != nil {
-		return fmt.Errorf("unable to apply cluster api stack to target cluster: %v", err)
-	}
+	targetClient := bootstrapClient
+	targetCluster := "bootstrap"
+	if d.pivotCluster {
+		targetClient = provisionedClient
+		targetCluster = "provisioned"
 
-	glog.Info("Saving provider components to the target cluster")
-	err = d.saveProviderComponentsToCluster(providerComponentsStoreFactory, kubeconfigOutput)
-	if err != nil {
-		return fmt.Errorf("unable to save provider components to target cluster: %v", err)
-	}
+		glog.Info("Applying Cluster API stack to provisioned cluster")
+		if err := d.applyClusterAPIStackWithPivoting(provisionedClient, bootstrapClient); err != nil {
+			return fmt.Errorf("unable to apply cluster api stack to target cluster: %v", err)
+		}
 
-	// For some reason, endpoint doesn't get updated in bootstrap cluster sometimes. So we
-	// update the target cluster endpoint as well to be sure.
-	glog.Infof("Updating target cluster object with master (%s) endpoint", master.Name)
-	if err := d.updateClusterEndpoint(targetClient, provider); err != nil {
-		return fmt.Errorf("unable to update target cluster endpoint: %v", err)
+		glog.Info("Saving provider components to the provisioned cluster")
+		err = d.saveProviderComponentsToCluster(providerComponentsStoreFactory, kubeconfigOutput)
+		if err != nil {
+			return fmt.Errorf("unable to save provider components to provisioned cluster: %v", err)
+		}
+
+		// For some reason, endpoint doesn't get updated in bootstrap cluster sometimes. So we
+		// update the target cluster endpoint as well to be sure.
+		glog.Infof("Updating target cluster object with master (%s) endpoint", master.Name)
+		if err := d.updateClusterEndpoint(provisionedClient, provider); err != nil {
+			return fmt.Errorf("unable to update provisioned cluster endpoint: %v", err)
+		}
 	}
 
-	glog.Info("Creating node machines in target cluster.")
+	glog.Infof("Creating node machines in %s cluster.", targetCluster)
 	if err := targetClient.CreateMachineObjects(nodes); err != nil {
 		return fmt.Errorf("unable to create node machines: %v", err)
 	}
 
 	if d.addonComponents != "" {
-		glog.Info("Creating addons in target cluster.")
-		if err := targetClient.Apply(d.addonComponents); err != nil {
+		// Always create addons in the provisioned cluster
+		glog.Info("Creating addons in provisioned cluster.")
+		if err := provisionedClient.Apply(d.addonComponents); err != nil {
 			return fmt.Errorf("unable to apply addons: %v", err)
 		}
 	}
@@ -231,7 +242,7 @@ func (d *ClusterDeployer) createBootstrapCluster() (ClusterClient, func(), error
 		return nil, cleanupFn, fmt.Errorf("could not create bootstrap control plane: %v", err)
 	}
 
-	if d.cleanupBootstrapCluster {
+	if d.cleanupBootstrapCluster && d.pivotCluster {
 		cleanupFn = func() {
 			glog.Info("Cleaning up bootstrap cluster.")
 			d.bootstrapProvisioner.Delete()
@@ -250,28 +261,28 @@ func (d *ClusterDeployer) createBootstrapCluster() (ClusterClient, func(), error
 	return bootstrapClient, cleanupFn, nil
 }
 
-func (d *ClusterDeployer) createTargetClusterClient(bootstrapClient ClusterClient, provider ProviderDeployer, kubeconfigOutput string) (ClusterClient, error) {
+func (d *ClusterDeployer) createProvisionedClusterClient(bootstrapClient ClusterClient, provider ProviderDeployer, kubeconfigOutput string) (ClusterClient, error) {
 	cluster, master, _, err := getClusterAPIObjects(bootstrapClient)
 	if err != nil {
 		return nil, err
 	}
 
-	glog.V(1).Info("Getting target cluster kubeconfig.")
-	targetKubeconfig, err := waitForKubeconfigReady(provider, cluster, master)
+	glog.V(1).Info("Getting provisioned cluster kubeconfig.")
+	provisionedKubeconfig, err := waitForKubeconfigReady(provider, cluster, master)
 	if err != nil {
-		return nil, fmt.Errorf("unable to get target cluster kubeconfig: %v", err)
+		return nil, fmt.Errorf("unable to get provisioned cluster kubeconfig: %v", err)
 	}
 
-	if err = d.writeKubeconfig(targetKubeconfig, kubeconfigOutput); err != nil {
+	if err = d.writeKubeconfig(provisionedKubeconfig, kubeconfigOutput); err != nil {
 		return nil, err
 	}
 
-	targetClient, err := d.clientFactory.NewClusterClientFromKubeconfig(targetKubeconfig)
+	provisionedClient, err := d.clientFactory.NewClusterClientFromKubeconfig(provisionedKubeconfig)
 	if err != nil {
-		return nil, fmt.Errorf("unable to create target cluster client: %v", err)
+		return nil, fmt.Errorf("unable to create provisioned cluster client: %v", err)
 	}
 
-	return targetClient, nil
+	return provisionedClient, nil
 }
 
 func (d *ClusterDeployer) updateClusterEndpoint(client ClusterClient, provider ProviderDeployer) error {

clusterctl/clusterdeployer/clusterdeployer_test.go

@@ -239,6 +239,7 @@ func TestCreate(t *testing.T) {
 		bootstrapClient          *testClusterClient
 		targetClient             *testClusterClient
 		cleanupExternal          bool
+		pivotCluster             bool
 		expectErr                bool
 		expectExternalExists     bool
 		expectExternalCreated    bool
@@ -250,6 +251,7 @@ func TestCreate(t *testing.T) {
 			targetClient:             &testClusterClient{},
 			bootstrapClient:          &testClusterClient{},
 			cleanupExternal:          true,
+			pivotCluster:             true,
 			expectExternalExists:     false,
 			expectExternalCreated:    true,
 			expectedInternalClusters: 1,
@@ -260,6 +262,7 @@ func TestCreate(t *testing.T) {
 			targetClient:             &testClusterClient{},
 			bootstrapClient:          &testClusterClient{},
 			cleanupExternal:          false,
+			pivotCluster:             true,
 			expectExternalExists:     true,
 			expectExternalCreated:    true,
 			expectedInternalClusters: 1,
@@ -277,6 +280,7 @@ func TestCreate(t *testing.T) {
 			targetClient:            &testClusterClient{},
 			bootstrapClient:         &testClusterClient{},
 			cleanupExternal:         true,
+			pivotCluster:            true,
 			expectExternalCreated:   true,
 			factoryClusterClientErr: fmt.Errorf("Test failure"),
 			expectErr:               true,
@@ -286,6 +290,7 @@ func TestCreate(t *testing.T) {
 			targetClient:          &testClusterClient{},
 			bootstrapClient:       &testClusterClient{ApplyErr: fmt.Errorf("Test failure")},
 			cleanupExternal:       true,
+			pivotCluster:          true,
 			expectExternalCreated: true,
 			expectErr:             true,
 		},
@@ -294,6 +299,7 @@ func TestCreate(t *testing.T) {
 			targetClient:          &testClusterClient{},
 			bootstrapClient:       &testClusterClient{WaitForClusterV1alpha1ReadyErr: fmt.Errorf("Test failure")},
 			cleanupExternal:       true,
+			pivotCluster:          true,
 			expectExternalCreated: true,
 			expectErr:             true,
 		},
@@ -302,6 +308,7 @@ func TestCreate(t *testing.T) {
 			targetClient:          &testClusterClient{},
 			bootstrapClient:       &testClusterClient{GetClusterObjectsErr: fmt.Errorf("Test failure")},
 			cleanupExternal:       true,
+			pivotCluster:          true,
 			expectExternalCreated: true,
 			expectErr:             true,
 		},
@@ -310,6 +317,7 @@ func TestCreate(t *testing.T) {
 			targetClient:          &testClusterClient{},
 			bootstrapClient:       &testClusterClient{GetMachineObjectsErr: fmt.Errorf("Test failure")},
 			cleanupExternal:       true,
+			pivotCluster:          true,
 			expectExternalCreated: true,
 			expectErr:             true,
 		},
@@ -318,6 +326,7 @@ func TestCreate(t *testing.T) {
 			targetClient:          &testClusterClient{},
 			bootstrapClient:       &testClusterClient{CreateClusterObjectErr: fmt.Errorf("Test failure")},
 			cleanupExternal:       true,
+			pivotCluster:          true,
 			expectExternalCreated: true,
 			expectErr:             true,
 		},
@@ -326,6 +335,7 @@ func TestCreate(t *testing.T) {
 			targetClient:          &testClusterClient{},
 			bootstrapClient:       &testClusterClient{CreateMachineObjectsErr: fmt.Errorf("Test failure")},
 			cleanupExternal:       true,
+			pivotCluster:          true,
 			expectExternalCreated: true,
 			expectErr:             true,
 		},
@@ -334,6 +344,7 @@ func TestCreate(t *testing.T) {
 			targetClient:          &testClusterClient{},
 			bootstrapClient:       &testClusterClient{UpdateClusterObjectEndpointErr: fmt.Errorf("Test failure")},
 			cleanupExternal:       true,
+			pivotCluster:          true,
 			expectExternalCreated: true,
 			expectErr:             true,
 		},
@@ -342,6 +353,7 @@ func TestCreate(t *testing.T) {
 			targetClient:          &testClusterClient{ApplyErr: fmt.Errorf("Test failure")},
 			bootstrapClient:       &testClusterClient{},
 			cleanupExternal:       true,
+			pivotCluster:          true,
 			expectExternalCreated: true,
 			expectErr:             true,
 		},
@@ -350,6 +362,7 @@ func TestCreate(t *testing.T) {
 			targetClient:          &testClusterClient{WaitForClusterV1alpha1ReadyErr: fmt.Errorf("Test failure")},
 			bootstrapClient:       &testClusterClient{},
 			cleanupExternal:       true,
+			pivotCluster:          true,
 			expectExternalCreated: true,
 			expectErr:             true,
 		},
@@ -358,6 +371,7 @@ func TestCreate(t *testing.T) {
 			targetClient:          &testClusterClient{CreateClusterObjectErr: fmt.Errorf("Test failure")},
 			bootstrapClient:       &testClusterClient{},
 			cleanupExternal:       true,
+			pivotCluster:          true,
 			expectExternalCreated: true,
 			expectErr:             true,
 		},
@@ -366,6 +380,7 @@ func TestCreate(t *testing.T) {
 			targetClient:             &testClusterClient{CreateMachineObjectsErr: fmt.Errorf("Test failure")},
 			bootstrapClient:          &testClusterClient{},
 			cleanupExternal:          true,
+			pivotCluster:             true,
 			expectExternalCreated:    true,
 			expectedInternalClusters: 1,
 			expectErr:                true,
@@ -375,6 +390,7 @@ func TestCreate(t *testing.T) {
 			targetClient:             &testClusterClient{UpdateClusterObjectEndpointErr: fmt.Errorf("Test failure")},
 			bootstrapClient:          &testClusterClient{},
 			cleanupExternal:          true,
+			pivotCluster:             true,
 			expectExternalCreated:    true,
 			expectedInternalClusters: 1,
 			expectedInternalMachines: 1,
@@ -404,7 +420,7 @@ func TestCreate(t *testing.T) {
 			inputMachines := generateMachines()
 			pcStore := mockProviderComponentsStore{}
 			pcFactory := mockProviderComponentsStoreFactory{NewFromCoreclientsetPCStore: &pcStore}
-			d := New(p, f, "", "", testcase.cleanupExternal)
+			d := New(p, f, "", "", testcase.cleanupExternal, testcase.pivotCluster)
 			err := d.Create(inputCluster, inputMachines, pd, kubeconfigOut, &pcFactory)
 
 			// Validate
@@ -447,7 +463,7 @@ func TestCreateProviderComponentsScenarios(t *testing.T) {
 		expectedError string
 	}{
 		{"success", mockProviderComponentsStore{SaveErr: nil}, ""},
-		{"error when saving", mockProviderComponentsStore{SaveErr: fmt.Errorf("pcstore save error")}, "unable to save provider components to target cluster: error saving provider components: pcstore save error"},
+		{"error when saving", mockProviderComponentsStore{SaveErr: fmt.Errorf("pcstore save error")}, "unable to save provider components to provisioned cluster: error saving provider components: pcstore save error"},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
@@ -468,7 +484,7 @@ func TestCreateProviderComponentsScenarios(t *testing.T) {
 			pcFactory := mockProviderComponentsStoreFactory{NewFromCoreclientsetPCStore: &tc.pcStore}
 			providerComponentsYaml := "-yaml\ndefinition"
 			addonsYaml := "-yaml\ndefinition"
-			d := New(p, f, providerComponentsYaml, addonsYaml, false)
+			d := New(p, f, providerComponentsYaml, addonsYaml, false, true)
 			err := d.Create(inputCluster, inputMachines, pd, kubeconfigOut, &pcFactory)
 			if err == nil && tc.expectedError != "" {
 				t.Fatalf("error mismatch: got '%v', want '%v'", err, tc.expectedError)
@@ -574,7 +590,7 @@ func TestDeleteCleanupExternalCluster(t *testing.T) {
 			f := newTestClusterClientFactory()
 			f.clusterClients[bootstrapKubeconfig] = tc.bootstrapClient
 			f.clusterClients[targetKubeconfig] = tc.targetClient
-			d := New(p, f, "", "", tc.cleanupExternalCluster)
+			d := New(p, f, "", "", tc.cleanupExternalCluster, true)
 			err := d.Delete(tc.targetClient)
 			if err != nil || tc.expectedErrorMessage != "" {
 				if err == nil {
@@ -630,7 +646,7 @@ func TestDeleteBasicScenarios(t *testing.T) {
 			f.clusterClients[bootstrapKubeconfig] = tc.bootstrapClient
 			f.clusterClients[targetKubeconfig] = tc.targetClient
 			f.ClusterClientErr = tc.NewCoreClientsetErr
-			d := New(p, f, "", "", true)
+			d := New(p, f, "", "", true, true)
 			err := d.Delete(tc.targetClient)
 			if err != nil || tc.expectedErrorMessage != "" {
 				if err == nil {

clusterctl/cmd/create_cluster.go

@@ -41,6 +41,7 @@ type CreateOptions struct {
 	Provider                      string
 	KubeconfigOutput              string
 	ExistingClusterKubeconfigPath string
+	NoPivotCluster                bool
 }
 
 var co = &CreateOptions{}
@@ -107,7 +108,8 @@ func RunCreate(co *CreateOptions) error {
 		clusterdeployer.NewClientFactory(),
 		string(pc),
 		string(ac),
-		co.CleanupBootstrapCluster)
+		co.CleanupBootstrapCluster,
+		!co.NoPivotCluster)
 	return d.Create(c, m, pd, co.KubeconfigOutput, pcsFactory)
 }
 
@@ -122,6 +124,7 @@ func init() {
 	// Optional flags
 	createClusterCmd.Flags().StringVarP(&co.AddonComponents, "addon-components", "a", "", "A yaml file containing cluster addons to apply to the internal cluster")
 	createClusterCmd.Flags().BoolVarP(&co.CleanupBootstrapCluster, "cleanup-bootstrap-cluster", "", true, "Whether to cleanup the bootstrap cluster after bootstrap")
+	createClusterCmd.Flags().BoolVarP(&co.NoPivotCluster, "no-pivot-cluster", "", false, "Whether skip pivoting the cluster-api to the provisioned cluster")
 	createClusterCmd.Flags().StringVarP(&co.VmDriver, "vm-driver", "", "", "Which vm driver to use for minikube")
 	createClusterCmd.Flags().StringVarP(&co.KubeconfigOutput, "kubeconfig-out", "", "kubeconfig", "Where to output the kubeconfig for the provisioned cluster")
 	createClusterCmd.Flags().StringVarP(&co.ExistingClusterKubeconfigPath, "existing-bootstrap-cluster-kubeconfig", "", "", "Path to an existing cluster's kubeconfig for bootstrapping (intead of using minikube)")

clusterctl/cmd/delete_cluster.go

@@ -72,6 +72,7 @@ func RunDelete() error {
 		clusterdeployer.NewClientFactory(),
 		providerComponents,
 		"",
+		true,
 		true)
 	return deployer.Delete(clusterClient)
 }

clusterctl/testdata/create-cluster-no-args-invalid-flag.golden

@@ -10,6 +10,7 @@ Flags:
   -h, --help                                           help for cluster
       --kubeconfig-out string                          Where to output the kubeconfig for the provisioned cluster (default "kubeconfig")
   -m, --machines string                                A yaml file containing machine object definition(s)
+      --no-pivot-cluster                               Whether skip pivoting the cluster-api to the provisioned cluster
       --provider string                                Which provider deployment logic to use (google/vsphere/azure)
   -p, --provider-components string                     A yaml file containing cluster api provider controllers and supporting objects
       --vm-driver string                               Which vm driver to use for minikube

clusterctl/testdata/create-cluster-no-args.golden

@@ -12,6 +12,7 @@ Flags:
   -h, --help                                           help for cluster
       --kubeconfig-out string                          Where to output the kubeconfig for the provisioned cluster (default "kubeconfig")
   -m, --machines string                                A yaml file containing machine object definition(s)
+      --no-pivot-cluster                               Whether skip pivoting the cluster-api to the provisioned cluster
       --provider string                                Which provider deployment logic to use (google/vsphere/azure)
   -p, --provider-components string                     A yaml file containing cluster api provider controllers and supporting objects
       --vm-driver string                               Which vm driver to use for minikube

pkg/deployer/clusterapiserver.go

@@ -29,7 +29,7 @@ import (
 	"k8s.io/client-go/util/cert/triple"
 )
 
-var apiServerImage = "gcr.io/k8s-cluster-api/cluster-apiserver:0.0.5"
+var apiServerImage = "gcr.io/k8s-cluster-api/cluster-apiserver:0.0.6"
 
 func init() {
 	if img, ok := os.LookupEnv("CLUSTER_API_SERVER_IMAGE"); ok {