diff --git a/clusterctl/clusterdeployer/clusterdeployer.go b/clusterctl/clusterdeployer/clusterdeployer.go index 7265a181d0be..a98519c3c615 100644 --- a/clusterctl/clusterdeployer/clusterdeployer.go +++ b/clusterctl/clusterdeployer/clusterdeployer.go @@ -89,6 +89,7 @@ type ClusterDeployer struct { providerComponents string addonComponents string cleanupBootstrapCluster bool + pivotCluster bool } func New( @@ -96,13 +97,15 @@ func New( clientFactory ClientFactory, providerComponents string, addonComponents string, - cleanupBootstrapCluster bool) *ClusterDeployer { + cleanupBootstrapCluster bool, + pivotCluster bool) *ClusterDeployer { return &ClusterDeployer{ bootstrapProvisioner: bootstrapProvisioner, clientFactory: clientFactory, providerComponents: providerComponents, addonComponents: addonComponents, cleanupBootstrapCluster: cleanupBootstrapCluster, + pivotCluster: pivotCluster, } } @@ -149,39 +152,47 @@ func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*cluster return fmt.Errorf("unable to update bootstrap cluster endpoint: %v", err) } - glog.Info("Creating target cluster") - targetClient, err := d.createTargetClusterClient(bootstrapClient, provider, kubeconfigOutput) + glog.Info("Creating provisioned cluster client") + provisionedClient, err := d.createProvisionedClusterClient(bootstrapClient, provider, kubeconfigOutput) if err != nil { - return fmt.Errorf("unable to create target cluster: %v", err) + return fmt.Errorf("unable to create provisioned cluster client: %v", err) } - defer closeClient(targetClient, "target") + defer closeClient(provisionedClient, "provisioned") - glog.Info("Applying Cluster API stack to target cluster") - if err := d.applyClusterAPIStackWithPivoting(targetClient, bootstrapClient); err != nil { - return fmt.Errorf("unable to apply cluster api stack to target cluster: %v", err) - } + targetClient := bootstrapClient + targetCluster := "bootstrap" + if d.pivotCluster { + targetClient = provisionedClient + targetCluster = "provisioned" - glog.Info("Saving provider components to the target cluster") - err = d.saveProviderComponentsToCluster(providerComponentsStoreFactory, kubeconfigOutput) - if err != nil { - return fmt.Errorf("unable to save provider components to target cluster: %v", err) - } + glog.Info("Applying Cluster API stack to provisioned cluster") + if err := d.applyClusterAPIStackWithPivoting(provisionedClient, bootstrapClient); err != nil { + return fmt.Errorf("unable to apply cluster api stack to target cluster: %v", err) + } - // For some reason, endpoint doesn't get updated in bootstrap cluster sometimes. So we - // update the target cluster endpoint as well to be sure. - glog.Infof("Updating target cluster object with master (%s) endpoint", master.Name) - if err := d.updateClusterEndpoint(targetClient, provider); err != nil { - return fmt.Errorf("unable to update target cluster endpoint: %v", err) + glog.Info("Saving provider components to the provisioned cluster") + err = d.saveProviderComponentsToCluster(providerComponentsStoreFactory, kubeconfigOutput) + if err != nil { + return fmt.Errorf("unable to save provider components to provisioned cluster: %v", err) + } + + // For some reason, endpoint doesn't get updated in bootstrap cluster sometimes. So we + // update the target cluster endpoint as well to be sure. + glog.Infof("Updating target cluster object with master (%s) endpoint", master.Name) + if err := d.updateClusterEndpoint(provisionedClient, provider); err != nil { + return fmt.Errorf("unable to update provisioned cluster endpoint: %v", err) + } } - glog.Info("Creating node machines in target cluster.") + glog.Infof("Creating node machines in %s cluster.", targetCluster) if err := targetClient.CreateMachineObjects(nodes); err != nil { return fmt.Errorf("unable to create node machines: %v", err) } if d.addonComponents != "" { - glog.Info("Creating addons in target cluster.") - if err := targetClient.Apply(d.addonComponents); err != nil { + // Always create addons in the provisioned cluster + glog.Info("Creating addons in provisioned cluster.") + if err := provisionedClient.Apply(d.addonComponents); err != nil { return fmt.Errorf("unable to apply addons: %v", err) } } @@ -231,7 +242,7 @@ func (d *ClusterDeployer) createBootstrapCluster() (ClusterClient, func(), error return nil, cleanupFn, fmt.Errorf("could not create bootstrap control plane: %v", err) } - if d.cleanupBootstrapCluster { + if d.cleanupBootstrapCluster && d.pivotCluster { cleanupFn = func() { glog.Info("Cleaning up bootstrap cluster.") d.bootstrapProvisioner.Delete() @@ -250,28 +261,28 @@ func (d *ClusterDeployer) createBootstrapCluster() (ClusterClient, func(), error return bootstrapClient, cleanupFn, nil } -func (d *ClusterDeployer) createTargetClusterClient(bootstrapClient ClusterClient, provider ProviderDeployer, kubeconfigOutput string) (ClusterClient, error) { +func (d *ClusterDeployer) createProvisionedClusterClient(bootstrapClient ClusterClient, provider ProviderDeployer, kubeconfigOutput string) (ClusterClient, error) { cluster, master, _, err := getClusterAPIObjects(bootstrapClient) if err != nil { return nil, err } - glog.V(1).Info("Getting target cluster kubeconfig.") - targetKubeconfig, err := waitForKubeconfigReady(provider, cluster, master) + glog.V(1).Info("Getting provisioned cluster kubeconfig.") + provisionedKubeconfig, err := waitForKubeconfigReady(provider, cluster, master) if err != nil { - return nil, fmt.Errorf("unable to get target cluster kubeconfig: %v", err) + return nil, fmt.Errorf("unable to get provisioned cluster kubeconfig: %v", err) } - if err = d.writeKubeconfig(targetKubeconfig, kubeconfigOutput); err != nil { + if err = d.writeKubeconfig(provisionedKubeconfig, kubeconfigOutput); err != nil { return nil, err } - targetClient, err := d.clientFactory.NewClusterClientFromKubeconfig(targetKubeconfig) + provisionedClient, err := d.clientFactory.NewClusterClientFromKubeconfig(provisionedKubeconfig) if err != nil { - return nil, fmt.Errorf("unable to create target cluster client: %v", err) + return nil, fmt.Errorf("unable to create provisioned cluster client: %v", err) } - return targetClient, nil + return provisionedClient, nil } func (d *ClusterDeployer) updateClusterEndpoint(client ClusterClient, provider ProviderDeployer) error { diff --git a/clusterctl/clusterdeployer/clusterdeployer_test.go b/clusterctl/clusterdeployer/clusterdeployer_test.go index da31a4d7631c..afaa59c4756f 100644 --- a/clusterctl/clusterdeployer/clusterdeployer_test.go +++ b/clusterctl/clusterdeployer/clusterdeployer_test.go @@ -239,6 +239,7 @@ func TestCreate(t *testing.T) { bootstrapClient *testClusterClient targetClient *testClusterClient cleanupExternal bool + pivotCluster bool expectErr bool expectExternalExists bool expectExternalCreated bool @@ -250,6 +251,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{}, bootstrapClient: &testClusterClient{}, cleanupExternal: true, + pivotCluster: true, expectExternalExists: false, expectExternalCreated: true, expectedInternalClusters: 1, @@ -260,6 +262,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{}, bootstrapClient: &testClusterClient{}, cleanupExternal: false, + pivotCluster: true, expectExternalExists: true, expectExternalCreated: true, expectedInternalClusters: 1, @@ -277,6 +280,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{}, bootstrapClient: &testClusterClient{}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, factoryClusterClientErr: fmt.Errorf("Test failure"), expectErr: true, @@ -286,6 +290,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{}, bootstrapClient: &testClusterClient{ApplyErr: fmt.Errorf("Test failure")}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, expectErr: true, }, @@ -294,6 +299,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{}, bootstrapClient: &testClusterClient{WaitForClusterV1alpha1ReadyErr: fmt.Errorf("Test failure")}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, expectErr: true, }, @@ -302,6 +308,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{}, bootstrapClient: &testClusterClient{GetClusterObjectsErr: fmt.Errorf("Test failure")}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, expectErr: true, }, @@ -310,6 +317,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{}, bootstrapClient: &testClusterClient{GetMachineObjectsErr: fmt.Errorf("Test failure")}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, expectErr: true, }, @@ -318,6 +326,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{}, bootstrapClient: &testClusterClient{CreateClusterObjectErr: fmt.Errorf("Test failure")}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, expectErr: true, }, @@ -326,6 +335,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{}, bootstrapClient: &testClusterClient{CreateMachineObjectsErr: fmt.Errorf("Test failure")}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, expectErr: true, }, @@ -334,6 +344,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{}, bootstrapClient: &testClusterClient{UpdateClusterObjectEndpointErr: fmt.Errorf("Test failure")}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, expectErr: true, }, @@ -342,6 +353,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{ApplyErr: fmt.Errorf("Test failure")}, bootstrapClient: &testClusterClient{}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, expectErr: true, }, @@ -350,6 +362,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{WaitForClusterV1alpha1ReadyErr: fmt.Errorf("Test failure")}, bootstrapClient: &testClusterClient{}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, expectErr: true, }, @@ -358,6 +371,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{CreateClusterObjectErr: fmt.Errorf("Test failure")}, bootstrapClient: &testClusterClient{}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, expectErr: true, }, @@ -366,6 +380,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{CreateMachineObjectsErr: fmt.Errorf("Test failure")}, bootstrapClient: &testClusterClient{}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, expectedInternalClusters: 1, expectErr: true, @@ -375,6 +390,7 @@ func TestCreate(t *testing.T) { targetClient: &testClusterClient{UpdateClusterObjectEndpointErr: fmt.Errorf("Test failure")}, bootstrapClient: &testClusterClient{}, cleanupExternal: true, + pivotCluster: true, expectExternalCreated: true, expectedInternalClusters: 1, expectedInternalMachines: 1, @@ -404,7 +420,7 @@ func TestCreate(t *testing.T) { inputMachines := generateMachines() pcStore := mockProviderComponentsStore{} pcFactory := mockProviderComponentsStoreFactory{NewFromCoreclientsetPCStore: &pcStore} - d := New(p, f, "", "", testcase.cleanupExternal) + d := New(p, f, "", "", testcase.cleanupExternal, testcase.pivotCluster) err := d.Create(inputCluster, inputMachines, pd, kubeconfigOut, &pcFactory) // Validate @@ -447,7 +463,7 @@ func TestCreateProviderComponentsScenarios(t *testing.T) { expectedError string }{ {"success", mockProviderComponentsStore{SaveErr: nil}, ""}, - {"error when saving", mockProviderComponentsStore{SaveErr: fmt.Errorf("pcstore save error")}, "unable to save provider components to target cluster: error saving provider components: pcstore save error"}, + {"error when saving", mockProviderComponentsStore{SaveErr: fmt.Errorf("pcstore save error")}, "unable to save provider components to provisioned cluster: error saving provider components: pcstore save error"}, } for _, tc := range testCases { t.Run(tc.name, func(t *testing.T) { @@ -468,7 +484,7 @@ func TestCreateProviderComponentsScenarios(t *testing.T) { pcFactory := mockProviderComponentsStoreFactory{NewFromCoreclientsetPCStore: &tc.pcStore} providerComponentsYaml := "-yaml\ndefinition" addonsYaml := "-yaml\ndefinition" - d := New(p, f, providerComponentsYaml, addonsYaml, false) + d := New(p, f, providerComponentsYaml, addonsYaml, false, true) err := d.Create(inputCluster, inputMachines, pd, kubeconfigOut, &pcFactory) if err == nil && tc.expectedError != "" { t.Fatalf("error mismatch: got '%v', want '%v'", err, tc.expectedError) @@ -574,7 +590,7 @@ func TestDeleteCleanupExternalCluster(t *testing.T) { f := newTestClusterClientFactory() f.clusterClients[bootstrapKubeconfig] = tc.bootstrapClient f.clusterClients[targetKubeconfig] = tc.targetClient - d := New(p, f, "", "", tc.cleanupExternalCluster) + d := New(p, f, "", "", tc.cleanupExternalCluster, true) err := d.Delete(tc.targetClient) if err != nil || tc.expectedErrorMessage != "" { if err == nil { @@ -630,7 +646,7 @@ func TestDeleteBasicScenarios(t *testing.T) { f.clusterClients[bootstrapKubeconfig] = tc.bootstrapClient f.clusterClients[targetKubeconfig] = tc.targetClient f.ClusterClientErr = tc.NewCoreClientsetErr - d := New(p, f, "", "", true) + d := New(p, f, "", "", true, true) err := d.Delete(tc.targetClient) if err != nil || tc.expectedErrorMessage != "" { if err == nil { diff --git a/clusterctl/cmd/create_cluster.go b/clusterctl/cmd/create_cluster.go index cadc422c33d8..ca72cc26e9c9 100644 --- a/clusterctl/cmd/create_cluster.go +++ b/clusterctl/cmd/create_cluster.go @@ -41,6 +41,7 @@ type CreateOptions struct { Provider string KubeconfigOutput string ExistingClusterKubeconfigPath string + NoPivotCluster bool } var co = &CreateOptions{} @@ -107,7 +108,8 @@ func RunCreate(co *CreateOptions) error { clusterdeployer.NewClientFactory(), string(pc), string(ac), - co.CleanupBootstrapCluster) + co.CleanupBootstrapCluster, + !co.NoPivotCluster) return d.Create(c, m, pd, co.KubeconfigOutput, pcsFactory) } @@ -122,6 +124,7 @@ func init() { // Optional flags createClusterCmd.Flags().StringVarP(&co.AddonComponents, "addon-components", "a", "", "A yaml file containing cluster addons to apply to the internal cluster") createClusterCmd.Flags().BoolVarP(&co.CleanupBootstrapCluster, "cleanup-bootstrap-cluster", "", true, "Whether to cleanup the bootstrap cluster after bootstrap") + createClusterCmd.Flags().BoolVarP(&co.NoPivotCluster, "no-pivot-cluster", "", false, "Whether skip pivoting the cluster-api to the provisioned cluster") createClusterCmd.Flags().StringVarP(&co.VmDriver, "vm-driver", "", "", "Which vm driver to use for minikube") createClusterCmd.Flags().StringVarP(&co.KubeconfigOutput, "kubeconfig-out", "", "kubeconfig", "Where to output the kubeconfig for the provisioned cluster") createClusterCmd.Flags().StringVarP(&co.ExistingClusterKubeconfigPath, "existing-bootstrap-cluster-kubeconfig", "", "", "Path to an existing cluster's kubeconfig for bootstrapping (intead of using minikube)") diff --git a/clusterctl/cmd/delete_cluster.go b/clusterctl/cmd/delete_cluster.go index a5feeb1ad8ea..9e94c7931034 100644 --- a/clusterctl/cmd/delete_cluster.go +++ b/clusterctl/cmd/delete_cluster.go @@ -72,6 +72,7 @@ func RunDelete() error { clusterdeployer.NewClientFactory(), providerComponents, "", + true, true) return deployer.Delete(clusterClient) } diff --git a/clusterctl/testdata/create-cluster-no-args-invalid-flag.golden b/clusterctl/testdata/create-cluster-no-args-invalid-flag.golden index 69e1aa9bb108..5b8c2d0e12fb 100644 --- a/clusterctl/testdata/create-cluster-no-args-invalid-flag.golden +++ b/clusterctl/testdata/create-cluster-no-args-invalid-flag.golden @@ -10,6 +10,7 @@ Flags: -h, --help help for cluster --kubeconfig-out string Where to output the kubeconfig for the provisioned cluster (default "kubeconfig") -m, --machines string A yaml file containing machine object definition(s) + --no-pivot-cluster Whether skip pivoting the cluster-api to the provisioned cluster --provider string Which provider deployment logic to use (google/vsphere/azure) -p, --provider-components string A yaml file containing cluster api provider controllers and supporting objects --vm-driver string Which vm driver to use for minikube diff --git a/clusterctl/testdata/create-cluster-no-args.golden b/clusterctl/testdata/create-cluster-no-args.golden index 94004907ded1..d56dcf29e751 100644 --- a/clusterctl/testdata/create-cluster-no-args.golden +++ b/clusterctl/testdata/create-cluster-no-args.golden @@ -12,6 +12,7 @@ Flags: -h, --help help for cluster --kubeconfig-out string Where to output the kubeconfig for the provisioned cluster (default "kubeconfig") -m, --machines string A yaml file containing machine object definition(s) + --no-pivot-cluster Whether skip pivoting the cluster-api to the provisioned cluster --provider string Which provider deployment logic to use (google/vsphere/azure) -p, --provider-components string A yaml file containing cluster api provider controllers and supporting objects --vm-driver string Which vm driver to use for minikube diff --git a/pkg/deployer/clusterapiserver.go b/pkg/deployer/clusterapiserver.go index a5d054d034e9..a8601f357c0b 100644 --- a/pkg/deployer/clusterapiserver.go +++ b/pkg/deployer/clusterapiserver.go @@ -29,7 +29,7 @@ import ( "k8s.io/client-go/util/cert/triple" ) -var apiServerImage = "gcr.io/k8s-cluster-api/cluster-apiserver:0.0.5" +var apiServerImage = "gcr.io/k8s-cluster-api/cluster-apiserver:0.0.6" func init() { if img, ok := os.LookupEnv("CLUSTER_API_SERVER_IMAGE"); ok {