Skip to content

Commit

Permalink
clusterctl - Do not pivot cluster by default
Browse files Browse the repository at this point in the history 
- Default to not pivoting cluster by default
- Add --pivot-cluster flag to enable pivoting of the cluster
  • Loading branch information
@detiber
detiber committed Sep 4, 2018
1 parent f01f498 commit 631690e
Show file tree
Hide file tree
Showing 6 changed files with 70 additions and 37 deletions.
73 changes: 42 additions & 31 deletions clusterctl/clusterdeployer/clusterdeployer.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -89,20 +89,23 @@ type ClusterDeployer struct {
providerComponents string
addonComponents string
cleanupBootstrapCluster bool
pivotCluster bool
}

func New(
bootstrapProvisioner ClusterProvisioner,
clientFactory ClientFactory,
providerComponents string,
addonComponents string,
cleanupBootstrapCluster bool) *ClusterDeployer {
cleanupBootstrapCluster bool,
pivotCluster bool) *ClusterDeployer {
return &ClusterDeployer{
bootstrapProvisioner: bootstrapProvisioner,
clientFactory: clientFactory,
providerComponents: providerComponents,
addonComponents: addonComponents,
cleanupBootstrapCluster: cleanupBootstrapCluster,
pivotCluster: pivotCluster,
}
}

Expand Down Expand Up @@ -149,39 +152,47 @@ func (d *ClusterDeployer) Create(cluster *clusterv1.Cluster, machines []*cluster
return fmt.Errorf("unable to update bootstrap cluster endpoint: %v", err)
}

glog.Info("Creating target cluster")
targetClient, err := d.createTargetClusterClient(bootstrapClient, provider, kubeconfigOutput)
glog.Info("Creating provisioned cluster client")
provisionedClient, err := d.createProvisionedClusterClient(bootstrapClient, provider, kubeconfigOutput)
if err != nil {
return fmt.Errorf("unable to create target cluster: %v", err)
return fmt.Errorf("unable to create provisioned cluster client: %v", err)
}
defer closeClient(targetClient, "target")
defer closeClient(provisionedClient, "provisioned")

glog.Info("Applying Cluster API stack to target cluster")
if err := d.applyClusterAPIStackWithPivoting(targetClient, bootstrapClient); err != nil {
return fmt.Errorf("unable to apply cluster api stack to target cluster: %v", err)
}
targetClient := bootstrapClient
targetCluster := "bootstrap"
if d.pivotCluster {
targetClient = provisionedClient
targetCluster = "provisioned"

glog.Info("Saving provider components to the target cluster")
err = d.saveProviderComponentsToCluster(providerComponentsStoreFactory, kubeconfigOutput)
if err != nil {
return fmt.Errorf("unable to save provider components to target cluster: %v", err)
}
glog.Info("Applying Cluster API stack to provisioned cluster")
if err := d.applyClusterAPIStackWithPivoting(provisionedClient, bootstrapClient); err != nil {
return fmt.Errorf("unable to apply cluster api stack to target cluster: %v", err)
}

// For some reason, endpoint doesn't get updated in bootstrap cluster sometimes. So we
// update the target cluster endpoint as well to be sure.
glog.Infof("Updating target cluster object with master (%s) endpoint", master.Name)
if err := d.updateClusterEndpoint(targetClient, provider); err != nil {
return fmt.Errorf("unable to update target cluster endpoint: %v", err)
glog.Info("Saving provider components to the provisioned cluster")
err = d.saveProviderComponentsToCluster(providerComponentsStoreFactory, kubeconfigOutput)
if err != nil {
return fmt.Errorf("unable to save provider components to provisioned cluster: %v", err)
}

// For some reason, endpoint doesn't get updated in bootstrap cluster sometimes. So we
// update the target cluster endpoint as well to be sure.
glog.Infof("Updating target cluster object with master (%s) endpoint", master.Name)
if err := d.updateClusterEndpoint(provisionedClient, provider); err != nil {
return fmt.Errorf("unable to update provisioned cluster endpoint: %v", err)
}
}

glog.Info("Creating node machines in target cluster.")
glog.Infof("Creating node machines in %s cluster.", targetCluster)
if err := targetClient.CreateMachineObjects(nodes); err != nil {
return fmt.Errorf("unable to create node machines: %v", err)
}

if d.addonComponents != "" {
glog.Info("Creating addons in target cluster.")
if err := targetClient.Apply(d.addonComponents); err != nil {
// Always create addons in the provisioned cluster
glog.Info("Creating addons in provisioned cluster.")
if err := provisionedClient.Apply(d.addonComponents); err != nil {
return fmt.Errorf("unable to apply addons: %v", err)
}
}
Expand Down Expand Up @@ -231,7 +242,7 @@ func (d *ClusterDeployer) createBootstrapCluster() (ClusterClient, func(), error
return nil, cleanupFn, fmt.Errorf("could not create bootstrap control plane: %v", err)
}

if d.cleanupBootstrapCluster {
if d.cleanupBootstrapCluster && d.pivotCluster {
cleanupFn = func() {
glog.Info("Cleaning up bootstrap cluster.")
d.bootstrapProvisioner.Delete()
Expand All @@ -250,28 +261,28 @@ func (d *ClusterDeployer) createBootstrapCluster() (ClusterClient, func(), error
return bootstrapClient, cleanupFn, nil
}

func (d *ClusterDeployer) createTargetClusterClient(bootstrapClient ClusterClient, provider ProviderDeployer, kubeconfigOutput string) (ClusterClient, error) {
func (d *ClusterDeployer) createProvisionedClusterClient(bootstrapClient ClusterClient, provider ProviderDeployer, kubeconfigOutput string) (ClusterClient, error) {
cluster, master, _, err := getClusterAPIObjects(bootstrapClient)
if err != nil {
return nil, err
}

glog.V(1).Info("Getting target cluster kubeconfig.")
targetKubeconfig, err := waitForKubeconfigReady(provider, cluster, master)
glog.V(1).Info("Getting provisioned cluster kubeconfig.")
provisionedKubeconfig, err := waitForKubeconfigReady(provider, cluster, master)
if err != nil {
return nil, fmt.Errorf("unable to get target cluster kubeconfig: %v", err)
return nil, fmt.Errorf("unable to get provisioned cluster kubeconfig: %v", err)
}

if err = d.writeKubeconfig(targetKubeconfig, kubeconfigOutput); err != nil {
if err = d.writeKubeconfig(provisionedKubeconfig, kubeconfigOutput); err != nil {
return nil, err
}

targetClient, err := d.clientFactory.NewClusterClientFromKubeconfig(targetKubeconfig)
provisionedClient, err := d.clientFactory.NewClusterClientFromKubeconfig(provisionedKubeconfig)
if err != nil {
return nil, fmt.Errorf("unable to create target cluster client: %v", err)
return nil, fmt.Errorf("unable to create provisioned cluster client: %v", err)
}

return targetClient, nil
return provisionedClient, nil
}

func (d *ClusterDeployer) updateClusterEndpoint(client ClusterClient, provider ProviderDeployer) error {
Expand Down
26 changes: 21 additions & 5 deletions clusterctl/clusterdeployer/clusterdeployer_test.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -239,6 +239,7 @@ func TestCreate(t *testing.T) {
bootstrapClient *testClusterClient
targetClient *testClusterClient
cleanupExternal bool
pivotCluster bool
expectErr bool
expectExternalExists bool
expectExternalCreated bool
Expand All @@ -250,6 +251,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{},
bootstrapClient: &testClusterClient{},
cleanupExternal: true,
pivotCluster: true,
expectExternalExists: false,
expectExternalCreated: true,
expectedInternalClusters: 1,
Expand All @@ -260,6 +262,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{},
bootstrapClient: &testClusterClient{},
cleanupExternal: false,
pivotCluster: true,
expectExternalExists: true,
expectExternalCreated: true,
expectedInternalClusters: 1,
Expand All @@ -277,6 +280,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{},
bootstrapClient: &testClusterClient{},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
factoryClusterClientErr: fmt.Errorf("Test failure"),
expectErr: true,
Expand All @@ -286,6 +290,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{},
bootstrapClient: &testClusterClient{ApplyErr: fmt.Errorf("Test failure")},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
expectErr: true,
},
Expand All @@ -294,6 +299,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{},
bootstrapClient: &testClusterClient{WaitForClusterV1alpha1ReadyErr: fmt.Errorf("Test failure")},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
expectErr: true,
},
Expand All @@ -302,6 +308,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{},
bootstrapClient: &testClusterClient{GetClusterObjectsErr: fmt.Errorf("Test failure")},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
expectErr: true,
},
Expand All @@ -310,6 +317,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{},
bootstrapClient: &testClusterClient{GetMachineObjectsErr: fmt.Errorf("Test failure")},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
expectErr: true,
},
Expand All @@ -318,6 +326,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{},
bootstrapClient: &testClusterClient{CreateClusterObjectErr: fmt.Errorf("Test failure")},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
expectErr: true,
},
Expand All @@ -326,6 +335,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{},
bootstrapClient: &testClusterClient{CreateMachineObjectsErr: fmt.Errorf("Test failure")},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
expectErr: true,
},
Expand All @@ -334,6 +344,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{},
bootstrapClient: &testClusterClient{UpdateClusterObjectEndpointErr: fmt.Errorf("Test failure")},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
expectErr: true,
},
Expand All @@ -342,6 +353,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{ApplyErr: fmt.Errorf("Test failure")},
bootstrapClient: &testClusterClient{},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
expectErr: true,
},
Expand All @@ -350,6 +362,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{WaitForClusterV1alpha1ReadyErr: fmt.Errorf("Test failure")},
bootstrapClient: &testClusterClient{},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
expectErr: true,
},
Expand All @@ -358,6 +371,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{CreateClusterObjectErr: fmt.Errorf("Test failure")},
bootstrapClient: &testClusterClient{},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
expectErr: true,
},
Expand All @@ -366,6 +380,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{CreateMachineObjectsErr: fmt.Errorf("Test failure")},
bootstrapClient: &testClusterClient{},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
expectedInternalClusters: 1,
expectErr: true,
Expand All @@ -375,6 +390,7 @@ func TestCreate(t *testing.T) {
targetClient: &testClusterClient{UpdateClusterObjectEndpointErr: fmt.Errorf("Test failure")},
bootstrapClient: &testClusterClient{},
cleanupExternal: true,
pivotCluster: true,
expectExternalCreated: true,
expectedInternalClusters: 1,
expectedInternalMachines: 1,
Expand Down Expand Up @@ -404,7 +420,7 @@ func TestCreate(t *testing.T) {
inputMachines := generateMachines()
pcStore := mockProviderComponentsStore{}
pcFactory := mockProviderComponentsStoreFactory{NewFromCoreclientsetPCStore: &pcStore}
d := New(p, f, "", "", testcase.cleanupExternal)
d := New(p, f, "", "", testcase.cleanupExternal, testcase.pivotCluster)
err := d.Create(inputCluster, inputMachines, pd, kubeconfigOut, &pcFactory)

// Validate
Expand Down Expand Up @@ -447,7 +463,7 @@ func TestCreateProviderComponentsScenarios(t *testing.T) {
expectedError string
}{
{"success", mockProviderComponentsStore{SaveErr: nil}, ""},
{"error when saving", mockProviderComponentsStore{SaveErr: fmt.Errorf("pcstore save error")}, "unable to save provider components to target cluster: error saving provider components: pcstore save error"},
{"error when saving", mockProviderComponentsStore{SaveErr: fmt.Errorf("pcstore save error")}, "unable to save provider components to provisioned cluster: error saving provider components: pcstore save error"},
}
for _, tc := range testCases {
t.Run(tc.name, func(t *testing.T) {
Expand All @@ -468,7 +484,7 @@ func TestCreateProviderComponentsScenarios(t *testing.T) {
pcFactory := mockProviderComponentsStoreFactory{NewFromCoreclientsetPCStore: &tc.pcStore}
providerComponentsYaml := "-yaml\ndefinition"
addonsYaml := "-yaml\ndefinition"
d := New(p, f, providerComponentsYaml, addonsYaml, false)
d := New(p, f, providerComponentsYaml, addonsYaml, false, true)
err := d.Create(inputCluster, inputMachines, pd, kubeconfigOut, &pcFactory)
if err == nil && tc.expectedError != "" {
t.Fatalf("error mismatch: got '%v', want '%v'", err, tc.expectedError)
Expand Down Expand Up @@ -574,7 +590,7 @@ func TestDeleteCleanupExternalCluster(t *testing.T) {
f := newTestClusterClientFactory()
f.clusterClients[bootstrapKubeconfig] = tc.bootstrapClient
f.clusterClients[targetKubeconfig] = tc.targetClient
d := New(p, f, "", "", tc.cleanupExternalCluster)
d := New(p, f, "", "", tc.cleanupExternalCluster, true)
err := d.Delete(tc.targetClient)
if err != nil || tc.expectedErrorMessage != "" {
if err == nil {
Expand Down Expand Up @@ -630,7 +646,7 @@ func TestDeleteBasicScenarios(t *testing.T) {
f.clusterClients[bootstrapKubeconfig] = tc.bootstrapClient
f.clusterClients[targetKubeconfig] = tc.targetClient
f.ClusterClientErr = tc.NewCoreClientsetErr
d := New(p, f, "", "", true)
d := New(p, f, "", "", true, true)
err := d.Delete(tc.targetClient)
if err != nil || tc.expectedErrorMessage != "" {
if err == nil {
Expand Down
5 changes: 4 additions & 1 deletion clusterctl/cmd/create_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -41,6 +41,7 @@ type CreateOptions struct {
Provider string
KubeconfigOutput string
ExistingClusterKubeconfigPath string
PivotCluster bool
}

var co = &CreateOptions{}
Expand Down Expand Up @@ -107,7 +108,8 @@ func RunCreate(co *CreateOptions) error {
clusterdeployer.NewClientFactory(),
string(pc),
string(ac),
co.CleanupBootstrapCluster)
co.CleanupBootstrapCluster,
co.PivotCluster)
return d.Create(c, m, pd, co.KubeconfigOutput, pcsFactory)
}

Expand All @@ -122,6 +124,7 @@ func init() {
// Optional flags
createClusterCmd.Flags().StringVarP(&co.AddonComponents, "addon-components", "a", "", "A yaml file containing cluster addons to apply to the internal cluster")
createClusterCmd.Flags().BoolVarP(&co.CleanupBootstrapCluster, "cleanup-bootstrap-cluster", "", true, "Whether to cleanup the bootstrap cluster after bootstrap")
createClusterCmd.Flags().BoolVarP(&co.PivotCluster, "pivot-cluster", "", false, "Whether to pivot the cluster-api to the provisioned cluster")
createClusterCmd.Flags().StringVarP(&co.VmDriver, "vm-driver", "", "", "Which vm driver to use for minikube")
createClusterCmd.Flags().StringVarP(&co.KubeconfigOutput, "kubeconfig-out", "", "kubeconfig", "Where to output the kubeconfig for the provisioned cluster")
createClusterCmd.Flags().StringVarP(&co.ExistingClusterKubeconfigPath, "existing-bootstrap-cluster-kubeconfig", "", "", "Path to an existing cluster's kubeconfig for bootstrapping (intead of using minikube)")
Expand Down
1 change: 1 addition & 0 deletions clusterctl/cmd/delete_cluster.go
View file
Original file line number Diff line number Diff line change
Expand Up @@ -72,6 +72,7 @@ func RunDelete() error {
clusterdeployer.NewClientFactory(),
providerComponents,
"",
true,
true)
return deployer.Delete(clusterClient)
}
Expand Down
1 change: 1 addition & 0 deletions clusterctl/testdata/create-cluster-no-args-invalid-flag.golden
View file
Original file line number Diff line number Diff line change
Expand Up @@ -10,6 +10,7 @@ Flags:
-h, --help help for cluster
--kubeconfig-out string Where to output the kubeconfig for the provisioned cluster (default "kubeconfig")
-m, --machines string A yaml file containing machine object definition(s)
--pivot-cluster Whether to pivot the cluster-api to the provisioned cluster
--provider string Which provider deployment logic to use (google/vsphere/azure)
-p, --provider-components string A yaml file containing cluster api provider controllers and supporting objects
--vm-driver string Which vm driver to use for minikube
Expand Down
1 change: 1 addition & 0 deletions clusterctl/testdata/create-cluster-no-args.golden
View file
Original file line number Diff line number Diff line change
Expand Up @@ -12,6 +12,7 @@ Flags:
-h, --help help for cluster
--kubeconfig-out string Where to output the kubeconfig for the provisioned cluster (default "kubeconfig")
-m, --machines string A yaml file containing machine object definition(s)
--pivot-cluster Whether to pivot the cluster-api to the provisioned cluster
--provider string Which provider deployment logic to use (google/vsphere/azure)
-p, --provider-components string A yaml file containing cluster api provider controllers and supporting objects
--vm-driver string Which vm driver to use for minikube
Expand Down

0 comments on commit 631690e

Please sign in to comment.