Skip to content

Commit

Permalink
feat(google_container_cluster): support enable k8s beta apis (#8355)
Browse files Browse the repository at this point in the history 
* feat(google_container_cluster): support enable k8s beta apis

Signed-off-by: toVersus <[email protected]>

* use TypeSet to avoid the re-ordering being treated as a diff
* add custom diff to recreate cluster when removing enabled apis
* add unit and integration tests

* fix: use contains instead of sets intersection

* Update mmv1/third_party/terraform/website/docs/r/container_cluster.html.markdown

---------

Co-authored-by: Stephen Lewis (Burrows) <[email protected]>
  • Loading branch information
@toVersus @melinath
toVersus and melinath authored Jul 27, 2023
1 parent 210b691 commit 27213b2
Show file tree
Hide file tree
Showing 4 changed files with 337 additions and 1 deletion.
121 changes: 121 additions & 0 deletions mmv1/third_party/terraform/services/container/resource_container_cluster.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -192,6 +192,7 @@ func ResourceContainerCluster() *schema.Resource {
containerClusterNodeVersionRemoveDefaultCustomizeDiff,
containerClusterNetworkPolicyEmptyCustomizeDiff,
containerClusterSurgeSettingsCustomizeDiff,
containerClusterEnableK8sBetaApisCustomizeDiff,
),

Timeouts: &schema.ResourceTimeout{
Expand Down Expand Up @@ -819,6 +820,23 @@ func ResourceContainerCluster() *schema.Resource {
Description: `Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days.`,
},

"enable_k8s_beta_apis": {
Type: schema.TypeList,
Optional: true,
MaxItems: 1,
Description: `Configuration for Kubernetes Beta APIs.`,
Elem: &schema.Resource{
Schema: map[string]*schema.Schema{
"enabled_apis": {
Type: schema.TypeSet,
Required: true,
Elem: &schema.Schema{Type: schema.TypeString},
Description: `Enabled Kubernetes Beta APIs.`,
},
},
},
},

"enable_tpu": {
Type: schema.TypeBool,
Optional: true,
Expand Down Expand Up @@ -2123,6 +2141,7 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er
ProtectConfig: expandProtectConfig(d.Get("protect_config")),
<% end -%>
CostManagementConfig: expandCostManagementConfig(d.Get("cost_management_config")),
EnableK8sBetaApis: expandEnableK8sBetaApis(d.Get("enable_k8s_beta_apis"), nil),
}

v:= d.Get("enable_shielded_nodes")
Expand Down Expand Up @@ -2660,6 +2679,9 @@ func resourceContainerClusterRead(d *schema.ResourceData, meta interface{}) erro
if err := d.Set("gateway_api_config", flattenGatewayApiConfig(cluster.NetworkConfig.GatewayApiConfig)); err != nil {
return err
}
if err := d.Set("enable_k8s_beta_apis", flattenEnableK8sBetaApis(cluster.EnableK8sBetaApis)); err != nil {
return err
}
if err := d.Set("logging_config", flattenContainerClusterLoggingConfig(cluster.LoggingConfig)); err != nil {
return err
}
Expand Down Expand Up @@ -3718,6 +3740,44 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er
}
}

if d.HasChange("enable_k8s_beta_apis") {
log.Print("[INFO] Enable Kubernetes Beta APIs")
if v, ok := d.GetOk("enable_k8s_beta_apis"); ok {
name := containerClusterFullName(project, location, clusterName)
clusterGetCall := config.NewContainerClient(userAgent).Projects.Locations.Clusters.Get(name)
if config.UserProjectOverride {
clusterGetCall.Header().Add("X-Goog-User-Project", project)
}
// Fetch the cluster information to get the already enabled Beta APIs.
cluster, err := clusterGetCall.Do()
if err != nil {
return err
}

// To avoid an already enabled Beta APIs error, we need to deduplicate the requested APIs
// with those that are already enabled.
var enabledAPIs []string
if cluster.EnableK8sBetaApis != nil && len(cluster.EnableK8sBetaApis.EnabledApis) > 0 {
enabledAPIs = cluster.EnableK8sBetaApis.EnabledApis
}
enableK8sBetaAPIs := expandEnableK8sBetaApis(v, enabledAPIs)

req := &container.UpdateClusterRequest{
Update: &container.ClusterUpdate{
DesiredK8sBetaApis: enableK8sBetaAPIs,
},
}

updateF := updateFunc(req, "updating enabled Kubernetes Beta APIs")
// Call update serially.
if err := transport_tpg.LockedCall(lockKey, updateF); err != nil {
return err
}

log.Printf("[INFO] GKE cluster %s enabled Kubernetes Beta APIs has been updated", d.Id())
}
}

if d.HasChange("node_pool_defaults") && d.HasChange("node_pool_defaults.0.node_config_defaults.0.logging_variant") {
if v, ok := d.GetOk("node_pool_defaults.0.node_config_defaults.0.logging_variant"); ok {
loggingVariant := v.(string)
Expand Down Expand Up @@ -4900,6 +4960,28 @@ func expandGatewayApiConfig(configured interface{}) *container.GatewayAPIConfig
}
}

func expandEnableK8sBetaApis(configured interface{}, enabledAPIs []string) *container.K8sBetaAPIConfig {
l := configured.([]interface{})
if len(l) == 0 || l[0] == nil {
return nil
}

config := l[0].(map[string]interface{})
result := &container.K8sBetaAPIConfig{}
if v, ok := config["enabled_apis"]; ok {
notEnabledAPIsSet := v.(*schema.Set)
for _, enabledAPI := range enabledAPIs {
if notEnabledAPIsSet.Contains(enabledAPI) {
notEnabledAPIsSet.Remove(enabledAPI)
}
}

result.EnabledApis = tpgresource.ConvertStringSet(notEnabledAPIsSet)
}

return result
}

func expandContainerClusterLoggingConfig(configured interface{}) *container.LoggingConfig {
l := configured.([]interface{})
if len(l) == 0 {
Expand Down Expand Up @@ -5706,6 +5788,17 @@ func flattenGatewayApiConfig(c *container.GatewayAPIConfig) []map[string]interfa
}
}

func flattenEnableK8sBetaApis(c *container.K8sBetaAPIConfig) []map[string]interface{} {
if c == nil {
return nil
}
return []map[string]interface{}{
{
"enabled_apis": c.EnabledApis,
},
}
}

func flattenContainerClusterLoggingConfig(c *container.LoggingConfig) []map[string]interface{} {
if c == nil {
return nil
Expand Down Expand Up @@ -6015,3 +6108,31 @@ func containerClusterSurgeSettingsCustomizeDiff(_ context.Context, d *schema.Res

return nil
}

func containerClusterEnableK8sBetaApisCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta interface{}) error {
// separate func to allow unit testing
return containerClusterEnableK8sBetaApisCustomizeDiffFunc(d)
}

func containerClusterEnableK8sBetaApisCustomizeDiffFunc(d tpgresource.TerraformResourceDiff) error {
// The Kubernetes Beta APIs cannot be disabled once they have been enabled by users.
// The reason why we don't allow disabling is that the controller does not have the
// ability to clean up the Kubernetes objects created by the APIs. If the user
// removes the already enabled Kubernetes Beta API from the list, we need to force
// a new cluster.
if !d.HasChange("enable_k8s_beta_apis.0.enabled_apis") {
return nil
}
old, new := d.GetChange("enable_k8s_beta_apis.0.enabled_apis")
if old != "" && new != "" {
oldAPIsSet := old.(*schema.Set)
newAPIsSet := new.(*schema.Set)
for _, oldAPI := range oldAPIsSet.List() {
if !newAPIsSet.Contains(oldAPI) {
return d.ForceNew("enable_k8s_beta_apis.0.enabled_apis")
}
}
}

return nil
}
101 changes: 100 additions & 1 deletion .../third_party/terraform/services/container/resource_container_cluster_internal_test.go.erb
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2,10 +2,14 @@
package container

import (
<% unless version == 'ga' -%>
"testing"

"github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
<% unless version == 'ga' -%>
"github.com/hashicorp/terraform-provider-google-beta/google-beta/tpgresource"
container "google.golang.org/api/container/v1beta1"
<% else -%>
"github.com/hashicorp/terraform-provider-google/google/tpgresource"
<% end -%>
)

Expand Down Expand Up @@ -91,3 +95,98 @@ func TestValidateNodePoolAutoConfig(t *testing.T) {
}
}
<% end -%>


func TestContainerClusterEnableK8sBetaApisCustomizeDiff(t *testing.T) {
t.Parallel()

cases := map[string]struct {
before *schema.Set
after *schema.Set
expectedForceNew bool
}{
"no need to force new from nil to empty apis": {
before: schema.NewSet(schema.HashString, nil),
after: schema.NewSet(schema.HashString, []interface{}{}),
expectedForceNew: false,
},
"no need to force new from empty apis to nil": {
before: schema.NewSet(schema.HashString, []interface{}{}),
after: schema.NewSet(schema.HashString, nil),
expectedForceNew: false,
},
"no need to force new from empty apis to empty apis": {
before: schema.NewSet(schema.HashString, []interface{}{}),
after: schema.NewSet(schema.HashString, []interface{}{}),
expectedForceNew: false,
},
"no need to force new from nil to empty string apis": {
before: schema.NewSet(schema.HashString, nil),
after: schema.NewSet(schema.HashString, []interface{}{""}),
expectedForceNew: false,
},
"no need to force new from empty string apis to empty string apis": {
before: schema.NewSet(schema.HashString, []interface{}{""}),
after: schema.NewSet(schema.HashString, []interface{}{""}),
expectedForceNew: false,
},
"no need to force new for enabling new api from empty apis": {
before: schema.NewSet(schema.HashString, []interface{}{}),
after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
expectedForceNew: false,
},
"no need to force new for enabling new api from nil": {
before: schema.NewSet(schema.HashString, nil),
after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
expectedForceNew: false,
},
"no need to force new for passing same apis": {
before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
expectedForceNew: false,
},
"no need to force new for passing same apis with inconsistent order": {
before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo", "dummy.k8s.io/v1beta1/bar"}),
after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/bar", "dummy.k8s.io/v1beta1/foo"}),
expectedForceNew: false,
},
"need to force new from empty string apis to nil": {
before: schema.NewSet(schema.HashString, []interface{}{""}),
after: schema.NewSet(schema.HashString, nil),
expectedForceNew: true,
},
"need to force new for disabling existing api": {
before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
after: schema.NewSet(schema.HashString, []interface{}{}),
expectedForceNew: true,
},
"need to force new for disabling existing api with nil": {
before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
after: schema.NewSet(schema.HashString, nil),
expectedForceNew: true,
},
"need to force new for disabling existing apis": {
before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo", "dummy.k8s.io/v1beta1/bar", "dummy.k8s.io/v1beta1/baz"}),
after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
expectedForceNew: true,
},
}

for tn, tc := range cases {
d := &tpgresource.ResourceDiffMock{
Before: map[string]interface{}{
"enable_k8s_beta_apis.0.enabled_apis": tc.before,
},
After: map[string]interface{}{
"enable_k8s_beta_apis.0.enabled_apis": tc.after,
},
}
err := containerClusterEnableK8sBetaApisCustomizeDiffFunc(d)
if err != nil {
t.Errorf("%s failed, found unexpected error: %s", tn, err)
}
if d.IsForceNew != tc.expectedForceNew {
t.Errorf("%v: expected d.IsForceNew to be %v, but was %v", tn, tc.expectedForceNew, d.IsForceNew)
}
}
}
Loading

0 comments on commit 27213b2

Please sign in to comment.