diff --git a/.changelog/8355.txt b/.changelog/8355.txt new file mode 100644 index 00000000000..0877dd18ff1 --- /dev/null +++ b/.changelog/8355.txt @@ -0,0 +1,3 @@ +```release-note:enhancement +container: added `enable_k8s_beta_apis.enabled_apis` field to `google_container_cluster` +``` diff --git a/google/resource_container_cluster_test.go b/google/resource_container_cluster_test.go index 67b8f6d432b..3f96794d739 100644 --- a/google/resource_container_cluster_test.go +++ b/google/resource_container_cluster_test.go @@ -2940,6 +2940,61 @@ func TestAccContainerCluster_withEnableKubernetesAlpha(t *testing.T) { }) } +func TestAccContainerCluster_withEnableKubernetesBetaAPIs(t *testing.T) { + t.Parallel() + + clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10)) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerCluster_withEnableKubernetesBetaAPIs(clusterName), + }, + { + ResourceName: "google_container_cluster.primary", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + }, + }) +} + +func TestAccContainerCluster_withEnableKubernetesBetaAPIsOnExistingCluster(t *testing.T) { + t.Parallel() + + clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10)) + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + CheckDestroy: testAccCheckContainerClusterDestroyProducer(t), + Steps: []resource.TestStep{ + { + Config: testAccContainerCluster_withoutEnableKubernetesBetaAPIs(clusterName), + }, + { + ResourceName: "google_container_cluster.primary", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + { + Config: testAccContainerCluster_withEnableKubernetesBetaAPIs(clusterName), + }, + { + ResourceName: "google_container_cluster.primary", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"min_master_version"}, + }, + }, + }) +} + func TestAccContainerCluster_withIPv4Error(t *testing.T) { t.Parallel() @@ -6215,6 +6270,60 @@ resource "google_container_cluster" "primary" { `, cluster, np) } +func testAccContainerCluster_withoutEnableKubernetesBetaAPIs(clusterName string) string { + return fmt.Sprintf(` +data "google_container_engine_versions" "central1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "primary" { + name = "%s" + location = "us-central1-a" + min_master_version = data.google_container_engine_versions.central1a.release_channel_latest_version["STABLE"] + initial_node_count = 1 +} +`, clusterName) +} + +func testAccContainerCluster_withEnableKubernetesBetaAPIs(cluster string) string { + return fmt.Sprintf(` +data "google_container_engine_versions" "uscentral1a" { + location = "us-central1-a" +} + +resource "google_container_cluster" "primary" { + name = "%s" + location = "us-central1-a" + min_master_version = data.google_container_engine_versions.uscentral1a.release_channel_latest_version["STABLE"] + initial_node_count = 1 + + # This feature has been available since GKE 1.27, and currently the only + # supported Beta API is authentication.k8s.io/v1beta1/selfsubjectreviews. + # However, in the future, more Beta APIs will be supported, such as the + # resource.k8s.io group. At the same time, some existing Beta APIs will be + # deprecated as the feature will be GAed, and the Beta API will be eventually + # removed. In the case of the SelfSubjectReview API, it is planned to be GAed + # in Kubernetes as of 1.28. And, the Beta API of SelfSubjectReview will be removed + # after at least 3 minor version bumps, so it will be removed as of Kubernetes 1.31 + # or later. + # https://pr.k8s.io/117713 + # https://kubernetes.io/docs/reference/using-api/deprecation-guide/ + # + # The new Beta APIs will be available since GKE 1.28 + # - admissionregistration.k8s.io/v1beta1/validatingadmissionpolicies + # - admissionregistration.k8s.io/v1beta1/validatingadmissionpolicybindings + # https://pr.k8s.io/118644 + # + # Removing the Beta API from Kubernetes will break the test. + # TODO: Replace the Beta API with one available on the version of GKE + # if the test is broken. + enable_k8s_beta_apis { + enabled_apis = ["authentication.k8s.io/v1beta1/selfsubjectreviews"] + } +} +`, cluster) +} + func testAccContainerCluster_withIPv4Error(name string) string { return fmt.Sprintf(` resource "google_container_cluster" "primary" { diff --git a/google/services/container/resource_container_cluster.go b/google/services/container/resource_container_cluster.go index a878ea25ead..50fcfed0a28 100644 --- a/google/services/container/resource_container_cluster.go +++ b/google/services/container/resource_container_cluster.go @@ -182,6 +182,7 @@ func ResourceContainerCluster() *schema.Resource { containerClusterNodeVersionRemoveDefaultCustomizeDiff, containerClusterNetworkPolicyEmptyCustomizeDiff, containerClusterSurgeSettingsCustomizeDiff, + containerClusterEnableK8sBetaApisCustomizeDiff, ), Timeouts: &schema.ResourceTimeout{ @@ -756,6 +757,23 @@ func ResourceContainerCluster() *schema.Resource { Description: `Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days.`, }, + "enable_k8s_beta_apis": { + Type: schema.TypeList, + Optional: true, + MaxItems: 1, + Description: `Configuration for Kubernetes Beta APIs.`, + Elem: &schema.Resource{ + Schema: map[string]*schema.Schema{ + "enabled_apis": { + Type: schema.TypeSet, + Required: true, + Elem: &schema.Schema{Type: schema.TypeString}, + Description: `Enabled Kubernetes Beta APIs.`, + }, + }, + }, + }, + "enable_tpu": { Type: schema.TypeBool, Optional: true, @@ -1855,6 +1873,7 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er ConfidentialNodes: expandConfidentialNodes(d.Get("confidential_nodes")), ResourceLabels: tpgresource.ExpandStringMap(d, "resource_labels"), CostManagementConfig: expandCostManagementConfig(d.Get("cost_management_config")), + EnableK8sBetaApis: expandEnableK8sBetaApis(d.Get("enable_k8s_beta_apis"), nil), } v := d.Get("enable_shielded_nodes") @@ -2354,6 +2373,9 @@ func resourceContainerClusterRead(d *schema.ResourceData, meta interface{}) erro if err := d.Set("gateway_api_config", flattenGatewayApiConfig(cluster.NetworkConfig.GatewayApiConfig)); err != nil { return err } + if err := d.Set("enable_k8s_beta_apis", flattenEnableK8sBetaApis(cluster.EnableK8sBetaApis)); err != nil { + return err + } if err := d.Set("logging_config", flattenContainerClusterLoggingConfig(cluster.LoggingConfig)); err != nil { return err } @@ -3346,6 +3368,44 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er } } + if d.HasChange("enable_k8s_beta_apis") { + log.Print("[INFO] Enable Kubernetes Beta APIs") + if v, ok := d.GetOk("enable_k8s_beta_apis"); ok { + name := containerClusterFullName(project, location, clusterName) + clusterGetCall := config.NewContainerClient(userAgent).Projects.Locations.Clusters.Get(name) + if config.UserProjectOverride { + clusterGetCall.Header().Add("X-Goog-User-Project", project) + } + // Fetch the cluster information to get the already enabled Beta APIs. + cluster, err := clusterGetCall.Do() + if err != nil { + return err + } + + // To avoid an already enabled Beta APIs error, we need to deduplicate the requested APIs + // with those that are already enabled. + var enabledAPIs []string + if cluster.EnableK8sBetaApis != nil && len(cluster.EnableK8sBetaApis.EnabledApis) > 0 { + enabledAPIs = cluster.EnableK8sBetaApis.EnabledApis + } + enableK8sBetaAPIs := expandEnableK8sBetaApis(v, enabledAPIs) + + req := &container.UpdateClusterRequest{ + Update: &container.ClusterUpdate{ + DesiredK8sBetaApis: enableK8sBetaAPIs, + }, + } + + updateF := updateFunc(req, "updating enabled Kubernetes Beta APIs") + // Call update serially. + if err := transport_tpg.LockedCall(lockKey, updateF); err != nil { + return err + } + + log.Printf("[INFO] GKE cluster %s enabled Kubernetes Beta APIs has been updated", d.Id()) + } + } + if d.HasChange("node_pool_defaults") && d.HasChange("node_pool_defaults.0.node_config_defaults.0.logging_variant") { if v, ok := d.GetOk("node_pool_defaults.0.node_config_defaults.0.logging_variant"); ok { loggingVariant := v.(string) @@ -4318,6 +4378,28 @@ func expandGatewayApiConfig(configured interface{}) *container.GatewayAPIConfig } } +func expandEnableK8sBetaApis(configured interface{}, enabledAPIs []string) *container.K8sBetaAPIConfig { + l := configured.([]interface{}) + if len(l) == 0 || l[0] == nil { + return nil + } + + config := l[0].(map[string]interface{}) + result := &container.K8sBetaAPIConfig{} + if v, ok := config["enabled_apis"]; ok { + notEnabledAPIsSet := v.(*schema.Set) + for _, enabledAPI := range enabledAPIs { + if notEnabledAPIsSet.Contains(enabledAPI) { + notEnabledAPIsSet.Remove(enabledAPI) + } + } + + result.EnabledApis = tpgresource.ConvertStringSet(notEnabledAPIsSet) + } + + return result +} + func expandContainerClusterLoggingConfig(configured interface{}) *container.LoggingConfig { l := configured.([]interface{}) if len(l) == 0 { @@ -5012,6 +5094,17 @@ func flattenGatewayApiConfig(c *container.GatewayAPIConfig) []map[string]interfa } } +func flattenEnableK8sBetaApis(c *container.K8sBetaAPIConfig) []map[string]interface{} { + if c == nil { + return nil + } + return []map[string]interface{}{ + { + "enabled_apis": c.EnabledApis, + }, + } +} + func flattenContainerClusterLoggingConfig(c *container.LoggingConfig) []map[string]interface{} { if c == nil { return nil @@ -5263,3 +5356,31 @@ func containerClusterSurgeSettingsCustomizeDiff(_ context.Context, d *schema.Res return nil } + +func containerClusterEnableK8sBetaApisCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta interface{}) error { + // separate func to allow unit testing + return containerClusterEnableK8sBetaApisCustomizeDiffFunc(d) +} + +func containerClusterEnableK8sBetaApisCustomizeDiffFunc(d tpgresource.TerraformResourceDiff) error { + // The Kubernetes Beta APIs cannot be disabled once they have been enabled by users. + // The reason why we don't allow disabling is that the controller does not have the + // ability to clean up the Kubernetes objects created by the APIs. If the user + // removes the already enabled Kubernetes Beta API from the list, we need to force + // a new cluster. + if !d.HasChange("enable_k8s_beta_apis.0.enabled_apis") { + return nil + } + old, new := d.GetChange("enable_k8s_beta_apis.0.enabled_apis") + if old != "" && new != "" { + oldAPIsSet := old.(*schema.Set) + newAPIsSet := new.(*schema.Set) + for _, oldAPI := range oldAPIsSet.List() { + if !newAPIsSet.Contains(oldAPI) { + return d.ForceNew("enable_k8s_beta_apis.0.enabled_apis") + } + } + } + + return nil +} diff --git a/google/services/container/resource_container_cluster_internal_test.go b/google/services/container/resource_container_cluster_internal_test.go index 8a866d39da9..89277e4de49 100644 --- a/google/services/container/resource_container_cluster_internal_test.go +++ b/google/services/container/resource_container_cluster_internal_test.go @@ -1,3 +1,104 @@ // Copyright (c) HashiCorp, Inc. // SPDX-License-Identifier: MPL-2.0 package container + +import ( + "testing" + + "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema" + "github.com/hashicorp/terraform-provider-google/google/tpgresource" +) + +func TestContainerClusterEnableK8sBetaApisCustomizeDiff(t *testing.T) { + t.Parallel() + + cases := map[string]struct { + before *schema.Set + after *schema.Set + expectedForceNew bool + }{ + "no need to force new from nil to empty apis": { + before: schema.NewSet(schema.HashString, nil), + after: schema.NewSet(schema.HashString, []interface{}{}), + expectedForceNew: false, + }, + "no need to force new from empty apis to nil": { + before: schema.NewSet(schema.HashString, []interface{}{}), + after: schema.NewSet(schema.HashString, nil), + expectedForceNew: false, + }, + "no need to force new from empty apis to empty apis": { + before: schema.NewSet(schema.HashString, []interface{}{}), + after: schema.NewSet(schema.HashString, []interface{}{}), + expectedForceNew: false, + }, + "no need to force new from nil to empty string apis": { + before: schema.NewSet(schema.HashString, nil), + after: schema.NewSet(schema.HashString, []interface{}{""}), + expectedForceNew: false, + }, + "no need to force new from empty string apis to empty string apis": { + before: schema.NewSet(schema.HashString, []interface{}{""}), + after: schema.NewSet(schema.HashString, []interface{}{""}), + expectedForceNew: false, + }, + "no need to force new for enabling new api from empty apis": { + before: schema.NewSet(schema.HashString, []interface{}{}), + after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}), + expectedForceNew: false, + }, + "no need to force new for enabling new api from nil": { + before: schema.NewSet(schema.HashString, nil), + after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}), + expectedForceNew: false, + }, + "no need to force new for passing same apis": { + before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}), + after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}), + expectedForceNew: false, + }, + "no need to force new for passing same apis with inconsistent order": { + before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo", "dummy.k8s.io/v1beta1/bar"}), + after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/bar", "dummy.k8s.io/v1beta1/foo"}), + expectedForceNew: false, + }, + "need to force new from empty string apis to nil": { + before: schema.NewSet(schema.HashString, []interface{}{""}), + after: schema.NewSet(schema.HashString, nil), + expectedForceNew: true, + }, + "need to force new for disabling existing api": { + before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}), + after: schema.NewSet(schema.HashString, []interface{}{}), + expectedForceNew: true, + }, + "need to force new for disabling existing api with nil": { + before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}), + after: schema.NewSet(schema.HashString, nil), + expectedForceNew: true, + }, + "need to force new for disabling existing apis": { + before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo", "dummy.k8s.io/v1beta1/bar", "dummy.k8s.io/v1beta1/baz"}), + after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}), + expectedForceNew: true, + }, + } + + for tn, tc := range cases { + d := &tpgresource.ResourceDiffMock{ + Before: map[string]interface{}{ + "enable_k8s_beta_apis.0.enabled_apis": tc.before, + }, + After: map[string]interface{}{ + "enable_k8s_beta_apis.0.enabled_apis": tc.after, + }, + } + err := containerClusterEnableK8sBetaApisCustomizeDiffFunc(d) + if err != nil { + t.Errorf("%s failed, found unexpected error: %s", tn, err) + } + if d.IsForceNew != tc.expectedForceNew { + t.Errorf("%v: expected d.IsForceNew to be %v, but was %v", tn, tc.expectedForceNew, d.IsForceNew) + } + } +} diff --git a/website/docs/r/container_cluster.html.markdown b/website/docs/r/container_cluster.html.markdown index 7532590da2b..871f8adc98d 100644 --- a/website/docs/r/container_cluster.html.markdown +++ b/website/docs/r/container_cluster.html.markdown @@ -164,6 +164,9 @@ for more information. this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days. +* `enable_k8s_beta_apis` - (Optional) Configuration for Kubernetes Beta APIs. + Structure is [documented below](#nested_enable_k8s_beta_apis). + * `enable_tpu` - (Optional) Whether to enable Cloud TPU resources in this cluster. See the [official documentation](https://cloud.google.com/tpu/docs/kubernetes-engine-setup). @@ -467,6 +470,10 @@ addons_config { * `key_name` - (Required) the key to use to encrypt/decrypt secrets. See the [DatabaseEncryption definition](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#Cluster.DatabaseEncryption) for more information. +The `enable_k8s_beta_apis` block supports: + +* `enabled_apis` - (Required) Enabled Kubernetes Beta APIs. To list a Beta API resource, use the representation {group}/{version}/{resource}. The version must be a Beta version. Note that you cannot disable beta APIs that are already enabled on a cluster without recreating it. See the [Configure beta APIs](https://cloud.google.com/kubernetes-engine/docs/how-to/use-beta-apis#configure-beta-apis) for more information. + The `cloudrun_config` block supports: * `disabled` - (Optional) The status of the CloudRun addon. It is disabled by default. Set `disabled=false` to enable.