diff --git a/.changelog/8355.txt b/.changelog/8355.txt
new file mode 100644
index 00000000000..0877dd18ff1
--- /dev/null
+++ b/.changelog/8355.txt
@@ -0,0 +1,3 @@
+```release-note:enhancement
+container: added `enable_k8s_beta_apis.enabled_apis` field to `google_container_cluster`
+```
diff --git a/google/resource_container_cluster_test.go b/google/resource_container_cluster_test.go
index 67b8f6d432b..3f96794d739 100644
--- a/google/resource_container_cluster_test.go
+++ b/google/resource_container_cluster_test.go
@@ -2940,6 +2940,61 @@ func TestAccContainerCluster_withEnableKubernetesAlpha(t *testing.T) {
})
}
+func TestAccContainerCluster_withEnableKubernetesBetaAPIs(t *testing.T) {
+ t.Parallel()
+
+ clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
+
+ acctest.VcrTest(t, resource.TestCase{
+ PreCheck: func() { acctest.AccTestPreCheck(t) },
+ ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+ CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
+ Steps: []resource.TestStep{
+ {
+ Config: testAccContainerCluster_withEnableKubernetesBetaAPIs(clusterName),
+ },
+ {
+ ResourceName: "google_container_cluster.primary",
+ ImportState: true,
+ ImportStateVerify: true,
+ ImportStateVerifyIgnore: []string{"min_master_version"},
+ },
+ },
+ })
+}
+
+func TestAccContainerCluster_withEnableKubernetesBetaAPIsOnExistingCluster(t *testing.T) {
+ t.Parallel()
+
+ clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
+
+ acctest.VcrTest(t, resource.TestCase{
+ PreCheck: func() { acctest.AccTestPreCheck(t) },
+ ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+ CheckDestroy: testAccCheckContainerClusterDestroyProducer(t),
+ Steps: []resource.TestStep{
+ {
+ Config: testAccContainerCluster_withoutEnableKubernetesBetaAPIs(clusterName),
+ },
+ {
+ ResourceName: "google_container_cluster.primary",
+ ImportState: true,
+ ImportStateVerify: true,
+ ImportStateVerifyIgnore: []string{"min_master_version"},
+ },
+ {
+ Config: testAccContainerCluster_withEnableKubernetesBetaAPIs(clusterName),
+ },
+ {
+ ResourceName: "google_container_cluster.primary",
+ ImportState: true,
+ ImportStateVerify: true,
+ ImportStateVerifyIgnore: []string{"min_master_version"},
+ },
+ },
+ })
+}
+
func TestAccContainerCluster_withIPv4Error(t *testing.T) {
t.Parallel()
@@ -6215,6 +6270,60 @@ resource "google_container_cluster" "primary" {
`, cluster, np)
}
+func testAccContainerCluster_withoutEnableKubernetesBetaAPIs(clusterName string) string {
+ return fmt.Sprintf(`
+data "google_container_engine_versions" "central1a" {
+ location = "us-central1-a"
+}
+
+resource "google_container_cluster" "primary" {
+ name = "%s"
+ location = "us-central1-a"
+ min_master_version = data.google_container_engine_versions.central1a.release_channel_latest_version["STABLE"]
+ initial_node_count = 1
+}
+`, clusterName)
+}
+
+func testAccContainerCluster_withEnableKubernetesBetaAPIs(cluster string) string {
+ return fmt.Sprintf(`
+data "google_container_engine_versions" "uscentral1a" {
+ location = "us-central1-a"
+}
+
+resource "google_container_cluster" "primary" {
+ name = "%s"
+ location = "us-central1-a"
+ min_master_version = data.google_container_engine_versions.uscentral1a.release_channel_latest_version["STABLE"]
+ initial_node_count = 1
+
+ # This feature has been available since GKE 1.27, and currently the only
+ # supported Beta API is authentication.k8s.io/v1beta1/selfsubjectreviews.
+ # However, in the future, more Beta APIs will be supported, such as the
+ # resource.k8s.io group. At the same time, some existing Beta APIs will be
+ # deprecated as the feature will be GAed, and the Beta API will be eventually
+ # removed. In the case of the SelfSubjectReview API, it is planned to be GAed
+ # in Kubernetes as of 1.28. And, the Beta API of SelfSubjectReview will be removed
+ # after at least 3 minor version bumps, so it will be removed as of Kubernetes 1.31
+ # or later.
+ # https://pr.k8s.io/117713
+ # https://kubernetes.io/docs/reference/using-api/deprecation-guide/
+ #
+ # The new Beta APIs will be available since GKE 1.28
+ # - admissionregistration.k8s.io/v1beta1/validatingadmissionpolicies
+ # - admissionregistration.k8s.io/v1beta1/validatingadmissionpolicybindings
+ # https://pr.k8s.io/118644
+ #
+ # Removing the Beta API from Kubernetes will break the test.
+ # TODO: Replace the Beta API with one available on the version of GKE
+ # if the test is broken.
+ enable_k8s_beta_apis {
+ enabled_apis = ["authentication.k8s.io/v1beta1/selfsubjectreviews"]
+ }
+}
+`, cluster)
+}
+
func testAccContainerCluster_withIPv4Error(name string) string {
return fmt.Sprintf(`
resource "google_container_cluster" "primary" {
diff --git a/google/services/container/resource_container_cluster.go b/google/services/container/resource_container_cluster.go
index a878ea25ead..50fcfed0a28 100644
--- a/google/services/container/resource_container_cluster.go
+++ b/google/services/container/resource_container_cluster.go
@@ -182,6 +182,7 @@ func ResourceContainerCluster() *schema.Resource {
containerClusterNodeVersionRemoveDefaultCustomizeDiff,
containerClusterNetworkPolicyEmptyCustomizeDiff,
containerClusterSurgeSettingsCustomizeDiff,
+ containerClusterEnableK8sBetaApisCustomizeDiff,
),
Timeouts: &schema.ResourceTimeout{
@@ -756,6 +757,23 @@ func ResourceContainerCluster() *schema.Resource {
Description: `Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days.`,
},
+ "enable_k8s_beta_apis": {
+ Type: schema.TypeList,
+ Optional: true,
+ MaxItems: 1,
+ Description: `Configuration for Kubernetes Beta APIs.`,
+ Elem: &schema.Resource{
+ Schema: map[string]*schema.Schema{
+ "enabled_apis": {
+ Type: schema.TypeSet,
+ Required: true,
+ Elem: &schema.Schema{Type: schema.TypeString},
+ Description: `Enabled Kubernetes Beta APIs.`,
+ },
+ },
+ },
+ },
+
"enable_tpu": {
Type: schema.TypeBool,
Optional: true,
@@ -1855,6 +1873,7 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er
ConfidentialNodes: expandConfidentialNodes(d.Get("confidential_nodes")),
ResourceLabels: tpgresource.ExpandStringMap(d, "resource_labels"),
CostManagementConfig: expandCostManagementConfig(d.Get("cost_management_config")),
+ EnableK8sBetaApis: expandEnableK8sBetaApis(d.Get("enable_k8s_beta_apis"), nil),
}
v := d.Get("enable_shielded_nodes")
@@ -2354,6 +2373,9 @@ func resourceContainerClusterRead(d *schema.ResourceData, meta interface{}) erro
if err := d.Set("gateway_api_config", flattenGatewayApiConfig(cluster.NetworkConfig.GatewayApiConfig)); err != nil {
return err
}
+ if err := d.Set("enable_k8s_beta_apis", flattenEnableK8sBetaApis(cluster.EnableK8sBetaApis)); err != nil {
+ return err
+ }
if err := d.Set("logging_config", flattenContainerClusterLoggingConfig(cluster.LoggingConfig)); err != nil {
return err
}
@@ -3346,6 +3368,44 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er
}
}
+ if d.HasChange("enable_k8s_beta_apis") {
+ log.Print("[INFO] Enable Kubernetes Beta APIs")
+ if v, ok := d.GetOk("enable_k8s_beta_apis"); ok {
+ name := containerClusterFullName(project, location, clusterName)
+ clusterGetCall := config.NewContainerClient(userAgent).Projects.Locations.Clusters.Get(name)
+ if config.UserProjectOverride {
+ clusterGetCall.Header().Add("X-Goog-User-Project", project)
+ }
+ // Fetch the cluster information to get the already enabled Beta APIs.
+ cluster, err := clusterGetCall.Do()
+ if err != nil {
+ return err
+ }
+
+ // To avoid an already enabled Beta APIs error, we need to deduplicate the requested APIs
+ // with those that are already enabled.
+ var enabledAPIs []string
+ if cluster.EnableK8sBetaApis != nil && len(cluster.EnableK8sBetaApis.EnabledApis) > 0 {
+ enabledAPIs = cluster.EnableK8sBetaApis.EnabledApis
+ }
+ enableK8sBetaAPIs := expandEnableK8sBetaApis(v, enabledAPIs)
+
+ req := &container.UpdateClusterRequest{
+ Update: &container.ClusterUpdate{
+ DesiredK8sBetaApis: enableK8sBetaAPIs,
+ },
+ }
+
+ updateF := updateFunc(req, "updating enabled Kubernetes Beta APIs")
+ // Call update serially.
+ if err := transport_tpg.LockedCall(lockKey, updateF); err != nil {
+ return err
+ }
+
+ log.Printf("[INFO] GKE cluster %s enabled Kubernetes Beta APIs has been updated", d.Id())
+ }
+ }
+
if d.HasChange("node_pool_defaults") && d.HasChange("node_pool_defaults.0.node_config_defaults.0.logging_variant") {
if v, ok := d.GetOk("node_pool_defaults.0.node_config_defaults.0.logging_variant"); ok {
loggingVariant := v.(string)
@@ -4318,6 +4378,28 @@ func expandGatewayApiConfig(configured interface{}) *container.GatewayAPIConfig
}
}
+func expandEnableK8sBetaApis(configured interface{}, enabledAPIs []string) *container.K8sBetaAPIConfig {
+ l := configured.([]interface{})
+ if len(l) == 0 || l[0] == nil {
+ return nil
+ }
+
+ config := l[0].(map[string]interface{})
+ result := &container.K8sBetaAPIConfig{}
+ if v, ok := config["enabled_apis"]; ok {
+ notEnabledAPIsSet := v.(*schema.Set)
+ for _, enabledAPI := range enabledAPIs {
+ if notEnabledAPIsSet.Contains(enabledAPI) {
+ notEnabledAPIsSet.Remove(enabledAPI)
+ }
+ }
+
+ result.EnabledApis = tpgresource.ConvertStringSet(notEnabledAPIsSet)
+ }
+
+ return result
+}
+
func expandContainerClusterLoggingConfig(configured interface{}) *container.LoggingConfig {
l := configured.([]interface{})
if len(l) == 0 {
@@ -5012,6 +5094,17 @@ func flattenGatewayApiConfig(c *container.GatewayAPIConfig) []map[string]interfa
}
}
+func flattenEnableK8sBetaApis(c *container.K8sBetaAPIConfig) []map[string]interface{} {
+ if c == nil {
+ return nil
+ }
+ return []map[string]interface{}{
+ {
+ "enabled_apis": c.EnabledApis,
+ },
+ }
+}
+
func flattenContainerClusterLoggingConfig(c *container.LoggingConfig) []map[string]interface{} {
if c == nil {
return nil
@@ -5263,3 +5356,31 @@ func containerClusterSurgeSettingsCustomizeDiff(_ context.Context, d *schema.Res
return nil
}
+
+func containerClusterEnableK8sBetaApisCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta interface{}) error {
+ // separate func to allow unit testing
+ return containerClusterEnableK8sBetaApisCustomizeDiffFunc(d)
+}
+
+func containerClusterEnableK8sBetaApisCustomizeDiffFunc(d tpgresource.TerraformResourceDiff) error {
+ // The Kubernetes Beta APIs cannot be disabled once they have been enabled by users.
+ // The reason why we don't allow disabling is that the controller does not have the
+ // ability to clean up the Kubernetes objects created by the APIs. If the user
+ // removes the already enabled Kubernetes Beta API from the list, we need to force
+ // a new cluster.
+ if !d.HasChange("enable_k8s_beta_apis.0.enabled_apis") {
+ return nil
+ }
+ old, new := d.GetChange("enable_k8s_beta_apis.0.enabled_apis")
+ if old != "" && new != "" {
+ oldAPIsSet := old.(*schema.Set)
+ newAPIsSet := new.(*schema.Set)
+ for _, oldAPI := range oldAPIsSet.List() {
+ if !newAPIsSet.Contains(oldAPI) {
+ return d.ForceNew("enable_k8s_beta_apis.0.enabled_apis")
+ }
+ }
+ }
+
+ return nil
+}
diff --git a/google/services/container/resource_container_cluster_internal_test.go b/google/services/container/resource_container_cluster_internal_test.go
index 8a866d39da9..89277e4de49 100644
--- a/google/services/container/resource_container_cluster_internal_test.go
+++ b/google/services/container/resource_container_cluster_internal_test.go
@@ -1,3 +1,104 @@
// Copyright (c) HashiCorp, Inc.
// SPDX-License-Identifier: MPL-2.0
package container
+
+import (
+ "testing"
+
+ "github.com/hashicorp/terraform-plugin-sdk/v2/helper/schema"
+ "github.com/hashicorp/terraform-provider-google/google/tpgresource"
+)
+
+func TestContainerClusterEnableK8sBetaApisCustomizeDiff(t *testing.T) {
+ t.Parallel()
+
+ cases := map[string]struct {
+ before *schema.Set
+ after *schema.Set
+ expectedForceNew bool
+ }{
+ "no need to force new from nil to empty apis": {
+ before: schema.NewSet(schema.HashString, nil),
+ after: schema.NewSet(schema.HashString, []interface{}{}),
+ expectedForceNew: false,
+ },
+ "no need to force new from empty apis to nil": {
+ before: schema.NewSet(schema.HashString, []interface{}{}),
+ after: schema.NewSet(schema.HashString, nil),
+ expectedForceNew: false,
+ },
+ "no need to force new from empty apis to empty apis": {
+ before: schema.NewSet(schema.HashString, []interface{}{}),
+ after: schema.NewSet(schema.HashString, []interface{}{}),
+ expectedForceNew: false,
+ },
+ "no need to force new from nil to empty string apis": {
+ before: schema.NewSet(schema.HashString, nil),
+ after: schema.NewSet(schema.HashString, []interface{}{""}),
+ expectedForceNew: false,
+ },
+ "no need to force new from empty string apis to empty string apis": {
+ before: schema.NewSet(schema.HashString, []interface{}{""}),
+ after: schema.NewSet(schema.HashString, []interface{}{""}),
+ expectedForceNew: false,
+ },
+ "no need to force new for enabling new api from empty apis": {
+ before: schema.NewSet(schema.HashString, []interface{}{}),
+ after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
+ expectedForceNew: false,
+ },
+ "no need to force new for enabling new api from nil": {
+ before: schema.NewSet(schema.HashString, nil),
+ after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
+ expectedForceNew: false,
+ },
+ "no need to force new for passing same apis": {
+ before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
+ after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
+ expectedForceNew: false,
+ },
+ "no need to force new for passing same apis with inconsistent order": {
+ before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo", "dummy.k8s.io/v1beta1/bar"}),
+ after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/bar", "dummy.k8s.io/v1beta1/foo"}),
+ expectedForceNew: false,
+ },
+ "need to force new from empty string apis to nil": {
+ before: schema.NewSet(schema.HashString, []interface{}{""}),
+ after: schema.NewSet(schema.HashString, nil),
+ expectedForceNew: true,
+ },
+ "need to force new for disabling existing api": {
+ before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
+ after: schema.NewSet(schema.HashString, []interface{}{}),
+ expectedForceNew: true,
+ },
+ "need to force new for disabling existing api with nil": {
+ before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
+ after: schema.NewSet(schema.HashString, nil),
+ expectedForceNew: true,
+ },
+ "need to force new for disabling existing apis": {
+ before: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo", "dummy.k8s.io/v1beta1/bar", "dummy.k8s.io/v1beta1/baz"}),
+ after: schema.NewSet(schema.HashString, []interface{}{"dummy.k8s.io/v1beta1/foo"}),
+ expectedForceNew: true,
+ },
+ }
+
+ for tn, tc := range cases {
+ d := &tpgresource.ResourceDiffMock{
+ Before: map[string]interface{}{
+ "enable_k8s_beta_apis.0.enabled_apis": tc.before,
+ },
+ After: map[string]interface{}{
+ "enable_k8s_beta_apis.0.enabled_apis": tc.after,
+ },
+ }
+ err := containerClusterEnableK8sBetaApisCustomizeDiffFunc(d)
+ if err != nil {
+ t.Errorf("%s failed, found unexpected error: %s", tn, err)
+ }
+ if d.IsForceNew != tc.expectedForceNew {
+ t.Errorf("%v: expected d.IsForceNew to be %v, but was %v", tn, tc.expectedForceNew, d.IsForceNew)
+ }
+ }
+}
diff --git a/website/docs/r/container_cluster.html.markdown b/website/docs/r/container_cluster.html.markdown
index 7532590da2b..871f8adc98d 100644
--- a/website/docs/r/container_cluster.html.markdown
+++ b/website/docs/r/container_cluster.html.markdown
@@ -164,6 +164,9 @@ for more information.
this cluster. Note that when this option is enabled, the cluster cannot be upgraded
and will be automatically deleted after 30 days.
+* `enable_k8s_beta_apis` - (Optional) Configuration for Kubernetes Beta APIs.
+ Structure is [documented below](#nested_enable_k8s_beta_apis).
+
* `enable_tpu` - (Optional) Whether to enable Cloud TPU resources in this cluster.
See the [official documentation](https://cloud.google.com/tpu/docs/kubernetes-engine-setup).
@@ -467,6 +470,10 @@ addons_config {
* `key_name` - (Required) the key to use to encrypt/decrypt secrets. See the [DatabaseEncryption definition](https://cloud.google.com/kubernetes-engine/docs/reference/rest/v1beta1/projects.locations.clusters#Cluster.DatabaseEncryption) for more information.
+The `enable_k8s_beta_apis` block supports:
+
+* `enabled_apis` - (Required) Enabled Kubernetes Beta APIs. To list a Beta API resource, use the representation {group}/{version}/{resource}. The version must be a Beta version. Note that you cannot disable beta APIs that are already enabled on a cluster without recreating it. See the [Configure beta APIs](https://cloud.google.com/kubernetes-engine/docs/how-to/use-beta-apis#configure-beta-apis) for more information.
+
The `cloudrun_config` block supports:
* `disabled` - (Optional) The status of the CloudRun addon. It is disabled by default. Set `disabled=false` to enable.