feat(google_container_cluster): support enable k8s beta apis

.changelog/8355.txt

@@ -0,0 +1,3 @@
+```release-note:enhancement
+container: added `enable_k8s_beta_apis.enabled_apis` field to `google_container_cluster`
+```

google/resource_container_cluster_test.go

@@ -2940,6 +2940,61 @@ func TestAccContainerCluster_withEnableKubernetesAlpha(t *testing.T) {
 	})
 }
 
+func TestAccContainerCluster_withEnableKubernetesBetaAPIs(t *testing.T) {
+	t.Parallel()
+
+	clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckContainerClusterDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccContainerCluster_withEnableKubernetesBetaAPIs(clusterName),
+			},
+			{
+				ResourceName:            "google_container_cluster.primary",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"min_master_version"},
+			},
+		},
+	})
+}
+
+func TestAccContainerCluster_withEnableKubernetesBetaAPIsOnExistingCluster(t *testing.T) {
+	t.Parallel()
+
+	clusterName := fmt.Sprintf("tf-test-cluster-%s", acctest.RandString(t, 10))
+
+	acctest.VcrTest(t, resource.TestCase{
+		PreCheck:                 func() { acctest.AccTestPreCheck(t) },
+		ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t),
+		CheckDestroy:             testAccCheckContainerClusterDestroyProducer(t),
+		Steps: []resource.TestStep{
+			{
+				Config: testAccContainerCluster_withoutEnableKubernetesBetaAPIs(clusterName),
+			},
+			{
+				ResourceName:            "google_container_cluster.primary",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"min_master_version"},
+			},
+			{
+				Config: testAccContainerCluster_withEnableKubernetesBetaAPIs(clusterName),
+			},
+			{
+				ResourceName:            "google_container_cluster.primary",
+				ImportState:             true,
+				ImportStateVerify:       true,
+				ImportStateVerifyIgnore: []string{"min_master_version"},
+			},
+		},
+	})
+}
+
 func TestAccContainerCluster_withIPv4Error(t *testing.T) {
 	t.Parallel()
 
@@ -6215,6 +6270,60 @@ resource "google_container_cluster" "primary" {
 `, cluster, np)
 }
 
+func testAccContainerCluster_withoutEnableKubernetesBetaAPIs(clusterName string) string {
+	return fmt.Sprintf(`
+data "google_container_engine_versions" "central1a" {
+  location = "us-central1-a"
+}
+
+resource "google_container_cluster" "primary" {
+  name               = "%s"
+  location           = "us-central1-a"
+  min_master_version = data.google_container_engine_versions.central1a.release_channel_latest_version["STABLE"]
+  initial_node_count = 1
+}
+`, clusterName)
+}
+
+func testAccContainerCluster_withEnableKubernetesBetaAPIs(cluster string) string {
+	return fmt.Sprintf(`
+data "google_container_engine_versions" "uscentral1a" {
+  location = "us-central1-a"
+}
+
+resource "google_container_cluster" "primary" {
+  name               = "%s"
+  location           = "us-central1-a"
+  min_master_version = data.google_container_engine_versions.uscentral1a.release_channel_latest_version["STABLE"]
+  initial_node_count = 1
+
+  # This feature has been available since GKE 1.27, and currently the only
+  # supported Beta API is authentication.k8s.io/v1beta1/selfsubjectreviews.
+  # However, in the future, more Beta APIs will be supported, such as the
+  # resource.k8s.io group. At the same time, some existing Beta APIs will be
+  # deprecated as the feature will be GAed, and the Beta API will be eventually
+  # removed. In the case of the SelfSubjectReview API, it is planned to be GAed
+  # in Kubernetes as of 1.28. And, the Beta API of SelfSubjectReview will be removed
+  # after at least 3 minor version bumps, so it will be removed as of Kubernetes 1.31
+  # or later.
+  # https://pr.k8s.io/117713
+  # https://kubernetes.io/docs/reference/using-api/deprecation-guide/
+  #
+  # The new Beta APIs will be available since GKE 1.28
+  # - admissionregistration.k8s.io/v1beta1/validatingadmissionpolicies
+  # - admissionregistration.k8s.io/v1beta1/validatingadmissionpolicybindings
+  # https://pr.k8s.io/118644
+  #
+  # Removing the Beta API from Kubernetes will break the test.
+  # TODO: Replace the Beta API with one available on the version of GKE
+  # if the test is broken.
+  enable_k8s_beta_apis {
+    enabled_apis = ["authentication.k8s.io/v1beta1/selfsubjectreviews"]
+  }
+}
+`, cluster)
+}
+
 func testAccContainerCluster_withIPv4Error(name string) string {
 	return fmt.Sprintf(`
 resource "google_container_cluster" "primary" {

google/services/container/resource_container_cluster.go

@@ -182,6 +182,7 @@ func ResourceContainerCluster() *schema.Resource {
 			containerClusterNodeVersionRemoveDefaultCustomizeDiff,
 			containerClusterNetworkPolicyEmptyCustomizeDiff,
 			containerClusterSurgeSettingsCustomizeDiff,
+			containerClusterEnableK8sBetaApisCustomizeDiff,
 		),
 
 		Timeouts: &schema.ResourceTimeout{
@@ -756,6 +757,23 @@ func ResourceContainerCluster() *schema.Resource {
 				Description: `Whether to enable Kubernetes Alpha features for this cluster. Note that when this option is enabled, the cluster cannot be upgraded and will be automatically deleted after 30 days.`,
 			},
 
+			"enable_k8s_beta_apis": {
+				Type:        schema.TypeList,
+				Optional:    true,
+				MaxItems:    1,
+				Description: `Configuration for Kubernetes Beta APIs.`,
+				Elem: &schema.Resource{
+					Schema: map[string]*schema.Schema{
+						"enabled_apis": {
+							Type:        schema.TypeSet,
+							Required:    true,
+							Elem:        &schema.Schema{Type: schema.TypeString},
+							Description: `Enabled Kubernetes Beta APIs.`,
+						},
+					},
+				},
+			},
+
 			"enable_tpu": {
 				Type:        schema.TypeBool,
 				Optional:    true,
@@ -1855,6 +1873,7 @@ func resourceContainerClusterCreate(d *schema.ResourceData, meta interface{}) er
 		ConfidentialNodes:    expandConfidentialNodes(d.Get("confidential_nodes")),
 		ResourceLabels:       tpgresource.ExpandStringMap(d, "resource_labels"),
 		CostManagementConfig: expandCostManagementConfig(d.Get("cost_management_config")),
+		EnableK8sBetaApis:    expandEnableK8sBetaApis(d.Get("enable_k8s_beta_apis"), nil),
 	}
 
 	v := d.Get("enable_shielded_nodes")
@@ -2354,6 +2373,9 @@ func resourceContainerClusterRead(d *schema.ResourceData, meta interface{}) erro
 	if err := d.Set("gateway_api_config", flattenGatewayApiConfig(cluster.NetworkConfig.GatewayApiConfig)); err != nil {
 		return err
 	}
+	if err := d.Set("enable_k8s_beta_apis", flattenEnableK8sBetaApis(cluster.EnableK8sBetaApis)); err != nil {
+		return err
+	}
 	if err := d.Set("logging_config", flattenContainerClusterLoggingConfig(cluster.LoggingConfig)); err != nil {
 		return err
 	}
@@ -3346,6 +3368,44 @@ func resourceContainerClusterUpdate(d *schema.ResourceData, meta interface{}) er
 		}
 	}
 
+	if d.HasChange("enable_k8s_beta_apis") {
+		log.Print("[INFO] Enable Kubernetes Beta APIs")
+		if v, ok := d.GetOk("enable_k8s_beta_apis"); ok {
+			name := containerClusterFullName(project, location, clusterName)
+			clusterGetCall := config.NewContainerClient(userAgent).Projects.Locations.Clusters.Get(name)
+			if config.UserProjectOverride {
+				clusterGetCall.Header().Add("X-Goog-User-Project", project)
+			}
+			// Fetch the cluster information to get the already enabled Beta APIs.
+			cluster, err := clusterGetCall.Do()
+			if err != nil {
+				return err
+			}
+
+			// To avoid an already enabled Beta APIs error, we need to deduplicate the requested APIs
+			// with those that are already enabled.
+			var enabledAPIs []string
+			if cluster.EnableK8sBetaApis != nil && len(cluster.EnableK8sBetaApis.EnabledApis) > 0 {
+				enabledAPIs = cluster.EnableK8sBetaApis.EnabledApis
+			}
+			enableK8sBetaAPIs := expandEnableK8sBetaApis(v, enabledAPIs)
+
+			req := &container.UpdateClusterRequest{
+				Update: &container.ClusterUpdate{
+					DesiredK8sBetaApis: enableK8sBetaAPIs,
+				},
+			}
+
+			updateF := updateFunc(req, "updating enabled Kubernetes Beta APIs")
+			// Call update serially.
+			if err := transport_tpg.LockedCall(lockKey, updateF); err != nil {
+				return err
+			}
+
+			log.Printf("[INFO] GKE cluster %s enabled Kubernetes Beta APIs has been updated", d.Id())
+		}
+	}
+
 	if d.HasChange("node_pool_defaults") && d.HasChange("node_pool_defaults.0.node_config_defaults.0.logging_variant") {
 		if v, ok := d.GetOk("node_pool_defaults.0.node_config_defaults.0.logging_variant"); ok {
 			loggingVariant := v.(string)
@@ -4318,6 +4378,28 @@ func expandGatewayApiConfig(configured interface{}) *container.GatewayAPIConfig
 	}
 }
 
+func expandEnableK8sBetaApis(configured interface{}, enabledAPIs []string) *container.K8sBetaAPIConfig {
+	l := configured.([]interface{})
+	if len(l) == 0 || l[0] == nil {
+		return nil
+	}
+
+	config := l[0].(map[string]interface{})
+	result := &container.K8sBetaAPIConfig{}
+	if v, ok := config["enabled_apis"]; ok {
+		notEnabledAPIsSet := v.(*schema.Set)
+		for _, enabledAPI := range enabledAPIs {
+			if notEnabledAPIsSet.Contains(enabledAPI) {
+				notEnabledAPIsSet.Remove(enabledAPI)
+			}
+		}
+
+		result.EnabledApis = tpgresource.ConvertStringSet(notEnabledAPIsSet)
+	}
+
+	return result
+}
+
 func expandContainerClusterLoggingConfig(configured interface{}) *container.LoggingConfig {
 	l := configured.([]interface{})
 	if len(l) == 0 {
@@ -5012,6 +5094,17 @@ func flattenGatewayApiConfig(c *container.GatewayAPIConfig) []map[string]interfa
 	}
 }
 
+func flattenEnableK8sBetaApis(c *container.K8sBetaAPIConfig) []map[string]interface{} {
+	if c == nil {
+		return nil
+	}
+	return []map[string]interface{}{
+		{
+			"enabled_apis": c.EnabledApis,
+		},
+	}
+}
+
 func flattenContainerClusterLoggingConfig(c *container.LoggingConfig) []map[string]interface{} {
 	if c == nil {
 		return nil
@@ -5263,3 +5356,31 @@ func containerClusterSurgeSettingsCustomizeDiff(_ context.Context, d *schema.Res
 
 	return nil
 }
+
+func containerClusterEnableK8sBetaApisCustomizeDiff(_ context.Context, d *schema.ResourceDiff, meta interface{}) error {
+	// separate func to allow unit testing
+	return containerClusterEnableK8sBetaApisCustomizeDiffFunc(d)
+}
+
+func containerClusterEnableK8sBetaApisCustomizeDiffFunc(d tpgresource.TerraformResourceDiff) error {
+	// The Kubernetes Beta APIs cannot be disabled once they have been enabled by users.
+	// The reason why we don't allow disabling is that the controller does not have the
+	// ability to clean up the Kubernetes objects created by the APIs. If the user
+	// removes the already enabled Kubernetes Beta API from the list, we need to force
+	// a new cluster.
+	if !d.HasChange("enable_k8s_beta_apis.0.enabled_apis") {
+		return nil
+	}
+	old, new := d.GetChange("enable_k8s_beta_apis.0.enabled_apis")
+	if old != "" && new != "" {
+		oldAPIsSet := old.(*schema.Set)
+		newAPIsSet := new.(*schema.Set)
+		for _, oldAPI := range oldAPIsSet.List() {
+			if !newAPIsSet.Contains(oldAPI) {
+				return d.ForceNew("enable_k8s_beta_apis.0.enabled_apis")
+			}
+		}
+	}
+
+	return nil
+}