Support BigQuery authorized routines (#6680)

* BigQuery: Added support for authorized routine * BigQuery: Added examples for authorized routine * Update mmv1/products/bigquery/terraform.yaml Co-authored-by: Stephen Lewis (Burrows) <[email protected]> * Update mmv1/templates/terraform/examples/bigquery_dataset_authorized_routine.tf.erb Co-authored-by: Stephen Lewis (Burrows) <[email protected]> * add legacy iam access to test case * remove legacy roles * correct variable names and values * replace - to _ * [wip]added hand written test * fix corrupted test * Update mmv1/third_party/terraform/tests/resource_bigquery_dataset_access_test.go Co-authored-by: Stephen Lewis (Burrows) <[email protected]> Co-authored-by: Stephen Lewis (Burrows) <[email protected]>
GoogleCloudPlatform · Nov 8, 2022 · c5063eb · c5063eb
1 parent ceb4c49
commit c5063eb
Show file tree

Hide file tree

Showing 6 changed files with 238 additions and 0 deletions.
diff --git a/mmv1/products/bigquery/api.yaml b/mmv1/products/bigquery/api.yaml
@@ -141,6 +141,30 @@ objects:
                     but additional target types may be added in the future. Possible values: VIEWS
                   item_type: Api::Type::String
                   required: true
+            - !ruby/object:Api::Type::NestedObject
+              name: 'routine'
+              description: |
+                A routine from a different dataset to grant access to. Queries
+                executed against that routine will have read access to tables in
+                this dataset. The role field is not required when this field is
+                set. If that routine is updated by any user, access to the routine
+                needs to be granted again via an update operation.
+              properties:
+                - !ruby/object:Api::Type::String
+                  name: 'datasetId'
+                  description: The ID of the dataset containing this table.
+                  required: true
+                - !ruby/object:Api::Type::String
+                  name: 'projectId'
+                  description: The ID of the project containing this table.
+                  required: true
+                - !ruby/object:Api::Type::String
+                  name: 'routineId'
+                  description: |
+                    The ID of the routine. The ID must contain only letters (a-z,
+                    A-Z), numbers (0-9), or underscores (_). The maximum length
+                    is 256 characters.
+                  required: true
       - !ruby/object:Api::Type::Integer
         name: 'creationTime'
         output: true
@@ -278,6 +302,7 @@ objects:
       - iamMember
       - view
       - dataset
+      - routine
     description: |
       Gives dataset access for a single entity. This resource is intended to be used in cases where
       it is not possible to compile a full list of access blocks to include in a
@@ -320,6 +345,7 @@ objects:
           - iam_member
           - view
           - dataset
+          - routine
       - !ruby/object:Api::Type::String
         name: 'groupByEmail'
         description: An email address of a Google Group to grant access to.
@@ -331,6 +357,7 @@ objects:
           - iam_member
           - view
           - dataset
+          - routine
       - !ruby/object:Api::Type::String
         name: 'domain'
         description: |
@@ -344,6 +371,7 @@ objects:
           - iam_member
           - view
           - dataset
+          - routine
       - !ruby/object:Api::Type::String
         name: 'specialGroup'
         description: |
@@ -368,6 +396,7 @@ objects:
           - iam_member
           - view
           - dataset
+          - routine
       - !ruby/object:Api::Type::String
         name: 'iamMember'
         description: |
@@ -381,6 +410,7 @@ objects:
           - iam_member
           - view
           - dataset
+          - routine
       - !ruby/object:Api::Type::NestedObject
         name: 'view'
         description: |
@@ -397,6 +427,7 @@ objects:
           - iam_member
           - view
           - dataset
+          - routine
         properties:
           - !ruby/object:Api::Type::String
             name: 'datasetId'
@@ -425,6 +456,7 @@ objects:
           - iam_member
           - view
           - dataset
+          - routine
         properties:
           - !ruby/object:Api::Type::NestedObject
             name: 'dataset'
@@ -447,6 +479,39 @@ objects:
               but additional target types may be added in the future. Possible values: VIEWS
             item_type: Api::Type::String
             required: true
+      - !ruby/object:Api::Type::NestedObject
+        name: 'routine'
+        description: |
+          A routine from a different dataset to grant access to. Queries
+          executed against that routine will have read access to tables in
+          this dataset. The role field is not required when this field is
+          set. If that routine is updated by any user, access to the routine
+          needs to be granted again via an update operation.
+        exactly_one_of:
+          - user_by_email
+          - group_by_email
+          - domain
+          - special_group
+          - iam_member
+          - view
+          - dataset
+          - routine
+        properties:
+          - !ruby/object:Api::Type::String
+            name: 'datasetId'
+            description: The ID of the dataset containing this table.
+            required: true
+          - !ruby/object:Api::Type::String
+            name: 'projectId'
+            description: The ID of the project containing this table.
+            required: true
+          - !ruby/object:Api::Type::String
+            name: 'routineId'
+            description: |
+              The ID of the routine. The ID must contain only letters (a-z,
+              A-Z), numbers (0-9), or underscores (_). The maximum length
+              is 256 characters.
+            required: true
   - !ruby/object:Api::Resource
     name: 'Job'
     kind: 'bigquery#job'

diff --git a/mmv1/products/bigquery/terraform.yaml b/mmv1/products/bigquery/terraform.yaml
@@ -48,6 +48,15 @@ overrides: !ruby/object:Overrides::ResourceOverrides
           private: "private"
           public: "public"
           account_name: "bqowner"
+      - !ruby/object:Provider::Terraform::Examples
+        name: "bigquery_dataset_authorized_routine"
+        primary_resource_id: "private"
+        vars:
+          private_dataset: "private_dataset"
+          public_dataset: "public_dataset"
+          public_routine: "public_routine"
+        test_env_vars:
+          service_account: :SERVICE_ACCT
     virtual_fields:
       - !ruby/object:Api::Type::Boolean
         name: 'delete_contents_on_destroy'
@@ -116,6 +125,15 @@ overrides: !ruby/object:Overrides::ResourceOverrides
         vars:
           private: "private"
           public: "public"
+      - !ruby/object:Provider::Terraform::Examples
+        name: "bigquery_dataset_access_authorized_routine"
+        skip_test: true  # not importable
+        primary_resource_type: "google_bigquery_dataset_access"
+        primary_resource_id: "authorized_routine"
+        vars:
+          private_dataset: "private_dataset"
+          public_dataset: "public_dataset"
+          public_routine: "public_routine"
     properties:
       datasetId: !ruby/object:Overrides::Terraform::PropertyOverride
         ignore_read: true

diff --git a/mmv1/templates/terraform/examples/bigquery_dataset_access_authorized_routine.tf.erb b/mmv1/templates/terraform/examples/bigquery_dataset_access_authorized_routine.tf.erb
@@ -0,0 +1,36 @@
+resource "google_bigquery_dataset" "public" {
+  dataset_id  = "<%= ctx[:vars]['public_dataset'] %>"
+  description = "This dataset is public"
+}
+
+resource "google_bigquery_routine" "public" {
+  dataset_id      = google_bigquery_dataset.public.dataset_id
+  routine_id      = "<%= ctx[:vars]['public_routine'] %>"
+  routine_type    = "TABLE_VALUED_FUNCTION"
+  language        = "SQL"
+  definition_body = <<-EOS
+    SELECT 1 + value AS value
+  EOS
+  arguments {
+    name          = "value"
+    argument_kind = "FIXED_TYPE"
+    data_type     = jsonencode({ "typeKind" = "INT64" })
+  }
+  return_table_type = jsonencode({ "columns" = [
+    { "name" = "value", "type" = { "typeKind" = "INT64" } },
+  ] })
+}
+
+resource "google_bigquery_dataset" "private" {
+  dataset_id  = "<%= ctx[:vars]['private_dataset'] %>"
+  description = "This dataset is private"
+}
+
+resource "google_bigquery_dataset_access" "authorized_routine" {
+  dataset_id = google_bigquery_dataset.private.dataset_id
+  routine {
+    project_id = google_bigquery_routine.public.project
+    dataset_id = google_bigquery_routine.public.dataset_id
+    routine_id = google_bigquery_routine.public.routine_id
+  }
+}
diff --git a/mmv1/templates/terraform/examples/bigquery_dataset_authorized_routine.tf.erb b/mmv1/templates/terraform/examples/bigquery_dataset_authorized_routine.tf.erb
@@ -0,0 +1,38 @@
+resource "google_bigquery_dataset" "public" {
+  dataset_id  = "<%= ctx[:vars]['public_dataset'] %>"
+  description = "This dataset is public"
+}
+
+resource "google_bigquery_routine" "public" {
+  dataset_id      = google_bigquery_dataset.public.dataset_id
+  routine_id      = "<%= ctx[:vars]['public_routine'] %>"
+  routine_type    = "TABLE_VALUED_FUNCTION"
+  language        = "SQL"
+  definition_body = <<-EOS
+    SELECT 1 + value AS value
+  EOS
+  arguments {
+    name          = "value"
+    argument_kind = "FIXED_TYPE"
+    data_type     = jsonencode({ "typeKind" = "INT64" })
+  }
+  return_table_type = jsonencode({ "columns" = [
+    { "name" = "value", "type" = { "typeKind" = "INT64" } },
+  ] })
+}
+
+resource "google_bigquery_dataset" "private" {
+  dataset_id  = "<%= ctx[:vars]['private_dataset'] %>"
+  description = "This dataset is private"
+  access {
+    role          = "OWNER"
+    user_by_email = "<%= ctx[:test_env_vars]['service_account'] %>"
+  }
+  access {
+    routine {
+      project_id = google_bigquery_routine.public.project
+      dataset_id = google_bigquery_routine.public.dataset_id
+      routine_id = google_bigquery_routine.public.routine_id
+    }
+  }
+}
diff --git a/mmv1/third_party/terraform/tests/resource_bigquery_dataset_access_test.go b/mmv1/third_party/terraform/tests/resource_bigquery_dataset_access_test.go
@@ -100,6 +100,42 @@ func TestAccBigQueryDatasetAccess_authorizedDataset(t *testing.T) {
 	})
 }
 
+func TestAccBigQueryDatasetAccess_authorizedRoutine(t *testing.T) {
+	// Multiple fine-grained resources
+	skipIfVcr(t)
+	t.Parallel()
+
+	context := map[string]interface{}{
+		"public_dataset":  fmt.Sprintf("tf_test_public_dataset_%s", randString(t, 10)),
+		"public_routine":  fmt.Sprintf("tf_test_public_routine_%s", randString(t, 10)),
+		"private_dataset": fmt.Sprintf("tf_test_private_dataset_%s", randString(t, 10)),
+	}
+
+	expected := map[string]interface{}{
+		"routine": map[string]interface{}{
+			"projectId": getTestProjectFromEnv(),
+			"datasetId": context["public_dataset"],
+			"routineId": context["public_routine"],
+		},
+	}
+
+	vcrTest(t, resource.TestCase{
+		PreCheck:  func() { testAccPreCheck(t) },
+		Providers: testAccProviders,
+		Steps: []resource.TestStep{
+			{
+				Config: testAccBigQueryDatasetAccess_authorizedRoutine(context),
+				Check:  testAccCheckBigQueryDatasetAccessPresent(t, "google_bigquery_dataset.private", expected),
+			},
+			{
+				// Destroy step instead of CheckDestroy so we can check the access is removed without deleting the dataset
+				Config: testAccBigQueryDatasetAccess_destroy(context["private_dataset"].(string), "private"),
+				Check:  testAccCheckBigQueryDatasetAccessAbsent(t, "google_bigquery_dataset.private", expected),
+			},
+		},
+	})
+}
+
 func TestAccBigQueryDatasetAccess_multiple(t *testing.T) {
 	// Multiple fine-grained resources
 	skipIfVcr(t)
@@ -358,6 +394,47 @@ resource "google_bigquery_dataset" "public" {
 `, datasetID, datasetID2)
 }
 
+func testAccBigQueryDatasetAccess_authorizedRoutine(context map[string]interface{}) string {
+	return Nprintf(`
+resource "google_bigquery_dataset" "public" {
+  dataset_id  = "%{public_dataset}"
+  description = "This dataset is public"
+}
+
+resource "google_bigquery_routine" "public" {
+  dataset_id      = google_bigquery_dataset.public.dataset_id
+  routine_id      = "%{public_routine}"
+  routine_type    = "TABLE_VALUED_FUNCTION"
+  language        = "SQL"
+  definition_body = <<-EOS
+    SELECT 1 + value AS value
+  EOS
+  arguments {
+    name          = "value"
+    argument_kind = "FIXED_TYPE"
+    data_type     = jsonencode({ "typeKind" = "INT64" })
+  }
+  return_table_type = jsonencode({ "columns" = [
+    { "name" = "value", "type" = { "typeKind" = "INT64" } },
+  ] })
+}
+
+resource "google_bigquery_dataset" "private" {
+  dataset_id  = "%{private_dataset}"
+  description = "This dataset is private"
+}
+
+resource "google_bigquery_dataset_access" "authorized_routine" {
+  dataset_id = google_bigquery_dataset.private.dataset_id
+  routine {
+    project_id = google_bigquery_routine.public.project
+    dataset_id = google_bigquery_routine.public.dataset_id
+    routine_id = google_bigquery_routine.public.routine_id
+  }
+}
+`, context)
+}
+
 func testAccBigQueryDatasetAccess_multiple(datasetID string) string {
 	return fmt.Sprintf(`
 resource "google_bigquery_dataset_access" "access" {

diff --git a/mmv1/third_party/terraform/utils/iam_bigquery_dataset.go b/mmv1/third_party/terraform/utils/iam_bigquery_dataset.go
@@ -242,6 +242,10 @@ func accessToIamMember(access map[string]interface{}) (string, error) {
 		// dataset does not map to an IAM member, use access instead
 		return "", fmt.Errorf("Failed to convert BigQuery Dataset access to IAM member. To use views with a dataset, please use dataset_access")
 	}
+	if _, ok := access["routine"]; ok {
+		// dataset does not map to an IAM member, use access instead
+		return "", fmt.Errorf("Failed to convert BigQuery Dataset access to IAM member. To use views with a dataset, please use dataset_access")
+	}
 	if member, ok := access["userByEmail"]; ok {
 		// service accounts have "gservice" in their email. This is best guess due to lost information
 		if strings.Contains(member.(string), "gserviceaccount") {