From 55c00d87104ff74ed63e2ba663de135e9063fa93 Mon Sep 17 00:00:00 2001 From: Vijaykanth Melugiri Date: Thu, 22 Aug 2024 22:08:49 +0000 Subject: [PATCH 1/5] Add Resource SCC Findings Export to BQ Project Config --- .../ProjectSccBigQueryExports.yaml | 139 ++++++++++++++++ ...oject_big_query_export_config_basic.tf.erb | 26 +++ ..._v2_project_big_query_export_config.go.erb | 12 ++ ...v2_project_big_query_export_config_test.go | 148 ++++++++++++++++++ 4 files changed, 325 insertions(+) create mode 100644 mmv1/products/securitycenterv2/ProjectSccBigQueryExports.yaml create mode 100644 mmv1/templates/terraform/examples/scc_v2_project_big_query_export_config_basic.tf.erb create mode 100644 mmv1/templates/terraform/post_import/scc_v2_project_big_query_export_config.go.erb create mode 100644 mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go diff --git a/mmv1/products/securitycenterv2/ProjectSccBigQueryExports.yaml b/mmv1/products/securitycenterv2/ProjectSccBigQueryExports.yaml new file mode 100644 index 000000000000..ca56985cc390 --- /dev/null +++ b/mmv1/products/securitycenterv2/ProjectSccBigQueryExports.yaml @@ -0,0 +1,139 @@ +# Copyright 2024 Google Inc. +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +--- !ruby/object:Api::Resource +name: 'ProjectSccBigQueryExports' +base_url: projects/{{project}}/locations/{{location}}/bigQueryExports +self_link: projects/{{project}}/locations/{{location}}/bigQueryExports/{{big_query_export_id}} +create_url: projects/{{project}}/locations/{{location}}/bigQueryExports?bigQueryExportId={{big_query_export_id}} +update_verb: :PATCH +update_mask: true +import_format: + - 'projects/{{project}}/locations/{{location}}/bigQueryExports/{{big_query_export_id}}' +custom_code: !ruby/object:Provider::Terraform::CustomCode + post_import: templates/terraform/post_import/scc_v2_project_big_query_export_config.go.erb +description: | + A Cloud Security Command Center (Cloud SCC) Big Query Export Config. + It represents exporting Security Command Center data, including assets, findings, and security marks + using gcloud scc bqexports + ~> **Note:** In order to use Cloud SCC resources, your organization must be enrolled + in [SCC Standard/Premium](https://cloud.google.com/security-command-center/docs/quickstart-security-command-center). + Without doing so, you may run into errors during resource creation. +references: !ruby/object:Api::Resource::ReferenceLinks + guides: + 'Official Documentation': 'https://cloud.google.com/security-command-center/docs/how-to-analyze-findings-in-big-query' + api: 'https://cloud.google.com/security-command-center/docs/reference/rest/v2/projects.locations.bigQueryExports' +examples: + - !ruby/object:Provider::Terraform::Examples + name: 'scc_v2_project_big_query_export_config_basic' + primary_resource_id: 'custom_big_query_export_config' + external_providers: ["random", "time"] + skip_test: true + vars: + big_query_export_id: 'my-export' + dataset: 'my-dataset' + dataset_id: 'my_dataset_id' + name: 'my-export' + test_env_vars: + org_id: :ORG_ID + project: :PROJECT_NAME + ignore_read_extra: + - 'project' + +parameters: + - !ruby/object:Api::Type::String + name: bigQueryExportId + required: true + immutable: true + url_param_only: true + description: | + This must be unique within the organization. + - !ruby/object:Api::Type::String + name: location + immutable: true + url_param_only: true + default_value: global + description: | + location Id is provided by organization. If not provided, Use global as default. +properties: + - !ruby/object:Api::Type::String + name: name + description: | + The resource name of this export, in the format + `projects/{{project}}/locations/{{location}}/bigQueryExports/{{big_query_export_id}}`. + This field is provided in responses, and is ignored when provided in create requests. + - !ruby/object:Api::Type::String + name: description + description: | + The description of the notification config (max of 1024 characters). + validation: !ruby/object:Provider::Terraform::Validation + function: 'validation.StringLenBetween(0, 1024)' + - !ruby/object:Api::Type::String + name: dataset + description: | + The dataset to write findings' updates to. + Its format is "projects/[projectId]/datasets/[bigquery_dataset_id]". + BigQuery Dataset unique ID must contain only letters (a-z, A-Z), numbers (0-9), or underscores (_). + - !ruby/object:Api::Type::String + name: createTime + output: true + description: | + The time at which the BigQuery export was created. This field is set by the server and will be ignored if provided on export on creation. + A timestamp in RFC3339 UTC "Zulu" format, with nanosecond resolution and up to nine fractional digits. + Examples: "2014-10-02T15:01:23Z" and "2014-10-02T15:01:23.045123456Z". + - !ruby/object:Api::Type::String + name: updateTime + output: true + description: | + The most recent time at which the BigQuery export was updated. This field is set by the server and will be ignored if provided on export creation or update. + A timestamp in RFC3339 UTC "Zulu" format, with nanosecond resolution and up to nine fractional digits. + Examples: "2014-10-02T15:01:23Z" and "2014-10-02T15:01:23.045123456Z". + - !ruby/object:Api::Type::String + name: mostRecentEditor + output: true + description: | + Email address of the user who last edited the BigQuery export. + This field is set by the server and will be ignored if provided on export creation or update. + - !ruby/object:Api::Type::String + name: principal + output: true + description: | + The service account that needs permission to create table and upload data to the BigQuery dataset. + - !ruby/object:Api::Type::String + name: filter + description: | + Expression that defines the filter to apply across create/update + events of findings. The + expression is a list of zero or more restrictions combined via + logical operators AND and OR. Parentheses are supported, and OR + has higher precedence than AND. + + Restrictions have the form and may have + a - character in front of them to indicate negation. The fields + map to those defined in the corresponding resource. + + The supported operators are: + + * = for all value types. + * >, <, >=, <= for integer values. + * :, meaning substring matching, for strings. + + The supported value types are: + + * string literals in quotes. + * integer literals without quotes. + * boolean literals true and false without quotes. + + See + [Filtering notifications](https://cloud.google.com/security-command-center/docs/how-to-api-filter-notifications) + for information on how to write a filter. diff --git a/mmv1/templates/terraform/examples/scc_v2_project_big_query_export_config_basic.tf.erb b/mmv1/templates/terraform/examples/scc_v2_project_big_query_export_config_basic.tf.erb new file mode 100644 index 000000000000..34861b883b0f --- /dev/null +++ b/mmv1/templates/terraform/examples/scc_v2_project_big_query_export_config_basic.tf.erb @@ -0,0 +1,26 @@ +resource "google_bigquery_dataset" "default" { + dataset_id = "<%= ctx[:vars]['dataset_id'] %>" + friendly_name = "test" + description = "This is a test description" + location = "US" + default_table_expiration_ms = 3600000 + default_partition_expiration_ms = null + + labels = { + env = "default" + } + + lifecycle { + ignore_changes = [default_partition_expiration_ms] + } +} + +resource "google_scc_v2_project_scc_big_query_exports" "<%= ctx[:primary_resource_id] %>" { + name = "<%= ctx[:vars]['name'] %>" + big_query_export_id = "<%= ctx[:vars]['big_query_export_id'] %>" + project = "<%= ctx[:test_env_vars]['project'] %>" + dataset = "<%= ctx[:vars]['dataset'] %>" + location = "global" + description = "Cloud Security Command Center Findings Big Query Export Config" + filter = "state=\"ACTIVE\" AND NOT mute=\"MUTED\"" +} diff --git a/mmv1/templates/terraform/post_import/scc_v2_project_big_query_export_config.go.erb b/mmv1/templates/terraform/post_import/scc_v2_project_big_query_export_config.go.erb new file mode 100644 index 000000000000..68b425773c18 --- /dev/null +++ b/mmv1/templates/terraform/post_import/scc_v2_project_big_query_export_config.go.erb @@ -0,0 +1,12 @@ +idParts := strings.Split(d.Id(), "/") +if len(idParts) != 6 { + return nil, fmt.Errorf("unexpected format of ID (%q), expected projects/{{project}}/locations/{{location}}/bigQueryExports/{{big_query_export_id}}", d.Id()) +} + +if err := d.Set("project", idParts[1]); err != nil { + return nil, fmt.Errorf("error setting project: %s", err) +} + +if err := d.Set("big_query_export_id", idParts[5]); err != nil { + return nil, fmt.Errorf("error setting big_query_export_id: %s", err) +} diff --git a/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go b/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go new file mode 100644 index 000000000000..cf6896b89d1a --- /dev/null +++ b/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go @@ -0,0 +1,148 @@ +package securitycenterv2_test + +import ( + "fmt" + "testing" + + "github.com/hashicorp/terraform-plugin-testing/helper/resource" + "github.com/hashicorp/terraform-provider-google/google/acctest" + "github.com/hashicorp/terraform-provider-google/google/envvar" +) + +func TestAccSecurityCenterV2ProjectBigQueryExportConfig_basic(t *testing.T) { + t.Parallel() + + randomSuffix := acctest.RandString(t, 10) + dataset_id := "tf_test_" + randomSuffix + orgID := envvar.GetTestOrgFromEnv(t) + + context := map[string]interface{}{ + "org_id": orgID, + "random_suffix": randomSuffix, + "dataset_id": dataset_id, + "dataset": fmt.Sprintf("projects/%s/datasets/%s", + envvar.GetTestProjectFromEnv(), dataset_id), + "big_query_export_id": "tf-test-export-" + randomSuffix, + "name": fmt.Sprintf("organizations/%s/locations/global/bigQueryExports/%s", + orgID, "tf-test-export-"+randomSuffix), + "project": envvar.GetTestProjectFromEnv(), + } + + acctest.VcrTest(t, resource.TestCase{ + PreCheck: func() { acctest.AccTestPreCheck(t) }, + ProtoV5ProviderFactories: acctest.ProtoV5ProviderFactories(t), + ExternalProviders: map[string]resource.ExternalProvider{ + "random": {}, + "time": {}, + }, + Steps: []resource.TestStep{ + { + Config: testAccSecurityCenterV2ProjectBigQueryExportConfig_basic(context), + }, + { + ResourceName: "google_scc_v2_project_scc_big_query_exports.default", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"update_time", "project"}, + }, + { + Config: testAccSecurityCenterV2ProjectBigQueryExportConfig_update(context), + }, + { + ResourceName: "google_scc_v2_project_scc_big_query_exports.default", + ImportState: true, + ImportStateVerify: true, + ImportStateVerifyIgnore: []string{"update_time", "project"}, + }, + }, + }) +} + +func testAccSecurityCenterV2ProjectBigQueryExportConfig_basic(context map[string]interface{}) string { + return acctest.Nprintf(` + +resource "google_bigquery_dataset" "default" { + dataset_id = "%{dataset_id}" + friendly_name = "test" + description = "This is a test description" + location = "US" + default_table_expiration_ms = 3600000 + default_partition_expiration_ms = null + + labels = { + env = "default" + } + + lifecycle { + ignore_changes = [default_partition_expiration_ms] + } +} + +resource "time_sleep" "wait_1_minute" { + depends_on = [google_bigquery_dataset.default] + create_duration = "3m" +} + +resource "google_scc_v2_project_scc_big_query_exports" "default" { + name = "%{name}" + big_query_export_id = "%{big_query_export_id}" + project = "%{project}" + dataset = "%{dataset}" + location = "global" + description = "Cloud Security Command Center Findings Big Query Export Config" + filter = "state=\"ACTIVE\" AND NOT mute=\"MUTED\"" + + lifecycle { + ignore_changes = [name] + } + + depends_on = [time_sleep.wait_1_minute] +} + +resource "time_sleep" "wait_for_cleanup" { + create_duration = "3m" + depends_on = [google_scc_v2_project_scc_big_query_exports.default] +} +`, context) +} + +func testAccSecurityCenterV2ProjectBigQueryExportConfig_update(context map[string]interface{}) string { + return acctest.Nprintf(` + +resource "google_bigquery_dataset" "default" { + dataset_id = "%{dataset_id}" + friendly_name = "test" + description = "This is a test description" + location = "US" + default_table_expiration_ms = 3600000 + default_partition_expiration_ms = null + + labels = { + env = "default" + } + + lifecycle { + ignore_changes = [default_partition_expiration_ms] + } +} + +resource "google_scc_v2_project_scc_big_query_exports" "default" { + name = "%{name}" + big_query_export_id = "%{big_query_export_id}" + project = "%{project}" + dataset = "%{dataset}" + location = "global" + description = "SCC Findings Big Query Export Update" + filter = "state=\"ACTIVE\" AND NOT mute=\"MUTED\"" + + lifecycle { + ignore_changes = [name] + } +} + +resource "time_sleep" "wait_for_cleanup" { + create_duration = "3m" + depends_on = [google_scc_v2_project_scc_big_query_exports.default] +} +`, context) +} From a1a238a07f454faa19e48fa5d3b5b9c004afd928 Mon Sep 17 00:00:00 2001 From: Vijaykanth Melugiri Date: Thu, 22 Aug 2024 22:33:55 +0000 Subject: [PATCH 2/5] Refactor the name property in the test file --- .../resource_scc_v2_project_big_query_export_config_test.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go b/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go index cf6896b89d1a..1a11984037e9 100644 --- a/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go +++ b/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go @@ -23,8 +23,8 @@ func TestAccSecurityCenterV2ProjectBigQueryExportConfig_basic(t *testing.T) { "dataset": fmt.Sprintf("projects/%s/datasets/%s", envvar.GetTestProjectFromEnv(), dataset_id), "big_query_export_id": "tf-test-export-" + randomSuffix, - "name": fmt.Sprintf("organizations/%s/locations/global/bigQueryExports/%s", - orgID, "tf-test-export-"+randomSuffix), + "name": fmt.Sprintf("projects/%s/locations/global/bigQueryExports/%s", + envvar.GetTestProjectFromEnv(), "tf-test-export-"+randomSuffix), "project": envvar.GetTestProjectFromEnv(), } From d324b20193aac7ad6ff4eecf9f5caa9ac1e282d6 Mon Sep 17 00:00:00 2001 From: Vijaykanth Melugiri Date: Tue, 27 Aug 2024 18:48:03 +0000 Subject: [PATCH 3/5] lint fix --- .../resource_scc_v2_project_big_query_export_config_test.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go b/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go index 1a11984037e9..4e1369150aa2 100644 --- a/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go +++ b/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go @@ -25,7 +25,7 @@ func TestAccSecurityCenterV2ProjectBigQueryExportConfig_basic(t *testing.T) { "big_query_export_id": "tf-test-export-" + randomSuffix, "name": fmt.Sprintf("projects/%s/locations/global/bigQueryExports/%s", envvar.GetTestProjectFromEnv(), "tf-test-export-"+randomSuffix), - "project": envvar.GetTestProjectFromEnv(), + "project": envvar.GetTestProjectFromEnv(), } acctest.VcrTest(t, resource.TestCase{ From e4eb261e0ccdb377e4bc47de218d538afd11bc98 Mon Sep 17 00:00:00 2001 From: Vijaykanth Melugiri Date: Tue, 27 Aug 2024 22:54:53 +0000 Subject: [PATCH 4/5] Address Comments --- .../ProjectSccBigQueryExports.yaml | 3 +-- ...oject_big_query_export_config_basic.tf.erb | 2 +- ..._v2_project_big_query_export_config.go.erb | 12 ----------- ...v2_project_big_query_export_config_test.go | 21 +++++-------------- 4 files changed, 7 insertions(+), 31 deletions(-) delete mode 100644 mmv1/templates/terraform/post_import/scc_v2_project_big_query_export_config.go.erb diff --git a/mmv1/products/securitycenterv2/ProjectSccBigQueryExports.yaml b/mmv1/products/securitycenterv2/ProjectSccBigQueryExports.yaml index ca56985cc390..52b2261aa31b 100644 --- a/mmv1/products/securitycenterv2/ProjectSccBigQueryExports.yaml +++ b/mmv1/products/securitycenterv2/ProjectSccBigQueryExports.yaml @@ -20,8 +20,6 @@ update_verb: :PATCH update_mask: true import_format: - 'projects/{{project}}/locations/{{location}}/bigQueryExports/{{big_query_export_id}}' -custom_code: !ruby/object:Provider::Terraform::CustomCode - post_import: templates/terraform/post_import/scc_v2_project_big_query_export_config.go.erb description: | A Cloud Security Command Center (Cloud SCC) Big Query Export Config. It represents exporting Security Command Center data, including assets, findings, and security marks @@ -68,6 +66,7 @@ parameters: properties: - !ruby/object:Api::Type::String name: name + output: true description: | The resource name of this export, in the format `projects/{{project}}/locations/{{location}}/bigQueryExports/{{big_query_export_id}}`. diff --git a/mmv1/templates/terraform/examples/scc_v2_project_big_query_export_config_basic.tf.erb b/mmv1/templates/terraform/examples/scc_v2_project_big_query_export_config_basic.tf.erb index 34861b883b0f..d88c63c15a4c 100644 --- a/mmv1/templates/terraform/examples/scc_v2_project_big_query_export_config_basic.tf.erb +++ b/mmv1/templates/terraform/examples/scc_v2_project_big_query_export_config_basic.tf.erb @@ -19,7 +19,7 @@ resource "google_scc_v2_project_scc_big_query_exports" "<%= ctx[:primary_resourc name = "<%= ctx[:vars]['name'] %>" big_query_export_id = "<%= ctx[:vars]['big_query_export_id'] %>" project = "<%= ctx[:test_env_vars]['project'] %>" - dataset = "<%= ctx[:vars]['dataset'] %>" + dataset = google_bigquery_dataset.default.id location = "global" description = "Cloud Security Command Center Findings Big Query Export Config" filter = "state=\"ACTIVE\" AND NOT mute=\"MUTED\"" diff --git a/mmv1/templates/terraform/post_import/scc_v2_project_big_query_export_config.go.erb b/mmv1/templates/terraform/post_import/scc_v2_project_big_query_export_config.go.erb deleted file mode 100644 index 68b425773c18..000000000000 --- a/mmv1/templates/terraform/post_import/scc_v2_project_big_query_export_config.go.erb +++ /dev/null @@ -1,12 +0,0 @@ -idParts := strings.Split(d.Id(), "/") -if len(idParts) != 6 { - return nil, fmt.Errorf("unexpected format of ID (%q), expected projects/{{project}}/locations/{{location}}/bigQueryExports/{{big_query_export_id}}", d.Id()) -} - -if err := d.Set("project", idParts[1]); err != nil { - return nil, fmt.Errorf("error setting project: %s", err) -} - -if err := d.Set("big_query_export_id", idParts[5]); err != nil { - return nil, fmt.Errorf("error setting big_query_export_id: %s", err) -} diff --git a/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go b/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go index 4e1369150aa2..d6a1d96325b7 100644 --- a/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go +++ b/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go @@ -17,11 +17,9 @@ func TestAccSecurityCenterV2ProjectBigQueryExportConfig_basic(t *testing.T) { orgID := envvar.GetTestOrgFromEnv(t) context := map[string]interface{}{ - "org_id": orgID, - "random_suffix": randomSuffix, - "dataset_id": dataset_id, - "dataset": fmt.Sprintf("projects/%s/datasets/%s", - envvar.GetTestProjectFromEnv(), dataset_id), + "org_id": orgID, + "random_suffix": randomSuffix, + "dataset_id": dataset_id, "big_query_export_id": "tf-test-export-" + randomSuffix, "name": fmt.Sprintf("projects/%s/locations/global/bigQueryExports/%s", envvar.GetTestProjectFromEnv(), "tf-test-export-"+randomSuffix), @@ -84,18 +82,13 @@ resource "time_sleep" "wait_1_minute" { } resource "google_scc_v2_project_scc_big_query_exports" "default" { - name = "%{name}" big_query_export_id = "%{big_query_export_id}" project = "%{project}" - dataset = "%{dataset}" + dataset = google_bigquery_dataset.default.id location = "global" description = "Cloud Security Command Center Findings Big Query Export Config" filter = "state=\"ACTIVE\" AND NOT mute=\"MUTED\"" - lifecycle { - ignore_changes = [name] - } - depends_on = [time_sleep.wait_1_minute] } @@ -127,17 +120,13 @@ resource "google_bigquery_dataset" "default" { } resource "google_scc_v2_project_scc_big_query_exports" "default" { - name = "%{name}" big_query_export_id = "%{big_query_export_id}" project = "%{project}" - dataset = "%{dataset}" + dataset = google_bigquery_dataset.default.id location = "global" description = "SCC Findings Big Query Export Update" filter = "state=\"ACTIVE\" AND NOT mute=\"MUTED\"" - lifecycle { - ignore_changes = [name] - } } resource "time_sleep" "wait_for_cleanup" { From 4873e456e2d238c21b7eb1709c516487469ff495 Mon Sep 17 00:00:00 2001 From: Vijaykanth Melugiri Date: Wed, 28 Aug 2024 17:57:36 +0000 Subject: [PATCH 5/5] Remove time sleep of cleanup --- ...esource_scc_v2_project_big_query_export_config_test.go | 8 -------- 1 file changed, 8 deletions(-) diff --git a/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go b/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go index d6a1d96325b7..7769b14b48c1 100644 --- a/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go +++ b/mmv1/third_party/terraform/services/securitycenterv2/resource_scc_v2_project_big_query_export_config_test.go @@ -92,10 +92,6 @@ resource "google_scc_v2_project_scc_big_query_exports" "default" { depends_on = [time_sleep.wait_1_minute] } -resource "time_sleep" "wait_for_cleanup" { - create_duration = "3m" - depends_on = [google_scc_v2_project_scc_big_query_exports.default] -} `, context) } @@ -129,9 +125,5 @@ resource "google_scc_v2_project_scc_big_query_exports" "default" { } -resource "time_sleep" "wait_for_cleanup" { - create_duration = "3m" - depends_on = [google_scc_v2_project_scc_big_query_exports.default] -} `, context) }