From 0ee660a8f924fc9ced5f0aaeb28200ab2f11abf6 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Mon, 19 Dec 2022 08:59:48 +0200 Subject: [PATCH 1/5] add telemetry config --- mmv1/products/metastore/api.yaml | 15 ++++++++++++++- mmv1/products/metastore/terraform.yaml | 2 ++ 2 files changed, 16 insertions(+), 1 deletion(-) diff --git a/mmv1/products/metastore/api.yaml b/mmv1/products/metastore/api.yaml index 9515f5fce842..adaa91facfba 100644 --- a/mmv1/products/metastore/api.yaml +++ b/mmv1/products/metastore/api.yaml @@ -313,7 +313,20 @@ objects: name: 'enabled' required: true description: | - Defines whether the metastore metadata should be synced to Data Catalog. The default value is to disable syncing metastore metadata to Data Catalog. + Defines whether the metastore metadata should be synced to Data Catalog. The default value is to disable syncing metastore metadata to Data Catalog. + - !ruby/object:Api::Type::NestedObject + name: 'telemetryConfig' + description: | + The configuration specifying telemetry settings for the Dataproc Metastore service. If unspecified defaults to JSON. + properties: + - !ruby/object:Api::Type::Enum + name: 'logFormat' + default_value: :JSON + description: | + The output format of the Dataproc Metastore service's logs. + values: + - :LEGACY + - :JSON - !ruby/object:Api::Resource name: 'Federation' min_version: beta diff --git a/mmv1/products/metastore/terraform.yaml b/mmv1/products/metastore/terraform.yaml index e87891836063..6dbb99557249 100644 --- a/mmv1/products/metastore/terraform.yaml +++ b/mmv1/products/metastore/terraform.yaml @@ -75,6 +75,8 @@ overrides: !ruby/object:Overrides::ResourceOverrides default_from_api: true tier: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true + telemetryConfig: !ruby/object:Overrides::Terraform::PropertyOverride + default_from_api: true hiveMetastoreConfig.configOverrides: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true diff_suppress_func: "dataprocMetastoreServiceOverrideSuppress" From b393be26b737e86ffb1fa1ac45a52e50f0161d7c Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Mon, 19 Dec 2022 09:02:17 +0200 Subject: [PATCH 2/5] add telemetry config --- mmv1/products/metastore/terraform.yaml | 7 +++++++ .../dataproc_metastore_service_telemetry.erb | 14 ++++++++++++++ 2 files changed, 21 insertions(+) create mode 100644 mmv1/templates/terraform/examples/dataproc_metastore_service_telemetry.erb diff --git a/mmv1/products/metastore/terraform.yaml b/mmv1/products/metastore/terraform.yaml index 6dbb99557249..52632b07c48e 100644 --- a/mmv1/products/metastore/terraform.yaml +++ b/mmv1/products/metastore/terraform.yaml @@ -68,6 +68,13 @@ overrides: !ruby/object:Overrides::ResourceOverrides metastore_service_name: "metastore-srv" network_name: "my-network" subnet_name: "my-subnetwork" + - !ruby/object:Provider::Terraform::Examples + name: "dataproc_metastore_service_telemetry" + min_version: beta + skip_docs: true + primary_resource_id: "telemetry" + vars: + metastore_service_name: "telemetry" properties: network: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true diff --git a/mmv1/templates/terraform/examples/dataproc_metastore_service_telemetry.erb b/mmv1/templates/terraform/examples/dataproc_metastore_service_telemetry.erb new file mode 100644 index 000000000000..91d804028fa6 --- /dev/null +++ b/mmv1/templates/terraform/examples/dataproc_metastore_service_telemetry.erb @@ -0,0 +1,14 @@ +resource "google_dataproc_metastore_service" "<%= ctx[:primary_resource_id] %>" { + service_id = "<%= ctx[:vars]['metastore_service_name'] %>" + location = "us-central1" + port = 9080 + tier = "DEVELOPER" + + hive_metastore_config { + version = "3.1.2" + } + + telemetry_config { + log_format = "LEGACY" + } +} \ No newline at end of file From d598a8967b56499cdc0a04d5b13d8e01ab623d71 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Mon, 19 Dec 2022 10:21:04 +0200 Subject: [PATCH 3/5] add telemetry config --- ..._telemetry.erb => dataproc_metastore_service_telemetry.tf.erb} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename mmv1/templates/terraform/examples/{dataproc_metastore_service_telemetry.erb => dataproc_metastore_service_telemetry.tf.erb} (100%) diff --git a/mmv1/templates/terraform/examples/dataproc_metastore_service_telemetry.erb b/mmv1/templates/terraform/examples/dataproc_metastore_service_telemetry.tf.erb similarity index 100% rename from mmv1/templates/terraform/examples/dataproc_metastore_service_telemetry.erb rename to mmv1/templates/terraform/examples/dataproc_metastore_service_telemetry.tf.erb From 9a621d9a6d199806e0856beb02190cf3359ee9a4 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Sat, 31 Dec 2022 20:03:51 +0200 Subject: [PATCH 4/5] not beta --- mmv1/products/metastore/api.yaml | 231 +++++++++++++------------ mmv1/products/metastore/terraform.yaml | 21 ++- 2 files changed, 133 insertions(+), 119 deletions(-) diff --git a/mmv1/products/metastore/api.yaml b/mmv1/products/metastore/api.yaml index adaa91facfba..746b3060cd75 100644 --- a/mmv1/products/metastore/api.yaml +++ b/mmv1/products/metastore/api.yaml @@ -11,7 +11,8 @@ # See the License for the specific language governing permissions and # limitations under the License. ---- !ruby/object:Api::Product +--- +!ruby/object:Api::Product name: DataprocMetastore versions: - !ruby/object:Api::Product::Version @@ -19,7 +20,7 @@ versions: base_url: https://metastore.googleapis.com/v1beta/ - !ruby/object:Api::Product::Version name: ga - base_url: https://metastore.googleapis.com/v1/ + base_url: https://metastore.googleapis.com/v1/ scopes: - https://www.googleapis.com/auth/cloud-identity apis_required: @@ -28,9 +29,9 @@ apis_required: url: https://console.cloud.google.com/apis/library/metastore.googleapis.com objects: - !ruby/object:Api::Resource - name: 'Service' + name: "Service" base_url: "projects/{{project}}/locations/{{location}}/services" - create_url: 'projects/{{project}}/locations/{{location}}/services?serviceId={{service_id}}' + create_url: "projects/{{project}}/locations/{{location}}/services?serviceId={{service_id}}" self_link: "projects/{{project}}/locations/{{location}}/services/{{service_id}}" update_verb: :PATCH update_mask: true @@ -38,36 +39,40 @@ objects: A managed metastore service that serves metadata queries. references: !ruby/object:Api::Resource::ReferenceLinks guides: - 'Official Documentation': 'https://cloud.google.com/dataproc-metastore/docs/overview' - api: 'https://cloud.google.com/dataproc-metastore/docs/reference/rest/v1/projects.locations.services' + "Official Documentation": "https://cloud.google.com/dataproc-metastore/docs/overview" + api: "https://cloud.google.com/dataproc-metastore/docs/reference/rest/v1/projects.locations.services" iam_policy: !ruby/object:Api::Resource::IamPolicy parent_resource_attribute: service_id exclude: false - method_name_separator: ':' - import_format: ["projects/{{project}}/locations/{{location}}/services/{{service_id}}", "{{service_id}}"] + method_name_separator: ":" + import_format: + [ + "projects/{{project}}/locations/{{location}}/services/{{service_id}}", + "{{service_id}}", + ] async: !ruby/object:Api::OpAsync operation: !ruby/object:Api::OpAsync::Operation - path: 'name' - base_url: '{{op_id}}' + path: "name" + base_url: "{{op_id}}" wait_ms: 1000 timeouts: !ruby/object:Api::Timeouts insert_minutes: 60 update_minutes: 60 delete_minutes: 60 result: !ruby/object:Api::OpAsync::Result - path: 'response' + path: "response" status: !ruby/object:Api::OpAsync::Status - path: 'done' + path: "done" complete: true allowed: - true - false error: !ruby/object:Api::OpAsync::Error - path: 'error' - message: 'message' + path: "error" + message: "message" parameters: - !ruby/object:Api::Type::String - name: 'serviceId' + name: "serviceId" required: true input: true url_param_only: true @@ -76,7 +81,7 @@ objects: and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 63 characters. - !ruby/object:Api::Type::String - name: 'location' + name: "location" url_param_only: true input: true default_value: global @@ -85,70 +90,70 @@ objects: The default value is `global`. properties: - !ruby/object:Api::Type::String - name: 'name' + name: "name" output: true description: | The relative resource name of the metastore service. - !ruby/object:Api::Type::KeyValuePairs - name: 'labels' - description: 'User-defined labels for the metastore service.' + name: "labels" + description: "User-defined labels for the metastore service." # This is an x-product resource reference. - !ruby/object:Api::Type::String - name: 'network' + name: "network" input: true description: | The relative resource name of the VPC network on which the instance can be accessed. It is specified in the following form: "projects/{projectNumber}/global/networks/{network_id}". - !ruby/object:Api::Type::String - name: 'endpointUri' + name: "endpointUri" output: true description: | The URI of the endpoint used to access the metastore service. - !ruby/object:Api::Type::Integer - name: 'port' + name: "port" description: | The TCP port at which the metastore service is reached. Default: 9083. - !ruby/object:Api::Type::String - name: 'state' + name: "state" output: true description: | The current state of the metastore service. - !ruby/object:Api::Type::String - name: 'stateMessage' + name: "stateMessage" output: true description: | Additional information about the current state of the metastore service, if available. - !ruby/object:Api::Type::String - name: 'artifactGcsUri' + name: "artifactGcsUri" output: true description: | A Cloud Storage URI (starting with gs://) that specifies where artifacts related to the metastore service are stored. - !ruby/object:Api::Type::Enum - name: 'tier' + name: "tier" description: | The tier of the service. values: - :DEVELOPER - :ENTERPRISE - !ruby/object:Api::Type::NestedObject - name: 'maintenanceWindow' + name: "maintenanceWindow" description: | The one hour maintenance window of the metastore service. This specifies when the service can be restarted for maintenance purposes in UTC time. Maintenance window is not needed for services with the `SPANNER` database type. properties: - !ruby/object:Api::Type::Integer - name: 'hourOfDay' + name: "hourOfDay" description: | The hour of day (0-23) when the window starts. required: true - !ruby/object:Api::Type::Enum - name: 'dayOfWeek' - description: | - The day of week, when the window starts. - required: true - values: + name: "dayOfWeek" + description: | + The day of week, when the window starts. + required: true + values: - :MONDAY - :TUESDAY - :WEDNESDAY @@ -157,25 +162,25 @@ objects: - :SATURDAY - :SUNDAY - !ruby/object:Api::Type::NestedObject - name: 'encryptionConfig' + name: "encryptionConfig" description: | Information used to configure the Dataproc Metastore service to encrypt customer data at rest. properties: - !ruby/object:Api::Type::String - name: 'kmsKey' + name: "kmsKey" description: | The fully qualified customer provided Cloud KMS key name to use for customer data encryption. Use the following format: `projects/([^/]+)/locations/([^/]+)/keyRings/([^/]+)/cryptoKeys/([^/]+)` required: true input: true - !ruby/object:Api::Type::NestedObject - name: 'hiveMetastoreConfig' + name: "hiveMetastoreConfig" description: | - Configuration information specific to running Hive metastore software as the metastore service. + Configuration information specific to running Hive metastore software as the metastore service. properties: - !ruby/object:Api::Type::Enum - name: 'endpointProtocol' + name: "endpointProtocol" min_version: beta input: true default_value: :THRIFT @@ -183,99 +188,99 @@ objects: The protocol to use for the metastore service endpoint. If unspecified, defaults to `THRIFT`. values: - :THRIFT - - :GRPC + - :GRPC - !ruby/object:Api::Type::String - name: 'version' + name: "version" input: true required: true description: | The Hive metastore schema version. - !ruby/object:Api::Type::KeyValuePairs - name: 'configOverrides' + name: "configOverrides" description: | A mapping of Hive metastore configuration key-value pairs to apply to the Hive metastore (configured in hive-site.xml). The mappings override system defaults (some keys cannot be overridden) - !ruby/object:Api::Type::NestedObject - name: 'kerberosConfig' + name: "kerberosConfig" description: | Information used to configure the Hive metastore service as a service principal in a Kerberos realm. properties: - !ruby/object:Api::Type::NestedObject - name: 'keytab' + name: "keytab" required: true description: | A Kerberos keytab file that can be used to authenticate a service principal with a Kerberos Key Distribution Center (KDC). properties: - !ruby/object:Api::Type::String required: true - name: 'cloudSecret' + name: "cloudSecret" description: | The relative resource name of a Secret Manager secret version, in the following form: "projects/{projectNumber}/secrets/{secret_id}/versions/{version_id}". - !ruby/object:Api::Type::String - name: 'principal' + name: "principal" required: true description: | A Kerberos principal that exists in the both the keytab the KDC to authenticate as. A typical principal is of the form "primary/instance@REALM", but there is no exact format. - !ruby/object:Api::Type::String - name: 'krb5ConfigGcsUri' + name: "krb5ConfigGcsUri" required: true description: | A Cloud Storage URI that specifies the path to a krb5.conf file. It is of the form gs://{bucket_name}/path/to/krb5.conf, although the file does not need to be named krb5.conf explicitly. - !ruby/object:Api::Type::Map - name: 'auxiliaryVersions' + name: "auxiliaryVersions" min_version: beta description: | - A mapping of Hive metastore version to the auxiliary version configuration. - When specified, a secondary Hive metastore service is created along with the primary service. - All auxiliary versions must be less than the service's primary version. - The key is the auxiliary service name and it must match the regular expression a-z?. - This means that the first character must be a lowercase letter, and all the following characters must be hyphens, lowercase letters, or digits, except the last character, which cannot be a hyphen. - key_name: 'key' + A mapping of Hive metastore version to the auxiliary version configuration. + When specified, a secondary Hive metastore service is created along with the primary service. + All auxiliary versions must be less than the service's primary version. + The key is the auxiliary service name and it must match the regular expression a-z?. + This means that the first character must be a lowercase letter, and all the following characters must be hyphens, lowercase letters, or digits, except the last character, which cannot be a hyphen. + key_name: "key" key_description: | The auxiliary service name value_type: !ruby/object:Api::Type::NestedObject properties: - !ruby/object:Api::Type::String - name: 'version' + name: "version" required: true description: | The Hive metastore version of the auxiliary service. It must be less than the primary Hive metastore service's version. - !ruby/object:Api::Type::KeyValuePairs - name: 'configOverrides' + name: "configOverrides" description: | A mapping of Hive metastore configuration key-value pairs to apply to the auxiliary Hive metastore (configured in hive-site.xml) in addition to the primary version's overrides. If keys are present in both the auxiliary version's overrides and the primary version's overrides, the value from the auxiliary version's overrides takes precedence. - !ruby/object:Api::Type::NestedObject - name: 'networkConfig' + name: "networkConfig" input: true description: | The configuration specifying the network settings for the Dataproc Metastore service. properties: - - !ruby/object:Api::Type::Array - name: 'consumers' - description: | - The consumer-side network configuration for the Dataproc Metastore instance. - required: true - input: true - item_type: !ruby/object:Api::Type::NestedObject - properties: - - !ruby/object:Api::Type::String - name: 'endpointUri' - description: | - The URI of the endpoint used to access the metastore service. - output: true - - !ruby/object:Api::Type::String - name: 'subnetwork' - required: true - description: | - The subnetwork of the customer project from which an IP address is reserved and used as the Dataproc Metastore service's endpoint. - It is accessible to hosts in the subnet and to all hosts in a subnet in the same region and same network. - There must be at least one IP address available in the subnet's primary range. The subnet is specified in the following form: - `projects/{projectNumber}/regions/{region_id}/subnetworks/{subnetwork_id} + - !ruby/object:Api::Type::Array + name: "consumers" + description: | + The consumer-side network configuration for the Dataproc Metastore instance. + required: true + input: true + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::String + name: "endpointUri" + description: | + The URI of the endpoint used to access the metastore service. + output: true + - !ruby/object:Api::Type::String + name: "subnetwork" + required: true + description: | + The subnetwork of the customer project from which an IP address is reserved and used as the Dataproc Metastore service's endpoint. + It is accessible to hosts in the subnet and to all hosts in a subnet in the same region and same network. + There must be at least one IP address available in the subnet's primary range. The subnet is specified in the following form: + `projects/{projectNumber}/regions/{region_id}/subnetworks/{subnetwork_id} - !ruby/object:Api::Type::Enum - name: 'databaseType' + name: "databaseType" input: true default_value: :MYSQL description: | @@ -284,7 +289,7 @@ objects: - :MYSQL - :SPANNER - !ruby/object:Api::Type::Enum - name: 'releaseChannel' + name: "releaseChannel" input: true default_value: :STABLE description: | @@ -293,46 +298,46 @@ objects: - :CANARY - :STABLE - !ruby/object:Api::Type::String - name: 'uid' + name: "uid" output: true description: | The globally unique resource identifier of the metastore service. - !ruby/object:Api::Type::NestedObject - name: 'metadataIntegration' + name: "metadataIntegration" min_version: beta description: | The setting that defines how metastore metadata should be integrated with external services and systems. properties: - !ruby/object:Api::Type::NestedObject - name: 'dataCatalogConfig' + name: "dataCatalogConfig" required: true description: | The integration config for the Data Catalog service. properties: - !ruby/object:Api::Type::Boolean - name: 'enabled' + name: "enabled" required: true description: | Defines whether the metastore metadata should be synced to Data Catalog. The default value is to disable syncing metastore metadata to Data Catalog. - !ruby/object:Api::Type::NestedObject - name: 'telemetryConfig' + name: "telemetryConfig" description: | The configuration specifying telemetry settings for the Dataproc Metastore service. If unspecified defaults to JSON. properties: - !ruby/object:Api::Type::Enum - name: 'logFormat' + name: "logFormat" default_value: :JSON description: | The output format of the Dataproc Metastore service's logs. values: - :LEGACY - - :JSON + - :JSON - !ruby/object:Api::Resource - name: 'Federation' + name: "Federation" min_version: beta base_url: "projects/{{project}}/locations/{{location}}/federations" - create_url: 'projects/{{project}}/locations/{{location}}/federations?federationId={{federation_id}}' - self_link: "projects/{{project}}/locations/{{location}}/federations/{{federation_id}}" + create_url: "projects/{{project}}/locations/{{location}}/federations?federationId={{federation_id}}" + self_link: "projects/{{project}}/locations/{{location}}/federations/{{federation_id}}" update_verb: :PATCH update_mask: true description: | @@ -340,27 +345,31 @@ objects: iam_policy: !ruby/object:Api::Resource::IamPolicy parent_resource_attribute: federation_id exclude: false - method_name_separator: ':' - import_format: ["projects/{{project}}/locations/{{location}}/federations/{{federation_id}}", "{{federation_id}}"] + method_name_separator: ":" + import_format: + [ + "projects/{{project}}/locations/{{location}}/federations/{{federation_id}}", + "{{federation_id}}", + ] async: !ruby/object:Api::OpAsync operation: !ruby/object:Api::OpAsync::Operation - path: 'name' - base_url: '{{op_id}}' + path: "name" + base_url: "{{op_id}}" wait_ms: 1000 result: !ruby/object:Api::OpAsync::Result - path: 'response' + path: "response" status: !ruby/object:Api::OpAsync::Status - path: 'done' + path: "done" complete: true allowed: - true - false error: !ruby/object:Api::OpAsync::Error - path: 'error' - message: 'message' + path: "error" + message: "message" parameters: - !ruby/object:Api::Type::String - name: 'federationId' + name: "federationId" required: true input: true url_param_only: true @@ -369,63 +378,63 @@ objects: and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 63 characters. - !ruby/object:Api::Type::String - name: 'location' + name: "location" url_param_only: true input: true description: | The location where the metastore federation should reside. properties: - !ruby/object:Api::Type::String - name: 'name' + name: "name" output: true description: | The relative resource name of the metastore federation. - !ruby/object:Api::Type::KeyValuePairs - name: 'labels' - description: 'User-defined labels for the metastore federation.' + name: "labels" + description: "User-defined labels for the metastore federation." - !ruby/object:Api::Type::String - name: 'endpointUri' + name: "endpointUri" output: true description: | The URI of the endpoint used to access the metastore federation. - !ruby/object:Api::Type::String - name: 'state' + name: "state" output: true description: | The current state of the metastore federation. - !ruby/object:Api::Type::String - name: 'stateMessage' + name: "stateMessage" output: true description: | Additional information about the current state of the metastore federation, if available. - !ruby/object:Api::Type::String - name: 'uid' + name: "uid" output: true description: | The globally unique resource identifier of the metastore federation. - !ruby/object:Api::Type::String - name: 'version' + name: "version" input: true required: true description: | The Apache Hive metastore version of the federation. All backend metastore versions must be compatible with the federation version. - !ruby/object:Api::Type::Map - name: 'backendMetastores' + name: "backendMetastores" description: | A map from BackendMetastore rank to BackendMetastores from which the federation service serves metadata at query time. The map key represents the order in which BackendMetastores should be evaluated to resolve database names at query time and should be greater than or equal to zero. A BackendMetastore with a lower number will be evaluated before a BackendMetastore with a higher number. required: true - key_name: 'rank' + key_name: "rank" key_description: | represents the order in which BackendMetastores should be evaluated to resolve database names at query time and should be greater than or equal to zero. value_type: !ruby/object:Api::Type::NestedObject properties: - !ruby/object:Api::Type::String - name: 'name' + name: "name" required: true description: | The relative resource name of the metastore that is being federated. The formats of the relative resource names for the currently supported metastores are listed below: Dataplex: projects/{projectId}/locations/{location}/lakes/{lake_id} BigQuery: projects/{projectId} Dataproc Metastore: projects/{projectId}/locations/{location}/services/{serviceId} - !ruby/object:Api::Type::Enum - name: 'metastoreType' + name: "metastoreType" required: true description: | The type of the backend metastore. diff --git a/mmv1/products/metastore/terraform.yaml b/mmv1/products/metastore/terraform.yaml index 52632b07c48e..0dfa9b233585 100644 --- a/mmv1/products/metastore/terraform.yaml +++ b/mmv1/products/metastore/terraform.yaml @@ -11,11 +11,13 @@ # See the License for the specific language governing permissions and # limitations under the License. ---- !ruby/object:Provider::Terraform::Config +--- +!ruby/object:Provider::Terraform::Config overrides: !ruby/object:Overrides::ResourceOverrides Service: !ruby/object:Overrides::Terraform::ResourceOverride autogen_async: true - import_format: ["projects/{{project}}/locations/{{location}}/services/{{service_id}}"] + import_format: + ["projects/{{project}}/locations/{{location}}/services/{{service_id}}"] examples: - !ruby/object:Provider::Terraform::Examples name: "dataproc_metastore_service_basic" @@ -70,11 +72,10 @@ overrides: !ruby/object:Overrides::ResourceOverrides subnet_name: "my-subnetwork" - !ruby/object:Provider::Terraform::Examples name: "dataproc_metastore_service_telemetry" - min_version: beta skip_docs: true primary_resource_id: "telemetry" vars: - metastore_service_name: "telemetry" + metastore_service_name: "telemetry" properties: network: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true @@ -83,13 +84,17 @@ overrides: !ruby/object:Overrides::ResourceOverrides tier: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true telemetryConfig: !ruby/object:Overrides::Terraform::PropertyOverride - default_from_api: true - hiveMetastoreConfig.configOverrides: !ruby/object:Overrides::Terraform::PropertyOverride + default_from_api: true + hiveMetastoreConfig.configOverrides: + !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true diff_suppress_func: "dataprocMetastoreServiceOverrideSuppress" Federation: !ruby/object:Overrides::Terraform::ResourceOverride autogen_async: true - import_format: ["projects/{{project}}/locations/{{location}}/federations/{{federation_id}}"] + import_format: + [ + "projects/{{project}}/locations/{{location}}/federations/{{federation_id}}", + ] examples: - !ruby/object:Provider::Terraform::Examples name: "dataproc_metastore_federation_basic" @@ -104,4 +109,4 @@ overrides: !ruby/object:Overrides::ResourceOverrides min_version: beta primary_resource_name: 'fmt.Sprintf("tf-test-metastore-fed-bq%s", context["random_suffix"])' vars: - metastore_federation_name: "fed-2" + metastore_federation_name: "fed-2" From 61340e0c5aa6980077c6fcd2aeb684e04d7170b6 Mon Sep 17 00:00:00 2001 From: drfaust92 Date: Wed, 4 Jan 2023 07:51:12 +0200 Subject: [PATCH 5/5] unformat --- mmv1/products/metastore/api.yaml | 227 ++++++++++++------------- mmv1/products/metastore/terraform.yaml | 22 +-- 2 files changed, 117 insertions(+), 132 deletions(-) diff --git a/mmv1/products/metastore/api.yaml b/mmv1/products/metastore/api.yaml index 746b3060cd75..739045b0c2c4 100644 --- a/mmv1/products/metastore/api.yaml +++ b/mmv1/products/metastore/api.yaml @@ -11,8 +11,7 @@ # See the License for the specific language governing permissions and # limitations under the License. ---- -!ruby/object:Api::Product +--- !ruby/object:Api::Product name: DataprocMetastore versions: - !ruby/object:Api::Product::Version @@ -20,7 +19,7 @@ versions: base_url: https://metastore.googleapis.com/v1beta/ - !ruby/object:Api::Product::Version name: ga - base_url: https://metastore.googleapis.com/v1/ + base_url: https://metastore.googleapis.com/v1/ scopes: - https://www.googleapis.com/auth/cloud-identity apis_required: @@ -29,9 +28,9 @@ apis_required: url: https://console.cloud.google.com/apis/library/metastore.googleapis.com objects: - !ruby/object:Api::Resource - name: "Service" + name: 'Service' base_url: "projects/{{project}}/locations/{{location}}/services" - create_url: "projects/{{project}}/locations/{{location}}/services?serviceId={{service_id}}" + create_url: 'projects/{{project}}/locations/{{location}}/services?serviceId={{service_id}}' self_link: "projects/{{project}}/locations/{{location}}/services/{{service_id}}" update_verb: :PATCH update_mask: true @@ -39,40 +38,36 @@ objects: A managed metastore service that serves metadata queries. references: !ruby/object:Api::Resource::ReferenceLinks guides: - "Official Documentation": "https://cloud.google.com/dataproc-metastore/docs/overview" - api: "https://cloud.google.com/dataproc-metastore/docs/reference/rest/v1/projects.locations.services" + 'Official Documentation': 'https://cloud.google.com/dataproc-metastore/docs/overview' + api: 'https://cloud.google.com/dataproc-metastore/docs/reference/rest/v1/projects.locations.services' iam_policy: !ruby/object:Api::Resource::IamPolicy parent_resource_attribute: service_id exclude: false - method_name_separator: ":" - import_format: - [ - "projects/{{project}}/locations/{{location}}/services/{{service_id}}", - "{{service_id}}", - ] + method_name_separator: ':' + import_format: ["projects/{{project}}/locations/{{location}}/services/{{service_id}}", "{{service_id}}"] async: !ruby/object:Api::OpAsync operation: !ruby/object:Api::OpAsync::Operation - path: "name" - base_url: "{{op_id}}" + path: 'name' + base_url: '{{op_id}}' wait_ms: 1000 timeouts: !ruby/object:Api::Timeouts insert_minutes: 60 update_minutes: 60 delete_minutes: 60 result: !ruby/object:Api::OpAsync::Result - path: "response" + path: 'response' status: !ruby/object:Api::OpAsync::Status - path: "done" + path: 'done' complete: true allowed: - true - false error: !ruby/object:Api::OpAsync::Error - path: "error" - message: "message" + path: 'error' + message: 'message' parameters: - !ruby/object:Api::Type::String - name: "serviceId" + name: 'serviceId' required: true input: true url_param_only: true @@ -81,7 +76,7 @@ objects: and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 63 characters. - !ruby/object:Api::Type::String - name: "location" + name: 'location' url_param_only: true input: true default_value: global @@ -90,70 +85,70 @@ objects: The default value is `global`. properties: - !ruby/object:Api::Type::String - name: "name" + name: 'name' output: true description: | The relative resource name of the metastore service. - !ruby/object:Api::Type::KeyValuePairs - name: "labels" - description: "User-defined labels for the metastore service." + name: 'labels' + description: 'User-defined labels for the metastore service.' # This is an x-product resource reference. - !ruby/object:Api::Type::String - name: "network" + name: 'network' input: true description: | The relative resource name of the VPC network on which the instance can be accessed. It is specified in the following form: "projects/{projectNumber}/global/networks/{network_id}". - !ruby/object:Api::Type::String - name: "endpointUri" + name: 'endpointUri' output: true description: | The URI of the endpoint used to access the metastore service. - !ruby/object:Api::Type::Integer - name: "port" + name: 'port' description: | The TCP port at which the metastore service is reached. Default: 9083. - !ruby/object:Api::Type::String - name: "state" + name: 'state' output: true description: | The current state of the metastore service. - !ruby/object:Api::Type::String - name: "stateMessage" + name: 'stateMessage' output: true description: | Additional information about the current state of the metastore service, if available. - !ruby/object:Api::Type::String - name: "artifactGcsUri" + name: 'artifactGcsUri' output: true description: | A Cloud Storage URI (starting with gs://) that specifies where artifacts related to the metastore service are stored. - !ruby/object:Api::Type::Enum - name: "tier" + name: 'tier' description: | The tier of the service. values: - :DEVELOPER - :ENTERPRISE - !ruby/object:Api::Type::NestedObject - name: "maintenanceWindow" + name: 'maintenanceWindow' description: | The one hour maintenance window of the metastore service. This specifies when the service can be restarted for maintenance purposes in UTC time. Maintenance window is not needed for services with the `SPANNER` database type. properties: - !ruby/object:Api::Type::Integer - name: "hourOfDay" + name: 'hourOfDay' description: | The hour of day (0-23) when the window starts. required: true - !ruby/object:Api::Type::Enum - name: "dayOfWeek" - description: | - The day of week, when the window starts. - required: true - values: + name: 'dayOfWeek' + description: | + The day of week, when the window starts. + required: true + values: - :MONDAY - :TUESDAY - :WEDNESDAY @@ -162,25 +157,25 @@ objects: - :SATURDAY - :SUNDAY - !ruby/object:Api::Type::NestedObject - name: "encryptionConfig" + name: 'encryptionConfig' description: | Information used to configure the Dataproc Metastore service to encrypt customer data at rest. properties: - !ruby/object:Api::Type::String - name: "kmsKey" + name: 'kmsKey' description: | The fully qualified customer provided Cloud KMS key name to use for customer data encryption. Use the following format: `projects/([^/]+)/locations/([^/]+)/keyRings/([^/]+)/cryptoKeys/([^/]+)` required: true input: true - !ruby/object:Api::Type::NestedObject - name: "hiveMetastoreConfig" + name: 'hiveMetastoreConfig' description: | - Configuration information specific to running Hive metastore software as the metastore service. + Configuration information specific to running Hive metastore software as the metastore service. properties: - !ruby/object:Api::Type::Enum - name: "endpointProtocol" + name: 'endpointProtocol' min_version: beta input: true default_value: :THRIFT @@ -188,99 +183,99 @@ objects: The protocol to use for the metastore service endpoint. If unspecified, defaults to `THRIFT`. values: - :THRIFT - - :GRPC + - :GRPC - !ruby/object:Api::Type::String - name: "version" + name: 'version' input: true required: true description: | The Hive metastore schema version. - !ruby/object:Api::Type::KeyValuePairs - name: "configOverrides" + name: 'configOverrides' description: | A mapping of Hive metastore configuration key-value pairs to apply to the Hive metastore (configured in hive-site.xml). The mappings override system defaults (some keys cannot be overridden) - !ruby/object:Api::Type::NestedObject - name: "kerberosConfig" + name: 'kerberosConfig' description: | Information used to configure the Hive metastore service as a service principal in a Kerberos realm. properties: - !ruby/object:Api::Type::NestedObject - name: "keytab" + name: 'keytab' required: true description: | A Kerberos keytab file that can be used to authenticate a service principal with a Kerberos Key Distribution Center (KDC). properties: - !ruby/object:Api::Type::String required: true - name: "cloudSecret" + name: 'cloudSecret' description: | The relative resource name of a Secret Manager secret version, in the following form: "projects/{projectNumber}/secrets/{secret_id}/versions/{version_id}". - !ruby/object:Api::Type::String - name: "principal" + name: 'principal' required: true description: | A Kerberos principal that exists in the both the keytab the KDC to authenticate as. A typical principal is of the form "primary/instance@REALM", but there is no exact format. - !ruby/object:Api::Type::String - name: "krb5ConfigGcsUri" + name: 'krb5ConfigGcsUri' required: true description: | A Cloud Storage URI that specifies the path to a krb5.conf file. It is of the form gs://{bucket_name}/path/to/krb5.conf, although the file does not need to be named krb5.conf explicitly. - !ruby/object:Api::Type::Map - name: "auxiliaryVersions" + name: 'auxiliaryVersions' min_version: beta description: | - A mapping of Hive metastore version to the auxiliary version configuration. - When specified, a secondary Hive metastore service is created along with the primary service. - All auxiliary versions must be less than the service's primary version. - The key is the auxiliary service name and it must match the regular expression a-z?. - This means that the first character must be a lowercase letter, and all the following characters must be hyphens, lowercase letters, or digits, except the last character, which cannot be a hyphen. - key_name: "key" + A mapping of Hive metastore version to the auxiliary version configuration. + When specified, a secondary Hive metastore service is created along with the primary service. + All auxiliary versions must be less than the service's primary version. + The key is the auxiliary service name and it must match the regular expression a-z?. + This means that the first character must be a lowercase letter, and all the following characters must be hyphens, lowercase letters, or digits, except the last character, which cannot be a hyphen. + key_name: 'key' key_description: | The auxiliary service name value_type: !ruby/object:Api::Type::NestedObject properties: - !ruby/object:Api::Type::String - name: "version" + name: 'version' required: true description: | The Hive metastore version of the auxiliary service. It must be less than the primary Hive metastore service's version. - !ruby/object:Api::Type::KeyValuePairs - name: "configOverrides" + name: 'configOverrides' description: | A mapping of Hive metastore configuration key-value pairs to apply to the auxiliary Hive metastore (configured in hive-site.xml) in addition to the primary version's overrides. If keys are present in both the auxiliary version's overrides and the primary version's overrides, the value from the auxiliary version's overrides takes precedence. - !ruby/object:Api::Type::NestedObject - name: "networkConfig" + name: 'networkConfig' input: true description: | The configuration specifying the network settings for the Dataproc Metastore service. properties: - - !ruby/object:Api::Type::Array - name: "consumers" - description: | - The consumer-side network configuration for the Dataproc Metastore instance. - required: true - input: true - item_type: !ruby/object:Api::Type::NestedObject - properties: - - !ruby/object:Api::Type::String - name: "endpointUri" - description: | - The URI of the endpoint used to access the metastore service. - output: true - - !ruby/object:Api::Type::String - name: "subnetwork" - required: true - description: | - The subnetwork of the customer project from which an IP address is reserved and used as the Dataproc Metastore service's endpoint. - It is accessible to hosts in the subnet and to all hosts in a subnet in the same region and same network. - There must be at least one IP address available in the subnet's primary range. The subnet is specified in the following form: - `projects/{projectNumber}/regions/{region_id}/subnetworks/{subnetwork_id} + - !ruby/object:Api::Type::Array + name: 'consumers' + description: | + The consumer-side network configuration for the Dataproc Metastore instance. + required: true + input: true + item_type: !ruby/object:Api::Type::NestedObject + properties: + - !ruby/object:Api::Type::String + name: 'endpointUri' + description: | + The URI of the endpoint used to access the metastore service. + output: true + - !ruby/object:Api::Type::String + name: 'subnetwork' + required: true + description: | + The subnetwork of the customer project from which an IP address is reserved and used as the Dataproc Metastore service's endpoint. + It is accessible to hosts in the subnet and to all hosts in a subnet in the same region and same network. + There must be at least one IP address available in the subnet's primary range. The subnet is specified in the following form: + `projects/{projectNumber}/regions/{region_id}/subnetworks/{subnetwork_id} - !ruby/object:Api::Type::Enum - name: "databaseType" + name: 'databaseType' input: true default_value: :MYSQL description: | @@ -289,7 +284,7 @@ objects: - :MYSQL - :SPANNER - !ruby/object:Api::Type::Enum - name: "releaseChannel" + name: 'releaseChannel' input: true default_value: :STABLE description: | @@ -298,27 +293,27 @@ objects: - :CANARY - :STABLE - !ruby/object:Api::Type::String - name: "uid" + name: 'uid' output: true description: | The globally unique resource identifier of the metastore service. - !ruby/object:Api::Type::NestedObject - name: "metadataIntegration" + name: 'metadataIntegration' min_version: beta description: | The setting that defines how metastore metadata should be integrated with external services and systems. properties: - !ruby/object:Api::Type::NestedObject - name: "dataCatalogConfig" + name: 'dataCatalogConfig' required: true description: | The integration config for the Data Catalog service. properties: - !ruby/object:Api::Type::Boolean - name: "enabled" + name: 'enabled' required: true description: | - Defines whether the metastore metadata should be synced to Data Catalog. The default value is to disable syncing metastore metadata to Data Catalog. + Defines whether the metastore metadata should be synced to Data Catalog. The default value is to disable syncing metastore metadata to Data Catalog. - !ruby/object:Api::Type::NestedObject name: "telemetryConfig" description: | @@ -333,11 +328,11 @@ objects: - :LEGACY - :JSON - !ruby/object:Api::Resource - name: "Federation" + name: 'Federation' min_version: beta base_url: "projects/{{project}}/locations/{{location}}/federations" - create_url: "projects/{{project}}/locations/{{location}}/federations?federationId={{federation_id}}" - self_link: "projects/{{project}}/locations/{{location}}/federations/{{federation_id}}" + create_url: 'projects/{{project}}/locations/{{location}}/federations?federationId={{federation_id}}' + self_link: "projects/{{project}}/locations/{{location}}/federations/{{federation_id}}" update_verb: :PATCH update_mask: true description: | @@ -345,31 +340,27 @@ objects: iam_policy: !ruby/object:Api::Resource::IamPolicy parent_resource_attribute: federation_id exclude: false - method_name_separator: ":" - import_format: - [ - "projects/{{project}}/locations/{{location}}/federations/{{federation_id}}", - "{{federation_id}}", - ] + method_name_separator: ':' + import_format: ["projects/{{project}}/locations/{{location}}/federations/{{federation_id}}", "{{federation_id}}"] async: !ruby/object:Api::OpAsync operation: !ruby/object:Api::OpAsync::Operation - path: "name" - base_url: "{{op_id}}" + path: 'name' + base_url: '{{op_id}}' wait_ms: 1000 result: !ruby/object:Api::OpAsync::Result - path: "response" + path: 'response' status: !ruby/object:Api::OpAsync::Status - path: "done" + path: 'done' complete: true allowed: - true - false error: !ruby/object:Api::OpAsync::Error - path: "error" - message: "message" + path: 'error' + message: 'message' parameters: - !ruby/object:Api::Type::String - name: "federationId" + name: 'federationId' required: true input: true url_param_only: true @@ -378,63 +369,63 @@ objects: and hyphens (-). Cannot begin or end with underscore or hyphen. Must consist of between 3 and 63 characters. - !ruby/object:Api::Type::String - name: "location" + name: 'location' url_param_only: true input: true description: | The location where the metastore federation should reside. properties: - !ruby/object:Api::Type::String - name: "name" + name: 'name' output: true description: | The relative resource name of the metastore federation. - !ruby/object:Api::Type::KeyValuePairs - name: "labels" - description: "User-defined labels for the metastore federation." + name: 'labels' + description: 'User-defined labels for the metastore federation.' - !ruby/object:Api::Type::String - name: "endpointUri" + name: 'endpointUri' output: true description: | The URI of the endpoint used to access the metastore federation. - !ruby/object:Api::Type::String - name: "state" + name: 'state' output: true description: | The current state of the metastore federation. - !ruby/object:Api::Type::String - name: "stateMessage" + name: 'stateMessage' output: true description: | Additional information about the current state of the metastore federation, if available. - !ruby/object:Api::Type::String - name: "uid" + name: 'uid' output: true description: | The globally unique resource identifier of the metastore federation. - !ruby/object:Api::Type::String - name: "version" + name: 'version' input: true required: true description: | The Apache Hive metastore version of the federation. All backend metastore versions must be compatible with the federation version. - !ruby/object:Api::Type::Map - name: "backendMetastores" + name: 'backendMetastores' description: | A map from BackendMetastore rank to BackendMetastores from which the federation service serves metadata at query time. The map key represents the order in which BackendMetastores should be evaluated to resolve database names at query time and should be greater than or equal to zero. A BackendMetastore with a lower number will be evaluated before a BackendMetastore with a higher number. required: true - key_name: "rank" + key_name: 'rank' key_description: | represents the order in which BackendMetastores should be evaluated to resolve database names at query time and should be greater than or equal to zero. value_type: !ruby/object:Api::Type::NestedObject properties: - !ruby/object:Api::Type::String - name: "name" + name: 'name' required: true description: | The relative resource name of the metastore that is being federated. The formats of the relative resource names for the currently supported metastores are listed below: Dataplex: projects/{projectId}/locations/{location}/lakes/{lake_id} BigQuery: projects/{projectId} Dataproc Metastore: projects/{projectId}/locations/{location}/services/{serviceId} - !ruby/object:Api::Type::Enum - name: "metastoreType" + name: 'metastoreType' required: true description: | The type of the backend metastore. diff --git a/mmv1/products/metastore/terraform.yaml b/mmv1/products/metastore/terraform.yaml index 0dfa9b233585..02da46b748c7 100644 --- a/mmv1/products/metastore/terraform.yaml +++ b/mmv1/products/metastore/terraform.yaml @@ -11,13 +11,11 @@ # See the License for the specific language governing permissions and # limitations under the License. ---- -!ruby/object:Provider::Terraform::Config +--- !ruby/object:Provider::Terraform::Config overrides: !ruby/object:Overrides::ResourceOverrides Service: !ruby/object:Overrides::Terraform::ResourceOverride autogen_async: true - import_format: - ["projects/{{project}}/locations/{{location}}/services/{{service_id}}"] + import_format: ["projects/{{project}}/locations/{{location}}/services/{{service_id}}"] examples: - !ruby/object:Provider::Terraform::Examples name: "dataproc_metastore_service_basic" @@ -75,7 +73,7 @@ overrides: !ruby/object:Overrides::ResourceOverrides skip_docs: true primary_resource_id: "telemetry" vars: - metastore_service_name: "telemetry" + metastore_service_name: "telemetry" properties: network: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true @@ -83,18 +81,14 @@ overrides: !ruby/object:Overrides::ResourceOverrides default_from_api: true tier: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true - telemetryConfig: !ruby/object:Overrides::Terraform::PropertyOverride - default_from_api: true - hiveMetastoreConfig.configOverrides: - !ruby/object:Overrides::Terraform::PropertyOverride + hiveMetastoreConfig.configOverrides: !ruby/object:Overrides::Terraform::PropertyOverride default_from_api: true diff_suppress_func: "dataprocMetastoreServiceOverrideSuppress" + telemetryConfig: !ruby/object:Overrides::Terraform::PropertyOverride + default_from_api: true Federation: !ruby/object:Overrides::Terraform::ResourceOverride autogen_async: true - import_format: - [ - "projects/{{project}}/locations/{{location}}/federations/{{federation_id}}", - ] + import_format: ["projects/{{project}}/locations/{{location}}/federations/{{federation_id}}"] examples: - !ruby/object:Provider::Terraform::Examples name: "dataproc_metastore_federation_basic" @@ -109,4 +103,4 @@ overrides: !ruby/object:Overrides::ResourceOverrides min_version: beta primary_resource_name: 'fmt.Sprintf("tf-test-metastore-fed-bq%s", context["random_suffix"])' vars: - metastore_federation_name: "fed-2" + metastore_federation_name: "fed-2"