From 44c10e17767135b7a5c9a5e22b82260be75459b1 Mon Sep 17 00:00:00 2001 From: "gcf-owl-bot[bot]" <78513119+gcf-owl-bot[bot]@users.noreply.github.com> Date: Mon, 24 Jan 2022 11:01:16 -0600 Subject: [PATCH] feat: Add task details and orchestration result details (#32) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit * feat!: Consolidate task details into service API and add orchestration result details BREAKING CHANGE: This change relocates some task proto definitions and updates message fields, necessitating updates to imports. PiperOrigin-RevId: 423360094 Source-Link: https://github.com/googleapis/googleapis/commit/c9a482e23c28251459afdc4b3b63cbf2fb1f163b Source-Link: https://github.com/googleapis/googleapis-gen/commit/0c2ce73c148389e4382c11eb7abfd0b26973cbb7 Copy-Tag: eyJwIjoiLmdpdGh1Yi8uT3dsQm90LnlhbWwiLCJoIjoiMGMyY2U3M2MxNDgzODllNDM4MmMxMWViN2FiZmQwYjI2OTczY2JiNyJ9 * 🦉 Updates from OwlBot See https://github.com/googleapis/repo-automation-bots/blob/main/packages/owl-bot/README.md Co-authored-by: Owl Bot Co-authored-by: Anthonios Partheniou --- google/cloud/bigquery_migration/__init__.py | 36 +++ .../bigquery_migration_v2alpha/__init__.py | 20 ++ .../types/__init__.py | 24 ++ .../types/assessment_task.py | 62 +++++ .../types/migration_entities.py | 71 ++++- .../types/translation_task.py | 244 ++++++++++++++++++ .../test_migration_service.py | 2 + 7 files changed, 455 insertions(+), 4 deletions(-) create mode 100644 google/cloud/bigquery_migration_v2alpha/types/assessment_task.py create mode 100644 google/cloud/bigquery_migration_v2alpha/types/translation_task.py diff --git a/google/cloud/bigquery_migration/__init__.py b/google/cloud/bigquery_migration/__init__.py index 80e5483..a7d2621 100644 --- a/google/cloud/bigquery_migration/__init__.py +++ b/google/cloud/bigquery_migration/__init__.py @@ -21,12 +21,21 @@ MigrationServiceAsyncClient, ) +from google.cloud.bigquery_migration_v2alpha.types.assessment_task import ( + AssessmentOrchestrationResultDetails, +) +from google.cloud.bigquery_migration_v2alpha.types.assessment_task import ( + AssessmentTaskDetails, +) from google.cloud.bigquery_migration_v2alpha.types.migration_entities import ( MigrationSubtask, ) from google.cloud.bigquery_migration_v2alpha.types.migration_entities import ( MigrationTask, ) +from google.cloud.bigquery_migration_v2alpha.types.migration_entities import ( + MigrationTaskOrchestrationResult, +) from google.cloud.bigquery_migration_v2alpha.types.migration_entities import ( MigrationWorkflow, ) @@ -70,12 +79,32 @@ from google.cloud.bigquery_migration_v2alpha.types.migration_service import ( StartMigrationWorkflowRequest, ) +from google.cloud.bigquery_migration_v2alpha.types.translation_task import BteqOptions +from google.cloud.bigquery_migration_v2alpha.types.translation_task import ( + DatasetReference, +) +from google.cloud.bigquery_migration_v2alpha.types.translation_task import Filter +from google.cloud.bigquery_migration_v2alpha.types.translation_task import ( + IdentifierSettings, +) +from google.cloud.bigquery_migration_v2alpha.types.translation_task import ( + TeradataOptions, +) +from google.cloud.bigquery_migration_v2alpha.types.translation_task import ( + TranslationFileMapping, +) +from google.cloud.bigquery_migration_v2alpha.types.translation_task import ( + TranslationTaskDetails, +) __all__ = ( "MigrationServiceClient", "MigrationServiceAsyncClient", + "AssessmentOrchestrationResultDetails", + "AssessmentTaskDetails", "MigrationSubtask", "MigrationTask", + "MigrationTaskOrchestrationResult", "MigrationWorkflow", "ErrorDetail", "ErrorLocation", @@ -93,4 +122,11 @@ "ListMigrationWorkflowsRequest", "ListMigrationWorkflowsResponse", "StartMigrationWorkflowRequest", + "BteqOptions", + "DatasetReference", + "Filter", + "IdentifierSettings", + "TeradataOptions", + "TranslationFileMapping", + "TranslationTaskDetails", ) diff --git a/google/cloud/bigquery_migration_v2alpha/__init__.py b/google/cloud/bigquery_migration_v2alpha/__init__.py index 4b5dcfa..d99fd78 100644 --- a/google/cloud/bigquery_migration_v2alpha/__init__.py +++ b/google/cloud/bigquery_migration_v2alpha/__init__.py @@ -17,8 +17,11 @@ from .services.migration_service import MigrationServiceClient from .services.migration_service import MigrationServiceAsyncClient +from .types.assessment_task import AssessmentOrchestrationResultDetails +from .types.assessment_task import AssessmentTaskDetails from .types.migration_entities import MigrationSubtask from .types.migration_entities import MigrationTask +from .types.migration_entities import MigrationTaskOrchestrationResult from .types.migration_entities import MigrationWorkflow from .types.migration_error_details import ErrorDetail from .types.migration_error_details import ErrorLocation @@ -36,15 +39,28 @@ from .types.migration_service import ListMigrationWorkflowsRequest from .types.migration_service import ListMigrationWorkflowsResponse from .types.migration_service import StartMigrationWorkflowRequest +from .types.translation_task import BteqOptions +from .types.translation_task import DatasetReference +from .types.translation_task import Filter +from .types.translation_task import IdentifierSettings +from .types.translation_task import TeradataOptions +from .types.translation_task import TranslationFileMapping +from .types.translation_task import TranslationTaskDetails __all__ = ( "MigrationServiceAsyncClient", + "AssessmentOrchestrationResultDetails", + "AssessmentTaskDetails", + "BteqOptions", "CreateMigrationWorkflowRequest", + "DatasetReference", "DeleteMigrationWorkflowRequest", "ErrorDetail", "ErrorLocation", + "Filter", "GetMigrationSubtaskRequest", "GetMigrationWorkflowRequest", + "IdentifierSettings", "ListMigrationSubtasksRequest", "ListMigrationSubtasksResponse", "ListMigrationWorkflowsRequest", @@ -52,11 +68,15 @@ "MigrationServiceClient", "MigrationSubtask", "MigrationTask", + "MigrationTaskOrchestrationResult", "MigrationWorkflow", "Point", "ResourceErrorDetail", "StartMigrationWorkflowRequest", + "TeradataOptions", "TimeInterval", "TimeSeries", + "TranslationFileMapping", + "TranslationTaskDetails", "TypedValue", ) diff --git a/google/cloud/bigquery_migration_v2alpha/types/__init__.py b/google/cloud/bigquery_migration_v2alpha/types/__init__.py index 8a7d29f..8763134 100644 --- a/google/cloud/bigquery_migration_v2alpha/types/__init__.py +++ b/google/cloud/bigquery_migration_v2alpha/types/__init__.py @@ -13,9 +13,14 @@ # See the License for the specific language governing permissions and # limitations under the License. # +from .assessment_task import ( + AssessmentOrchestrationResultDetails, + AssessmentTaskDetails, +) from .migration_entities import ( MigrationSubtask, MigrationTask, + MigrationTaskOrchestrationResult, MigrationWorkflow, ) from .migration_error_details import ( @@ -40,10 +45,22 @@ ListMigrationWorkflowsResponse, StartMigrationWorkflowRequest, ) +from .translation_task import ( + BteqOptions, + DatasetReference, + Filter, + IdentifierSettings, + TeradataOptions, + TranslationFileMapping, + TranslationTaskDetails, +) __all__ = ( + "AssessmentOrchestrationResultDetails", + "AssessmentTaskDetails", "MigrationSubtask", "MigrationTask", + "MigrationTaskOrchestrationResult", "MigrationWorkflow", "ErrorDetail", "ErrorLocation", @@ -61,4 +78,11 @@ "ListMigrationWorkflowsRequest", "ListMigrationWorkflowsResponse", "StartMigrationWorkflowRequest", + "BteqOptions", + "DatasetReference", + "Filter", + "IdentifierSettings", + "TeradataOptions", + "TranslationFileMapping", + "TranslationTaskDetails", ) diff --git a/google/cloud/bigquery_migration_v2alpha/types/assessment_task.py b/google/cloud/bigquery_migration_v2alpha/types/assessment_task.py new file mode 100644 index 0000000..719f44d --- /dev/null +++ b/google/cloud/bigquery_migration_v2alpha/types/assessment_task.py @@ -0,0 +1,62 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + + +__protobuf__ = proto.module( + package="google.cloud.bigquery.migration.v2alpha", + manifest={"AssessmentTaskDetails", "AssessmentOrchestrationResultDetails",}, +) + + +class AssessmentTaskDetails(proto.Message): + r"""Assessment task config. + + Attributes: + input_path (str): + Required. The Cloud Storage path for + assessment input files. + output_dataset (str): + Required. The BigQuery dataset for output. + querylogs_path (str): + Optional. An optional Cloud Storage path to + write the query logs (which is then used as an + input path on the translation task) + data_source (str): + Required. The data source or data warehouse + type (eg: TERADATA/REDSHIFT) from which the + input data is extracted. + """ + + input_path = proto.Field(proto.STRING, number=1,) + output_dataset = proto.Field(proto.STRING, number=2,) + querylogs_path = proto.Field(proto.STRING, number=3,) + data_source = proto.Field(proto.STRING, number=4,) + + +class AssessmentOrchestrationResultDetails(proto.Message): + r"""Details for an assessment task orchestration result. + + Attributes: + output_tables_schema_version (str): + Optional. The version used for the output + table schemas. + """ + + output_tables_schema_version = proto.Field(proto.STRING, number=1,) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_migration_v2alpha/types/migration_entities.py b/google/cloud/bigquery_migration_v2alpha/types/migration_entities.py index 1e99224..55daf49 100644 --- a/google/cloud/bigquery_migration_v2alpha/types/migration_entities.py +++ b/google/cloud/bigquery_migration_v2alpha/types/migration_entities.py @@ -15,8 +15,10 @@ # import proto # type: ignore +from google.cloud.bigquery_migration_v2alpha.types import assessment_task from google.cloud.bigquery_migration_v2alpha.types import migration_error_details from google.cloud.bigquery_migration_v2alpha.types import migration_metrics +from google.cloud.bigquery_migration_v2alpha.types import translation_task from google.protobuf import any_pb2 # type: ignore from google.protobuf import timestamp_pb2 # type: ignore from google.rpc import error_details_pb2 # type: ignore @@ -24,7 +26,12 @@ __protobuf__ = proto.module( package="google.cloud.bigquery.migration.v2alpha", - manifest={"MigrationWorkflow", "MigrationTask", "MigrationSubtask",}, + manifest={ + "MigrationWorkflow", + "MigrationTask", + "MigrationSubtask", + "MigrationTaskOrchestrationResult", + }, ) @@ -79,7 +86,23 @@ class MigrationTask(proto.Message): r"""A single task for a migration which has details about the configuration of the task. + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + Attributes: + assessment_task_details (google.cloud.bigquery_migration_v2alpha.types.AssessmentTaskDetails): + Task configuration for Assessment. + + This field is a member of `oneof`_ ``task_details``. + translation_task_details (google.cloud.bigquery_migration_v2alpha.types.TranslationTaskDetails): + Task configuration for Batch/Offline SQL + Translation. + + This field is a member of `oneof`_ ``task_details``. id (str): Output only. Immutable. The unique identifier for the migration task. The ID is server- @@ -88,9 +111,9 @@ class MigrationTask(proto.Message): The type of the task. This must be a supported task type. details (google.protobuf.any_pb2.Any): - The details of the task. The type URL must be - one of the supported task details messages and - correspond to the Task's type. + DEPRECATED! Use one of the task_details below. The details + of the task. The type URL must be one of the supported task + details messages and correspond to the Task's type. state (google.cloud.bigquery_migration_v2alpha.types.MigrationTask.State): Output only. The current state of the task. processing_error (google.rpc.error_details_pb2.ErrorInfo): @@ -100,6 +123,9 @@ class MigrationTask(proto.Message): Time when the task was created. last_update_time (google.protobuf.timestamp_pb2.Timestamp): Time when the task was last updated. + orchestration_result (google.cloud.bigquery_migration_v2alpha.types.MigrationTaskOrchestrationResult): + Output only. Additional information about the + orchestration. """ class State(proto.Enum): @@ -112,6 +138,18 @@ class State(proto.Enum): SUCCEEDED = 5 FAILED = 6 + assessment_task_details = proto.Field( + proto.MESSAGE, + number=12, + oneof="task_details", + message=assessment_task.AssessmentTaskDetails, + ) + translation_task_details = proto.Field( + proto.MESSAGE, + number=13, + oneof="task_details", + message=translation_task.TranslationTaskDetails, + ) id = proto.Field(proto.STRING, number=1,) type_ = proto.Field(proto.STRING, number=2,) details = proto.Field(proto.MESSAGE, number=3, message=any_pb2.Any,) @@ -123,6 +161,9 @@ class State(proto.Enum): last_update_time = proto.Field( proto.MESSAGE, number=7, message=timestamp_pb2.Timestamp, ) + orchestration_result = proto.Field( + proto.MESSAGE, number=10, message="MigrationTaskOrchestrationResult", + ) class MigrationSubtask(proto.Message): @@ -202,4 +243,26 @@ class State(proto.Enum): ) +class MigrationTaskOrchestrationResult(proto.Message): + r"""Additional information from the orchestrator when it is done + with the task orchestration. + + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + assessment_details (google.cloud.bigquery_migration_v2alpha.types.AssessmentOrchestrationResultDetails): + Details specific to assessment task types. + + This field is a member of `oneof`_ ``details``. + """ + + assessment_details = proto.Field( + proto.MESSAGE, + number=1, + oneof="details", + message=assessment_task.AssessmentOrchestrationResultDetails, + ) + + __all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/google/cloud/bigquery_migration_v2alpha/types/translation_task.py b/google/cloud/bigquery_migration_v2alpha/types/translation_task.py new file mode 100644 index 0000000..e2be946 --- /dev/null +++ b/google/cloud/bigquery_migration_v2alpha/types/translation_task.py @@ -0,0 +1,244 @@ +# -*- coding: utf-8 -*- +# Copyright 2020 Google LLC +# +# Licensed under the Apache License, Version 2.0 (the "License"); +# you may not use this file except in compliance with the License. +# You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# +import proto # type: ignore + + +__protobuf__ = proto.module( + package="google.cloud.bigquery.migration.v2alpha", + manifest={ + "TranslationFileMapping", + "TranslationTaskDetails", + "Filter", + "IdentifierSettings", + "TeradataOptions", + "BteqOptions", + "DatasetReference", + }, +) + + +class TranslationFileMapping(proto.Message): + r"""Mapping between an input and output file to be translated in + a subtask. + + Attributes: + input_path (str): + The Cloud Storage path for a file to + translation in a subtask. + output_path (str): + The Cloud Storage path to write back the + corresponding input file to. + """ + + input_path = proto.Field(proto.STRING, number=1,) + output_path = proto.Field(proto.STRING, number=2,) + + +class TranslationTaskDetails(proto.Message): + r"""The translation task config to capture necessary settings for + a translation task and subtask. + + This message has `oneof`_ fields (mutually exclusive fields). + For each oneof, at most one member field can be set at the same time. + Setting any member of the oneof automatically clears all other + members. + + .. _oneof: https://proto-plus-python.readthedocs.io/en/stable/fields.html#oneofs-mutually-exclusive-fields + + Attributes: + teradata_options (google.cloud.bigquery_migration_v2alpha.types.TeradataOptions): + The Teradata SQL specific settings for the + translation task. + + This field is a member of `oneof`_ ``language_options``. + bteq_options (google.cloud.bigquery_migration_v2alpha.types.BteqOptions): + The BTEQ specific settings for the + translation task. + + This field is a member of `oneof`_ ``language_options``. + input_path (str): + The Cloud Storage path for translation input + files. + output_path (str): + The Cloud Storage path for translation output + files. + file_paths (Sequence[google.cloud.bigquery_migration_v2alpha.types.TranslationFileMapping]): + Cloud Storage files to be processed for + translation. + schema_path (str): + The Cloud Storage path to DDL files as table + schema to assist semantic translation. + file_encoding (google.cloud.bigquery_migration_v2alpha.types.TranslationTaskDetails.FileEncoding): + The file encoding type. + identifier_settings (google.cloud.bigquery_migration_v2alpha.types.IdentifierSettings): + The settings for SQL identifiers. + special_token_map (Sequence[google.cloud.bigquery_migration_v2alpha.types.TranslationTaskDetails.SpecialTokenMapEntry]): + The map capturing special tokens to be + replaced during translation. The key is special + token in string. The value is the token data + type. This is used to translate SQL query + template which contains special token as place + holder. The special token makes a query invalid + to parse. This map will be applied to annotate + those special token with types to let parser + understand how to parse them into proper + structure with type information. + filter (google.cloud.bigquery_migration_v2alpha.types.Filter): + The filter applied to translation details. + translation_exception_table (str): + Specifies the exact name of the bigquery + table ("dataset.table") to be used for surfacing + raw translation errors. If the table does not + exist, we will create it. If it already exists + and the schema is the same, we will re-use. If + the table exists and the schema is different, we + will throw an error. + """ + + class FileEncoding(proto.Enum): + r"""The file encoding types.""" + FILE_ENCODING_UNSPECIFIED = 0 + UTF_8 = 1 + ISO_8859_1 = 2 + US_ASCII = 3 + UTF_16 = 4 + UTF_16LE = 5 + UTF_16BE = 6 + + class TokenType(proto.Enum): + r"""The special token data type.""" + TOKEN_TYPE_UNSPECIFIED = 0 + STRING = 1 + INT64 = 2 + NUMERIC = 3 + BOOL = 4 + FLOAT64 = 5 + DATE = 6 + TIMESTAMP = 7 + + teradata_options = proto.Field( + proto.MESSAGE, number=10, oneof="language_options", message="TeradataOptions", + ) + bteq_options = proto.Field( + proto.MESSAGE, number=11, oneof="language_options", message="BteqOptions", + ) + input_path = proto.Field(proto.STRING, number=1,) + output_path = proto.Field(proto.STRING, number=2,) + file_paths = proto.RepeatedField( + proto.MESSAGE, number=12, message="TranslationFileMapping", + ) + schema_path = proto.Field(proto.STRING, number=3,) + file_encoding = proto.Field(proto.ENUM, number=4, enum=FileEncoding,) + identifier_settings = proto.Field( + proto.MESSAGE, number=5, message="IdentifierSettings", + ) + special_token_map = proto.MapField( + proto.STRING, proto.ENUM, number=6, enum=TokenType, + ) + filter = proto.Field(proto.MESSAGE, number=7, message="Filter",) + translation_exception_table = proto.Field(proto.STRING, number=13,) + + +class Filter(proto.Message): + r"""The filter applied to fields of translation details. + + Attributes: + input_file_exclusion_prefixes (Sequence[str]): + The list of prefixes used to exclude + processing for input files. + """ + + input_file_exclusion_prefixes = proto.RepeatedField(proto.STRING, number=1,) + + +class IdentifierSettings(proto.Message): + r"""Settings related to SQL identifiers. + + Attributes: + output_identifier_case (google.cloud.bigquery_migration_v2alpha.types.IdentifierSettings.IdentifierCase): + The setting to control output queries' + identifier case. + identifier_rewrite_mode (google.cloud.bigquery_migration_v2alpha.types.IdentifierSettings.IdentifierRewriteMode): + Specifies the rewrite mode for SQL + identifiers. + """ + + class IdentifierCase(proto.Enum): + r"""The identifier case type.""" + IDENTIFIER_CASE_UNSPECIFIED = 0 + ORIGINAL = 1 + UPPER = 2 + LOWER = 3 + + class IdentifierRewriteMode(proto.Enum): + r"""The SQL identifier rewrite mode.""" + IDENTIFIER_REWRITE_MODE_UNSPECIFIED = 0 + NONE = 1 + REWRITE_ALL = 2 + + output_identifier_case = proto.Field(proto.ENUM, number=1, enum=IdentifierCase,) + identifier_rewrite_mode = proto.Field( + proto.ENUM, number=2, enum=IdentifierRewriteMode, + ) + + +class TeradataOptions(proto.Message): + r"""Teradata SQL specific translation task related settings. + """ + + +class BteqOptions(proto.Message): + r"""BTEQ translation task related settings. + + Attributes: + project_dataset (google.cloud.bigquery_migration_v2alpha.types.DatasetReference): + Specifies the project and dataset in BigQuery + that will be used for external table creation + during the translation. + default_path_uri (str): + The Cloud Storage location to be used as the + default path for files that are not otherwise + specified in the file replacement map. + file_replacement_map (Sequence[google.cloud.bigquery_migration_v2alpha.types.BteqOptions.FileReplacementMapEntry]): + Maps the local paths that are used in BTEQ + scripts (the keys) to the paths in Cloud Storage + that should be used in their stead in the + translation (the value). + """ + + project_dataset = proto.Field(proto.MESSAGE, number=1, message="DatasetReference",) + default_path_uri = proto.Field(proto.STRING, number=2,) + file_replacement_map = proto.MapField(proto.STRING, proto.STRING, number=3,) + + +class DatasetReference(proto.Message): + r"""Reference to a BigQuery dataset. + + Attributes: + dataset_id (str): + A unique ID for this dataset, without the project name. The + ID must contain only letters (a-z, A-Z), numbers (0-9), or + underscores (_). The maximum length is 1,024 characters. + project_id (str): + The ID of the project containing this + dataset. + """ + + dataset_id = proto.Field(proto.STRING, number=1,) + project_id = proto.Field(proto.STRING, number=2,) + + +__all__ = tuple(sorted(__protobuf__.manifest)) diff --git a/tests/unit/gapic/bigquery_migration_v2alpha/test_migration_service.py b/tests/unit/gapic/bigquery_migration_v2alpha/test_migration_service.py index 38c993b..d6d7508 100644 --- a/tests/unit/gapic/bigquery_migration_v2alpha/test_migration_service.py +++ b/tests/unit/gapic/bigquery_migration_v2alpha/test_migration_service.py @@ -41,10 +41,12 @@ from google.cloud.bigquery_migration_v2alpha.services.migration_service import ( transports, ) +from google.cloud.bigquery_migration_v2alpha.types import assessment_task from google.cloud.bigquery_migration_v2alpha.types import migration_entities from google.cloud.bigquery_migration_v2alpha.types import migration_error_details from google.cloud.bigquery_migration_v2alpha.types import migration_metrics from google.cloud.bigquery_migration_v2alpha.types import migration_service +from google.cloud.bigquery_migration_v2alpha.types import translation_task from google.oauth2 import service_account from google.protobuf import any_pb2 # type: ignore from google.protobuf import field_mask_pb2 # type: ignore