From 9fd440edba6d6f45821fea244fa00e2ed1f0f363 Mon Sep 17 00:00:00 2001 From: Kevin Liu Date: Fri, 1 Nov 2024 14:47:44 -0400 Subject: [PATCH] Deprecate for 0.8.0 release (#1269) * remove deprecated functions * remove deprecated properties * fix tests based on deprecated properties * update readme * get_first_property_value * deprecate properties --- mkdocs/docs/configuration.md | 8 +++--- pyiceberg/catalog/__init__.py | 23 ---------------- pyiceberg/catalog/dynamodb.py | 23 ++++------------ pyiceberg/catalog/glue.py | 23 ++++------------ pyiceberg/io/pyarrow.py | 17 ------------ pyiceberg/table/__init__.py | 50 ---------------------------------- tests/catalog/test_dynamodb.py | 27 ------------------ tests/catalog/test_glue.py | 27 +----------------- tests/conftest.py | 8 ------ 9 files changed, 15 insertions(+), 191 deletions(-) diff --git a/mkdocs/docs/configuration.md b/mkdocs/docs/configuration.md index 1269e34a83..2ed58091bb 100644 --- a/mkdocs/docs/configuration.md +++ b/mkdocs/docs/configuration.md @@ -340,8 +340,8 @@ catalog: -!!! warning "Deprecated Properties" - `profile_name`, `region_name`, `botocore_session`, `aws_access_key_id`, `aws_secret_access_key`, `aws_session_token` are deprecated and will be removed in 0.8.0: +!!! warning "Removed Properties" + The properties `profile_name`, `region_name`, `botocore_session`, `aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` were deprecated and removed in 0.8.0 @@ -396,8 +396,8 @@ catalog: -!!! warning "Deprecated Properties" - `profile_name`, `region_name`, `botocore_session`, `aws_access_key_id`, `aws_secret_access_key`, `aws_session_token` are deprecated and will be removed in 0.8.0: +!!! warning "Removed Properties" + The properties `profile_name`, `region_name`, `botocore_session`, `aws_access_key_id`, `aws_secret_access_key`, and `aws_session_token` were deprecated and removed in 0.8.0 diff --git a/pyiceberg/catalog/__init__.py b/pyiceberg/catalog/__init__.py index cc953a8849..2104fcab7a 100644 --- a/pyiceberg/catalog/__init__.py +++ b/pyiceberg/catalog/__init__.py @@ -106,21 +106,6 @@ re.X, ) -DEPRECATED_PROFILE_NAME = "profile_name" -DEPRECATED_REGION = "region_name" -DEPRECATED_BOTOCORE_SESSION = "botocore_session" -DEPRECATED_ACCESS_KEY_ID = "aws_access_key_id" -DEPRECATED_SECRET_ACCESS_KEY = "aws_secret_access_key" -DEPRECATED_SESSION_TOKEN = "aws_session_token" -DEPRECATED_PROPERTY_NAMES = { - DEPRECATED_PROFILE_NAME, - DEPRECATED_REGION, - DEPRECATED_BOTOCORE_SESSION, - DEPRECATED_ACCESS_KEY_ID, - DEPRECATED_SECRET_ACCESS_KEY, - DEPRECATED_SESSION_TOKEN, -} - class CatalogType(Enum): REST = "rest" @@ -794,14 +779,6 @@ class MetastoreCatalog(Catalog, ABC): def __init__(self, name: str, **properties: str): super().__init__(name, **properties) - for property_name in DEPRECATED_PROPERTY_NAMES: - if self.properties.get(property_name): - deprecation_message( - deprecated_in="0.7.0", - removed_in="0.8.0", - help_message=f"The property {property_name} is deprecated. Please use properties that start with client., glue., and dynamo. instead", - ) - def create_table_transaction( self, identifier: Union[str, Identifier], diff --git a/pyiceberg/catalog/dynamodb.py b/pyiceberg/catalog/dynamodb.py index 59cbe8ce28..b999df218a 100644 --- a/pyiceberg/catalog/dynamodb.py +++ b/pyiceberg/catalog/dynamodb.py @@ -30,12 +30,6 @@ import boto3 from pyiceberg.catalog import ( - DEPRECATED_ACCESS_KEY_ID, - DEPRECATED_BOTOCORE_SESSION, - DEPRECATED_PROFILE_NAME, - DEPRECATED_REGION, - DEPRECATED_SECRET_ACCESS_KEY, - DEPRECATED_SESSION_TOKEN, ICEBERG, METADATA_LOCATION, PREVIOUS_METADATA_LOCATION, @@ -102,18 +96,11 @@ def __init__(self, name: str, **properties: str): super().__init__(name, **properties) session = boto3.Session( - profile_name=get_first_property_value(properties, DYNAMODB_PROFILE_NAME, DEPRECATED_PROFILE_NAME), - region_name=get_first_property_value(properties, DYNAMODB_REGION, AWS_REGION, DEPRECATED_REGION), - botocore_session=properties.get(DEPRECATED_BOTOCORE_SESSION), - aws_access_key_id=get_first_property_value( - properties, DYNAMODB_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID, DEPRECATED_ACCESS_KEY_ID - ), - aws_secret_access_key=get_first_property_value( - properties, DYNAMODB_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, DEPRECATED_SECRET_ACCESS_KEY - ), - aws_session_token=get_first_property_value( - properties, DYNAMODB_SESSION_TOKEN, AWS_SESSION_TOKEN, DEPRECATED_SESSION_TOKEN - ), + profile_name=properties.get(DYNAMODB_PROFILE_NAME), + region_name=get_first_property_value(properties, DYNAMODB_REGION, AWS_REGION), + aws_access_key_id=get_first_property_value(properties, DYNAMODB_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), + aws_secret_access_key=get_first_property_value(properties, DYNAMODB_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY), + aws_session_token=get_first_property_value(properties, DYNAMODB_SESSION_TOKEN, AWS_SESSION_TOKEN), ) self.dynamodb = session.client(DYNAMODB_CLIENT) self.dynamodb_table_name = self.properties.get(DYNAMODB_TABLE_NAME, DYNAMODB_TABLE_NAME_DEFAULT) diff --git a/pyiceberg/catalog/glue.py b/pyiceberg/catalog/glue.py index 79850dbb89..385873c45b 100644 --- a/pyiceberg/catalog/glue.py +++ b/pyiceberg/catalog/glue.py @@ -40,12 +40,6 @@ ) from pyiceberg.catalog import ( - DEPRECATED_ACCESS_KEY_ID, - DEPRECATED_BOTOCORE_SESSION, - DEPRECATED_PROFILE_NAME, - DEPRECATED_REGION, - DEPRECATED_SECRET_ACCESS_KEY, - DEPRECATED_SESSION_TOKEN, EXTERNAL_TABLE, ICEBERG, LOCATION, @@ -303,18 +297,11 @@ def __init__(self, name: str, **properties: Any): super().__init__(name, **properties) session = boto3.Session( - profile_name=get_first_property_value(properties, GLUE_PROFILE_NAME, DEPRECATED_PROFILE_NAME), - region_name=get_first_property_value(properties, GLUE_REGION, AWS_REGION, DEPRECATED_REGION), - botocore_session=properties.get(DEPRECATED_BOTOCORE_SESSION), - aws_access_key_id=get_first_property_value( - properties, GLUE_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID, DEPRECATED_ACCESS_KEY_ID - ), - aws_secret_access_key=get_first_property_value( - properties, GLUE_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY, DEPRECATED_SECRET_ACCESS_KEY - ), - aws_session_token=get_first_property_value( - properties, GLUE_SESSION_TOKEN, AWS_SESSION_TOKEN, DEPRECATED_SESSION_TOKEN - ), + profile_name=properties.get(GLUE_PROFILE_NAME), + region_name=get_first_property_value(properties, GLUE_REGION, AWS_REGION), + aws_access_key_id=get_first_property_value(properties, GLUE_ACCESS_KEY_ID, AWS_ACCESS_KEY_ID), + aws_secret_access_key=get_first_property_value(properties, GLUE_SECRET_ACCESS_KEY, AWS_SECRET_ACCESS_KEY), + aws_session_token=get_first_property_value(properties, GLUE_SESSION_TOKEN, AWS_SESSION_TOKEN), ) self.glue: GlueClient = session.client("glue", endpoint_url=properties.get(GLUE_CATALOG_ENDPOINT)) diff --git a/pyiceberg/io/pyarrow.py b/pyiceberg/io/pyarrow.py index c6ecd4a6de..ab4de5185b 100644 --- a/pyiceberg/io/pyarrow.py +++ b/pyiceberg/io/pyarrow.py @@ -1688,23 +1688,6 @@ def project_batches( total_row_count += len(batch) -@deprecated( - deprecated_in="0.7.0", - removed_in="0.8.0", - help_message="The public API for 'to_requested_schema' is deprecated and is replaced by '_to_requested_schema'", -) -def to_requested_schema(requested_schema: Schema, file_schema: Schema, table: pa.Table) -> pa.Table: - struct_array = visit_with_partner(requested_schema, table, ArrowProjectionVisitor(file_schema), ArrowAccessor(file_schema)) - - arrays = [] - fields = [] - for pos, field in enumerate(requested_schema.fields): - array = struct_array.field(pos) - arrays.append(array) - fields.append(pa.field(field.name, array.type, field.optional)) - return pa.Table.from_arrays(arrays, schema=pa.schema(fields)) - - def _to_requested_schema( requested_schema: Schema, file_schema: Schema, diff --git a/pyiceberg/table/__init__.py b/pyiceberg/table/__init__.py index 0163c425ed..e431101ba9 100644 --- a/pyiceberg/table/__init__.py +++ b/pyiceberg/table/__init__.py @@ -89,7 +89,6 @@ from pyiceberg.table.update import ( AddPartitionSpecUpdate, AddSchemaUpdate, - AddSnapshotUpdate, AddSortOrderUpdate, AssertCreate, AssertRefSnapshotId, @@ -314,55 +313,6 @@ def set_properties(self, properties: Properties = EMPTY_DICT, **kwargs: Any) -> updates = properties or kwargs return self._apply((SetPropertiesUpdate(updates=updates),)) - @deprecated( - deprecated_in="0.7.0", - removed_in="0.8.0", - help_message="Please use one of the functions in ManageSnapshots instead", - ) - def add_snapshot(self, snapshot: Snapshot) -> Transaction: - """Add a new snapshot to the table. - - Returns: - The transaction with the add-snapshot staged. - """ - updates = (AddSnapshotUpdate(snapshot=snapshot),) - - return self._apply(updates, ()) - - @deprecated( - deprecated_in="0.7.0", - removed_in="0.8.0", - help_message="Please use one of the functions in ManageSnapshots instead", - ) - def set_ref_snapshot( - self, - snapshot_id: int, - parent_snapshot_id: Optional[int], - ref_name: str, - type: str, - max_ref_age_ms: Optional[int] = None, - max_snapshot_age_ms: Optional[int] = None, - min_snapshots_to_keep: Optional[int] = None, - ) -> Transaction: - """Update a ref to a snapshot. - - Returns: - The transaction with the set-snapshot-ref staged - """ - updates = ( - SetSnapshotRefUpdate( - snapshot_id=snapshot_id, - ref_name=ref_name, - type=type, - max_ref_age_ms=max_ref_age_ms, - max_snapshot_age_ms=max_snapshot_age_ms, - min_snapshots_to_keep=min_snapshots_to_keep, - ), - ) - - requirements = (AssertRefSnapshotId(snapshot_id=parent_snapshot_id, ref="main"),) - return self._apply(updates, requirements) - def _set_ref_snapshot( self, snapshot_id: int, diff --git a/tests/catalog/test_dynamodb.py b/tests/catalog/test_dynamodb.py index 0289c5d0bb..194caf46b7 100644 --- a/tests/catalog/test_dynamodb.py +++ b/tests/catalog/test_dynamodb.py @@ -45,7 +45,6 @@ from pyiceberg.typedef import Properties from tests.conftest import ( BUCKET_NAME, - DEPRECATED_AWS_SESSION_PROPERTIES, TABLE_METADATA_LOCATION_REGEX, UNIFIED_AWS_SESSION_PROPERTIES, ) @@ -563,28 +562,6 @@ def test_update_namespace_properties_overlap_update_removal(_bucket_initialize: assert test_catalog.load_namespace_properties(database_name) == test_properties -def test_passing_provided_profile() -> None: - catalog_name = "test_ddb_catalog" - session_props = { - "aws_access_key_id": "abc", - "aws_secret_access_key": "def", - "aws_session_token": "ghi", - "region_name": "eu-central-1", - "botocore_session": None, - "profile_name": None, - } - props = {"py-io-impl": "pyiceberg.io.fsspec.FsspecFileIO"} - props.update(session_props) # type: ignore - with mock.patch("boto3.Session", return_value=mock.Mock()) as mock_session: - mock_client = mock.Mock() - mock_session.return_value.client.return_value = mock_client - mock_client.describe_table.return_value = {"Table": {"TableStatus": "ACTIVE"}} - test_catalog = DynamoDbCatalog(catalog_name, **props) - assert test_catalog.dynamodb is mock_client - mock_session.assert_called_with(**session_props) - assert test_catalog.dynamodb is mock_session().client() - - @mock_aws def test_passing_glue_session_properties() -> None: session_properties: Properties = { @@ -594,7 +571,6 @@ def test_passing_glue_session_properties() -> None: "dynamodb.region": "dynamodb.region", "dynamodb.session-token": "dynamodb.session-token", **UNIFIED_AWS_SESSION_PROPERTIES, - **DEPRECATED_AWS_SESSION_PROPERTIES, } with mock.patch("boto3.Session") as mock_session: @@ -609,7 +585,6 @@ def test_passing_glue_session_properties() -> None: aws_session_token="dynamodb.session-token", region_name="dynamodb.region", profile_name="dynamodb.profile-name", - botocore_session=None, ) assert test_catalog.dynamodb is mock_session().client() @@ -619,7 +594,6 @@ def test_passing_unified_session_properties_to_dynamodb() -> None: session_properties: Properties = { "dynamodb.profile-name": "dynamodb.profile-name", **UNIFIED_AWS_SESSION_PROPERTIES, - **DEPRECATED_AWS_SESSION_PROPERTIES, } with mock.patch("boto3.Session") as mock_session: @@ -634,7 +608,6 @@ def test_passing_unified_session_properties_to_dynamodb() -> None: aws_session_token="client.session-token", region_name="client.region", profile_name="dynamodb.profile-name", - botocore_session=None, ) assert test_catalog.dynamodb is mock_session().client() diff --git a/tests/catalog/test_glue.py b/tests/catalog/test_glue.py index 53af750446..41b196facf 100644 --- a/tests/catalog/test_glue.py +++ b/tests/catalog/test_glue.py @@ -14,7 +14,7 @@ # KIND, either express or implied. See the License for the # specific language governing permissions and limitations # under the License. -from typing import Any, Dict, List +from typing import List from unittest import mock import boto3 @@ -40,7 +40,6 @@ from pyiceberg.types import IntegerType from tests.conftest import ( BUCKET_NAME, - DEPRECATED_AWS_SESSION_PROPERTIES, TABLE_METADATA_LOCATION_REGEX, UNIFIED_AWS_SESSION_PROPERTIES, ) @@ -638,26 +637,6 @@ def test_update_namespace_properties_overlap_update_removal( assert test_catalog.load_namespace_properties(database_name) == test_properties -@mock_aws -def test_passing_profile_name() -> None: - session_properties: Dict[str, Any] = { - "aws_access_key_id": "abc", - "aws_secret_access_key": "def", - "aws_session_token": "ghi", - "region_name": "eu-central-1", - "profile_name": "sandbox", - "botocore_session": None, - } - test_properties = {"type": "glue"} - test_properties.update(session_properties) - - with mock.patch("boto3.Session") as mock_session: - test_catalog = GlueCatalog("glue", **test_properties) - - mock_session.assert_called_with(**session_properties) - assert test_catalog.glue is mock_session().client() - - @mock_aws def test_passing_glue_session_properties() -> None: session_properties: Properties = { @@ -667,7 +646,6 @@ def test_passing_glue_session_properties() -> None: "glue.region": "glue.region", "glue.session-token": "glue.session-token", **UNIFIED_AWS_SESSION_PROPERTIES, - **DEPRECATED_AWS_SESSION_PROPERTIES, } with mock.patch("boto3.Session") as mock_session: @@ -679,7 +657,6 @@ def test_passing_glue_session_properties() -> None: aws_session_token="glue.session-token", region_name="glue.region", profile_name="glue.profile-name", - botocore_session=None, ) assert test_catalog.glue is mock_session().client() @@ -689,7 +666,6 @@ def test_passing_unified_session_properties_to_glue() -> None: session_properties: Properties = { "glue.profile-name": "glue.profile-name", **UNIFIED_AWS_SESSION_PROPERTIES, - **DEPRECATED_AWS_SESSION_PROPERTIES, } with mock.patch("boto3.Session") as mock_session: @@ -701,7 +677,6 @@ def test_passing_unified_session_properties_to_glue() -> None: aws_session_token="client.session-token", region_name="client.region", profile_name="glue.profile-name", - botocore_session=None, ) assert test_catalog.glue is mock_session().client() diff --git a/tests/conftest.py b/tests/conftest.py index b05947ebe6..e7e73375d7 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -2029,14 +2029,6 @@ def hierarchical_namespace_list(hierarchical_namespace_name: str) -> List[str]: re.X, ) -DEPRECATED_AWS_SESSION_PROPERTIES = { - "aws_access_key_id": "aws_access_key_id", - "aws_secret_access_key": "aws_secret_access_key", - "aws_session_token": "aws_session_token", - "region_name": "region_name", - "profile_name": "profile_name", -} - UNIFIED_AWS_SESSION_PROPERTIES = { "client.access-key-id": "client.access-key-id", "client.secret-access-key": "client.secret-access-key",