diff --git a/sdk/tables/azure-data-tables/azure/data/tables/__init__.py b/sdk/tables/azure-data-tables/azure/data/tables/__init__.py index 784eae1b1948..5424c62cb7ca 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/__init__.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/__init__.py @@ -6,24 +6,27 @@ from azure.data.tables._models import TableServiceStats from ._entity import TableEntity, EntityProperty, EdmType -from ._shared.table_shared_access_signature import generate_table_sas, \ +from ._table_shared_access_signature import generate_table_sas, \ generate_account_sas from ._table_client import TableClient from ._table_service_client import TableServiceClient - from ._models import ( AccessPolicy, Metrics, - RetentionPolicy, TableAnalyticsLogging, TableSasPermissions, CorsRule, UpdateMode, SASProtocol, Table, -) -from ._shared.models import ( + RetentionPolicy, + TableAnalyticsLogging, + TableSasPermissions, + CorsRule, + UpdateMode, + SASProtocol, + Table, LocationMode, ResourceTypes, AccountSasPermissions, - TableErrorCode ) -from ._shared.policies import ExponentialRetry, LinearRetry +from ._policies import ExponentialRetry, LinearRetry from ._version import VERSION +from ._deserialize import TableErrorCode __version__ = VERSION diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/authentication.py b/sdk/tables/azure-data-tables/azure/data/tables/_authentication.py similarity index 79% rename from sdk/tables/azure-data-tables/azure/data/tables/_shared/authentication.py rename to sdk/tables/azure-data-tables/azure/data/tables/_authentication.py index c479b6a51963..3d44e4f0d3ea 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/authentication.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_authentication.py @@ -5,7 +5,6 @@ # -------------------------------------------------------------------------- import logging -import sys try: from urllib.parse import urlparse except ImportError: @@ -16,7 +15,7 @@ from azure.core.exceptions import ClientAuthenticationError from azure.core.pipeline.policies import SansIOHTTPPolicy -from azure.data.tables._shared._constants import ( +from ._constants import ( DEV_ACCOUNT_NAME, DEV_ACCOUNT_SECONDARY_NAME ) @@ -32,21 +31,6 @@ logger = logging.getLogger(__name__) -# wraps a given exception with the desired exception type -def _wrap_exception(ex, desired_type): - msg = "" - if ex.args: - msg = ex.args[0] - if sys.version_info >= (3,): - # Automatic chaining in Python 3 means we keep the trace - return desired_type(msg) - # There isn't a good solution in 2 for keeping the stack trace - # in general, or that will not result in an error in 3 - # However, we can keep the previous error type and message - # TODO: In the future we will log the trace - return desired_type('{}: {}'.format(ex.__class__.__name__, msg)) - - class AzureSigningError(ClientAuthenticationError): """ Represents a fatal error when attempting to sign a request. @@ -126,13 +110,3 @@ def _get_canonicalized_resource_query(self, request): if name == 'comp': return '?comp=' + value return '' - - # def _get_canonicalized_resource_query(self, request): - # sorted_queries = [(name, value) for name, value in request.query.items()] - # sorted_queries.sort() - # - # string_to_sign = '' - # for name, value in sorted_queries: - # if value is not None: - # string_to_sign += '\n' + name.lower() + ':' + value - # return string_to_sign diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/base_client.py b/sdk/tables/azure-data-tables/azure/data/tables/_base_client.py similarity index 97% rename from sdk/tables/azure-data-tables/azure/data/tables/_shared/base_client.py rename to sdk/tables/azure-data-tables/azure/data/tables/_base_client.py index 30007bf7b9b9..af7a125d5082 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/base_client.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_base_client.py @@ -26,7 +26,6 @@ from urllib2 import quote # type: ignore import six -from azure.data.tables._shared.shared_access_signature import QueryStringConstants from azure.core.configuration import Configuration from azure.core.exceptions import HttpResponseError from azure.core.pipeline import Pipeline @@ -41,10 +40,11 @@ UserAgentPolicy ) -from .constants import STORAGE_OAUTH_SCOPE, SERVICE_HOST_BASE, CONNECTION_TIMEOUT, READ_TIMEOUT -from .models import LocationMode -from .authentication import SharedKeyCredentialPolicy -from .policies import ( +from ._shared_access_signature import QueryStringConstants +from ._constants import STORAGE_OAUTH_SCOPE, SERVICE_HOST_BASE, CONNECTION_TIMEOUT, READ_TIMEOUT +from ._models import LocationMode +from ._authentication import SharedKeyCredentialPolicy +from ._policies import ( StorageHeadersPolicy, StorageContentValidation, StorageRequestHook, @@ -52,8 +52,9 @@ StorageLoggingPolicy, StorageHosts, ExponentialRetry, ) -from .._version import VERSION -from .response_handlers import process_table_error, PartialBatchErrorException +from ._version import VERSION +from ._error import _process_table_error +from ._models import PartialBatchErrorException _LOGGER = logging.getLogger(__name__) @@ -296,7 +297,7 @@ def _batch_send( return iter(parts) return parts except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) class TransportWrapper(HttpTransport): """Wrapper class that ensures that an inner client created diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/__init__.py b/sdk/tables/azure-data-tables/azure/data/tables/_common_conversion.py similarity index 62% rename from sdk/tables/azure-data-tables/azure/data/tables/_shared/__init__.py rename to sdk/tables/azure-data-tables/azure/data/tables/_common_conversion.py index 160f88223820..e3d45124a78c 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/__init__.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_common_conversion.py @@ -3,48 +3,45 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- - import base64 import hashlib import hmac +from sys import version_info +import six -try: - from urllib.parse import quote, unquote -except ImportError: - from urllib2 import quote, unquote # type: ignore -import six +if version_info < (3,): + def _str(value): + if isinstance(value, unicode): # pylint: disable=undefined-variable + return value.encode('utf-8') + return str(value) +else: + _str = str -def url_quote(url): - return quote(url) +def _to_str(value): + return _str(value) if value is not None else None -def url_unquote(url): - return unquote(url) +def _to_utc_datetime(value): + return value.strftime('%Y-%m-%dT%H:%M:%SZ') -def encode_base64(data): +def _encode_base64(data): if isinstance(data, six.text_type): data = data.encode('utf-8') encoded = base64.b64encode(data) return encoded.decode('utf-8') -def decode_base64_to_bytes(data): +def _decode_base64_to_bytes(data): if isinstance(data, six.text_type): data = data.encode('utf-8') return base64.b64decode(data) - -def decode_base64_to_text(data): - decoded_bytes = decode_base64_to_bytes(data) - return decoded_bytes.decode('utf-8') - - -def sign_string(key, string_to_sign, key_is_base64=True): +def _sign_string(key, string_to_sign, key_is_base64=True): if key_is_base64: - key = decode_base64_to_bytes(key) + key = _decode_base64_to_bytes(key) else: if isinstance(key, six.text_type): key = key.encode('utf-8') @@ -52,5 +49,5 @@ def sign_string(key, string_to_sign, key_is_base64=True): string_to_sign = string_to_sign.encode('utf-8') signed_hmac_sha256 = hmac.HMAC(key, string_to_sign, hashlib.sha256) digest = signed_hmac_sha256.digest() - encoded_digest = encode_base64(digest) + encoded_digest = _encode_base64(digest) return encoded_digest diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/_constants.py b/sdk/tables/azure-data-tables/azure/data/tables/_constants.py similarity index 76% rename from sdk/tables/azure-data-tables/azure/data/tables/_shared/_constants.py rename to sdk/tables/azure-data-tables/azure/data/tables/_constants.py index 858875b6af28..c8f85fca9e14 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/_constants.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_constants.py @@ -5,20 +5,16 @@ # -------------------------------------------------------------------------- import platform import sys - -__author__ = 'Microsoft Corp. ' -__version__ = '1.4.2' +from ._generated.version import VERSION # UserAgent string sample: 'Azure-Storage/0.37.0-0.38.0 (Python CPython 3.4.2; Windows 8)' # First version(0.37.0) is the common package, and the second version(0.38.0) is the service package -USER_AGENT_STRING_PREFIX = 'Azure-Storage/{}-'.format(__version__) USER_AGENT_STRING_SUFFIX = '(Python {} {}; {} {})'.format(platform.python_implementation(), platform.python_version(), platform.system(), platform.release()) # default values for common package, in case it is used directly DEFAULT_X_MS_VERSION = '2018-03-28' -DEFAULT_USER_AGENT_STRING = '{}None {}'.format(USER_AGENT_STRING_PREFIX, USER_AGENT_STRING_SUFFIX) # Live ServiceClient URLs SERVICE_HOST_BASE = 'core.windows.net' @@ -49,3 +45,19 @@ _AUTHORIZATION_HEADER_NAME = 'Authorization' _COPY_SOURCE_HEADER_NAME = 'x-ms-copy-source' _REDACTED_VALUE = 'REDACTED' + + +X_MS_VERSION = VERSION + +# Socket timeout in seconds +CONNECTION_TIMEOUT = 20 +READ_TIMEOUT = 20 + +# for python 3.5+, there was a change to the definition of the socket timeout (as far as socket.sendall is concerned) +# The socket timeout is now the maximum total duration to send all data. +if sys.version_info >= (3, 5): + # the timeout to connect is 20 seconds, and the read timeout is 2000 seconds + # the 2000 seconds was calculated with: 100MB (max block size)/ 50KB/s (an arbitrarily chosen minimum upload speed) + READ_TIMEOUT = 2000 + +STORAGE_OAUTH_SCOPE = "https://storage.azure.com/.default" diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_deserialize.py b/sdk/tables/azure-data-tables/azure/data/tables/_deserialize.py index 4555dad01241..fa65406bcbde 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_deserialize.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_deserialize.py @@ -4,32 +4,48 @@ # license information. # -------------------------------------------------------------------------- # pylint: disable=unused-argument -import datetime +from typing import ( # pylint: disable=unused-import + Union, Optional, Any, Iterable, Dict, List, Type, Tuple, + TYPE_CHECKING +) from uuid import UUID +import logging +import datetime + from azure.core.exceptions import ResourceExistsError -from ._shared import url_quote + from ._entity import EntityProperty, EdmType, TableEntity -from ._shared._common_conversion import _decode_base64_to_bytes +from ._common_conversion import _decode_base64_to_bytes from ._generated.models import TableProperties +from ._error import TableErrorCode + +if TYPE_CHECKING: + from datetime import datetime + from azure.core.exceptions import AzureError + +_LOGGER = logging.getLogger(__name__) -from ._shared.models import TableErrorCode +try: + from urllib.parse import quote +except ImportError: + from urllib2 import quote # type: ignore -def deserialize_metadata(response, _, headers): - return {k[10:]: v for k, v in response.headers.items() if k.startswith("x-ms-meta-")} +def url_quote(url): + return quote(url) -def deserialize_table_properties(response, obj, headers): - metadata = deserialize_metadata(response, obj, headers) - table_properties = TableProperties( - metadata=metadata, - **headers - ) - return table_properties +def get_enum_value(value): + if value is None or value in ["None", ""]: + return None + try: + return value.value + except AttributeError: + return value -def deserialize_table_creation(response, _, headers): +def _deserialize_table_creation(response, _, headers): if response.status_code == 204: error_code = TableErrorCode.table_already_exists error = ResourceExistsError( @@ -174,3 +190,20 @@ def _extract_etag(response): return response.headers.get('etag') return None + + +def _normalize_headers(headers): + normalized = {} + for key, value in headers.items(): + if key.startswith('x-ms-'): + key = key[5:] + normalized[key.lower().replace('-', '_')] = get_enum_value(value) + return normalized + + +def _return_headers_and_deserialized(response, deserialized, response_headers): # pylint: disable=unused-argument + return _normalize_headers(response_headers), deserialized + + +def _return_context_and_deserialized(response, deserialized, response_headers): # pylint: disable=unused-argument + return response.http_response.location_mode, deserialized, response_headers diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_entity.py b/sdk/tables/azure-data-tables/azure/data/tables/_entity.py index cc3265c2860c..faed3d8ae453 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_entity.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_entity.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- from enum import Enum -from ._shared._error import _ERROR_ATTRIBUTE_MISSING +from ._error import _ERROR_ATTRIBUTE_MISSING class TableEntity(dict): diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_error.py b/sdk/tables/azure-data-tables/azure/data/tables/_error.py new file mode 100644 index 000000000000..cb2349c6bb04 --- /dev/null +++ b/sdk/tables/azure-data-tables/azure/data/tables/_error.py @@ -0,0 +1,205 @@ +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for +# license information. +# -------------------------------------------------------------------------- +from sys import version_info +from re import match +from enum import Enum + +from azure.core.exceptions import ( + HttpResponseError, + ResourceNotFoundError, + ResourceModifiedError, + ResourceExistsError, + ClientAuthenticationError, + DecodeError) +from azure.core.pipeline.policies import ContentDecodePolicy + +if version_info < (3,): + def _str(value): + if isinstance(value, unicode): # pylint: disable=undefined-variable + return value.encode('utf-8') + + return str(value) +else: + _str = str + + +def _to_utc_datetime(value): + return value.strftime('%Y-%m-%dT%H:%M:%SZ') + + +def _to_str(value): + return _str(value) if value is not None else None + + +_ERROR_ATTRIBUTE_MISSING = '\'{0}\' object has no attribute \'{1}\'' +_ERROR_BATCH_COMMIT_FAIL = 'Batch Commit Fail' +_ERROR_TYPE_NOT_SUPPORTED = 'Type not supported when sending data to the service: {0}.' +_ERROR_VALUE_TOO_LARGE = '{0} is too large to be cast to type {1}.' +_ERROR_ATTRIBUTE_MISSING = '\'{0}\' object has no attribute \'{1}\'' +_ERROR_UNKNOWN = 'Unknown error ({0})' +_ERROR_VALUE_NONE = '{0} should not be None.' +_ERROR_UNKNOWN_KEY_WRAP_ALGORITHM = 'Unknown key wrap algorithm.' + + +def _validate_not_none(param_name, param): + if param is None: + raise ValueError(_ERROR_VALUE_NONE.format(param_name)) + + +def _wrap_exception(ex, desired_type): + msg = "" + if len(ex.args) > 0: # pylint: disable=C1801 + msg = ex.args[0] + if version_info >= (3,): # pylint: disable=R1705 + # Automatic chaining in Python 3 means we keep the trace + return desired_type(msg) + else: + # There isn't a good solution in 2 for keeping the stack trace + # in general, or that will not result in an error in 3 + # However, we can keep the previous error type and message + # TODO: In the future we will log the trace + return desired_type('{}: {}'.format(ex.__class__.__name__, msg)) + + +def _validate_table_name(table_name): + if match("^[a-zA-Z]{1}[a-zA-Z0-9]{2,62}$", table_name) is None: + raise ValueError( + "Table names must be alphanumeric, cannot begin with a number, and must be between 3-63 characters long." + ) + + +def _process_table_error(storage_error): + raise_error = HttpResponseError + error_code = storage_error.response.headers.get('x-ms-error-code') + error_message = storage_error.message + additional_data = {} + try: + error_body = ContentDecodePolicy.deserialize_from_http_generics(storage_error.response) + if isinstance(error_body, dict): + for info in error_body['odata.error']: + if info == 'code': + error_code = error_body['odata.error'][info] + elif info == 'message': + error_message = error_body['odata.error'][info]['value'] + else: + additional_data[info.tag] = info.text + else: + if error_body: + for info in error_body.iter(): + if info.tag.lower().find('code') != -1: + error_code = info.text + elif info.tag.lower().find('message') != -1: + error_message = info.text + else: + additional_data[info.tag] = info.text + except DecodeError: + pass + + try: + if error_code: + error_code = TableErrorCode(error_code) + if error_code in [TableErrorCode.condition_not_met]: + raise_error = ResourceModifiedError + if error_code in [TableErrorCode.invalid_authentication_info, + TableErrorCode.authentication_failed]: + raise_error = ClientAuthenticationError + if error_code in [TableErrorCode.resource_not_found, + TableErrorCode.table_not_found, + TableErrorCode.entity_not_found, + ResourceNotFoundError]: + raise_error = ResourceNotFoundError + if error_code in [TableErrorCode.resource_already_exists, + TableErrorCode.table_already_exists, + TableErrorCode.account_already_exists, + TableErrorCode.entity_already_exists, + ResourceExistsError]: + raise_error = ResourceExistsError + except ValueError: + # Got an unknown error code + pass + + try: + error_message += "\nErrorCode:{}".format(error_code.value) + except AttributeError: + error_message += "\nErrorCode:{}".format(error_code) + for name, info in additional_data.items(): + error_message += "\n{}:{}".format(name, info) + + error = raise_error(message=error_message, response=storage_error.response) + error.error_code = error_code + error.additional_info = additional_data + raise error + + +class TableErrorCode(str, Enum): + # Generic storage values + account_already_exists = "AccountAlreadyExists" + account_being_created = "AccountBeingCreated" + account_is_disabled = "AccountIsDisabled" + authentication_failed = "AuthenticationFailed" + authorization_failure = "AuthorizationFailure" + no_authentication_information = "NoAuthenticationInformation" + condition_headers_not_supported = "ConditionHeadersNotSupported" + condition_not_met = "ConditionNotMet" + empty_metadata_key = "EmptyMetadataKey" + insufficient_account_permissions = "InsufficientAccountPermissions" + internal_error = "InternalError" + invalid_authentication_info = "InvalidAuthenticationInfo" + invalid_header_value = "InvalidHeaderValue" + invalid_http_verb = "InvalidHttpVerb" + invalid_input = "InvalidInput" + invalid_md5 = "InvalidMd5" + invalid_metadata = "InvalidMetadata" + invalid_query_parameter_value = "InvalidQueryParameterValue" + invalid_range = "InvalidRange" + invalid_resource_name = "InvalidResourceName" + invalid_uri = "InvalidUri" + invalid_xml_document = "InvalidXmlDocument" + invalid_xml_node_value = "InvalidXmlNodeValue" + md5_mismatch = "Md5Mismatch" + metadata_too_large = "MetadataTooLarge" + missing_content_length_header = "MissingContentLengthHeader" + missing_required_query_parameter = "MissingRequiredQueryParameter" + missing_required_header = "MissingRequiredHeader" + missing_required_xml_node = "MissingRequiredXmlNode" + multiple_condition_headers_not_supported = "MultipleConditionHeadersNotSupported" + operation_timed_out = "OperationTimedOut" + out_of_range_input = "OutOfRangeInput" + out_of_range_query_parameter_value = "OutOfRangeQueryParameterValue" + request_body_too_large = "RequestBodyTooLarge" + resource_type_mismatch = "ResourceTypeMismatch" + request_url_failed_to_parse = "RequestUrlFailedToParse" + resource_already_exists = "ResourceAlreadyExists" + resource_not_found = "ResourceNotFound" + server_busy = "ServerBusy" + unsupported_header = "UnsupportedHeader" + unsupported_xml_node = "UnsupportedXmlNode" + unsupported_query_parameter = "UnsupportedQueryParameter" + unsupported_http_verb = "UnsupportedHttpVerb" + + # table error codes + duplicate_properties_specified = "DuplicatePropertiesSpecified" + entity_not_found = "EntityNotFound" + entity_already_exists = "EntityAlreadyExists" + entity_too_large = "EntityTooLarge" + host_information_not_present = "HostInformationNotPresent" + invalid_duplicate_row = "InvalidDuplicateRow" + invalid_value_type = "InvalidValueType" + json_format_not_supported = "JsonFormatNotSupported" + method_not_allowed = "MethodNotAllowed" + not_implemented = "NotImplemented" + properties_need_value = "PropertiesNeedValue" + property_name_invalid = "PropertyNameInvalid" + property_name_too_long = "PropertyNameTooLong" + property_value_too_large = "PropertyValueTooLarge" + table_already_exists = "TableAlreadyExists" + table_being_deleted = "TableBeingDeleted" + table_not_found = "TableNotFound" + too_many_properties = "TooManyProperties" + update_condition_not_satisfied = "UpdateConditionNotSatisfied" + x_method_incorrect_count = "XMethodIncorrectCount" + x_method_incorrect_value = "XMethodIncorrectValue" + x_method_not_using_post = "XMethodNotUsingPost" diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_models.py b/sdk/tables/azure-data-tables/azure/data/tables/_models.py index 36d203b46690..5dbb662aff4e 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_models.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_models.py @@ -8,15 +8,16 @@ from azure.core.paging import PageIterator from azure.data.tables._generated.models import TableServiceStats as GenTableServiceStats -from ._deserialize import _convert_to_entity -from ._shared.models import Services -from ._shared.response_handlers import return_context_and_deserialized, process_table_error from ._generated.models import AccessPolicy as GenAccessPolicy from ._generated.models import Logging as GeneratedLogging from ._generated.models import Metrics as GeneratedMetrics from ._generated.models import RetentionPolicy as GeneratedRetentionPolicy from ._generated.models import CorsRule as GeneratedCorsRule - +from ._deserialize import ( + _convert_to_entity, + _return_context_and_deserialized +) +from ._error import _process_table_error class TableServiceStats(GenTableServiceStats): """Stats for the service @@ -285,15 +286,15 @@ def __init__(self, command, prefix=None, continuation_token=None): self._headers = None self.location_mode = None - def _get_next_cb(self, continuation_token): + def _get_next_cb(self, continuation_token, **kwargs): try: return self._command( next_table_name=continuation_token or None, - cls=return_context_and_deserialized, + cls=kwargs.pop('cls', None) or _return_context_and_deserialized, use_location=self.location_mode ) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) def _extract_data_cb(self, get_next_return): self.location_mode, self._response, self._headers = get_next_return @@ -333,7 +334,7 @@ def __init__(self, command, results_per_page=None, table=None, self.table = table self.location_mode = None - def _get_next_cb(self, continuation_token): + def _get_next_cb(self, continuation_token, **kwargs): row_key = "" partition_key = "" for key, value in continuation_token.items(): @@ -347,11 +348,11 @@ def _get_next_cb(self, continuation_token): next_row_key=row_key or None, next_partition_key=partition_key or None, table=self.table, - cls=return_context_and_deserialized, + cls=kwargs.pop('cls', None) or _return_context_and_deserialized, use_location=self.location_mode ) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) def _extract_data_cb(self, get_next_return): self.location_mode, self._response, self._headers = get_next_return @@ -457,11 +458,6 @@ def service_properties_deserialize(generated): } -class TableServices(Services): - def __str__(self): - return 't' - - class Table(object): """ Represents an Azure Table. Returned by list_tables. @@ -497,3 +493,152 @@ class UpdateMode(str, Enum): class SASProtocol(str, Enum): HTTPS = "https" HTTP = "http" + + +class PartialBatchErrorException(HttpResponseError): + """There is a partial failure in batch operations. + + :param str message: The message of the exception. + :param response: Server response to be deserialized. + :param list parts: A list of the parts in multipart response. + """ + + def __init__(self, message, response, parts): + self.parts = parts + super(PartialBatchErrorException, self).__init__(message=message, response=response) + + + +class LocationMode(object): + """ + Specifies the location the request should be sent to. This mode only applies + for RA-GRS accounts which allow secondary read access. All other account types + must use PRIMARY. + """ + + PRIMARY = 'primary' #: Requests should be sent to the primary location. + SECONDARY = 'secondary' #: Requests should be sent to the secondary location, if possible. + + +class ResourceTypes(object): + """ + Specifies the resource types that are accessible with the account SAS. + + :param bool service: + Access to service-level APIs (e.g., Get/Set Service Properties, + Get Service Stats, List Containers/Queues/Shares) + :param bool object: + Access to object-level APIs for blobs, queue messages, and + files(e.g. Put Blob, Query Entity, Get Messages, Create File, etc.) + """ + + def __init__(self, service=False, object=False): # pylint: disable=redefined-builtin + self.service = service + self.object = object + self._str = (('s' if self.service else '') + + ('o' if self.object else '')) + + def __str__(self): + return self._str + + @classmethod + def from_string(cls, string): + """Create a ResourceTypes from a string. + + To specify service, container, or object you need only to + include the first letter of the word in the string. E.g. service and container, + you would provide a string "sc". + + :param str string: Specify service, container, or object in + in the string with the first letter of the word. + :return: A ResourceTypes object + :rtype: ~azure.data.tables.ResourceTypes + """ + res_service = 's' in string + res_object = 'o' in string + + parsed = cls(res_service, res_object) + parsed._str = string # pylint: disable = protected-access + return parsed + + +class AccountSasPermissions(object): + """ + :class:`~ResourceTypes` class to be used with generate_account_sas + function and for the AccessPolicies used with set_*_acl. There are two types of + SAS which may be used to grant resource access. One is to grant access to a + specific resource (resource-specific). Another is to grant access to the + entire service for a specific account and allow certain operations based on + perms found here. + + :ivar bool read: + Valid for all signed resources types (Service, Container, and Object). + Permits read permissions to the specified resource type. + :ivar bool write: + Valid for all signed resources types (Service, Container, and Object). + Permits write permissions to the specified resource type. + :ivar bool delete: + Valid for Container and Object resource types, except for queue messages. + :ivar bool list: + Valid for Service and Container resource types only. + :ivar bool add: + Valid for the following Object resource types only: queue messages, and append blobs. + :ivar bool create: + Valid for the following Object resource types only: blobs and files. + Users can create new blobs or files, but may not overwrite existing + blobs or files. + :ivar bool update: + Valid for the following Object resource types only: queue messages. + :ivar bool process: + Valid for the following Object resource type only: queue messages. + """ + + def __init__(self, **kwargs): # pylint: disable=redefined-builtin + self.read = kwargs.pop('read', None) + self.write = kwargs.pop('write', None) + self.delete = kwargs.pop('delete', None) + self.list = kwargs.pop('list', None) + self.add = kwargs.pop('add', None) + self.create = kwargs.pop('create', None) + self.update = kwargs.pop('update', None) + self.process = kwargs.pop('process', None) + self._str = (('r' if self.read else '') + + ('w' if self.write else '') + + ('d' if self.delete else '') + + ('l' if self.list else '') + + ('a' if self.add else '') + + ('c' if self.create else '') + + ('u' if self.update else '') + + ('p' if self.process else '')) + + def __str__(self): + return self._str + + @classmethod + def from_string(cls, permission, **kwargs): # pylint:disable=W0613 + """Create AccountSasPermissions from a string. + + To specify read, write, delete, etc. permissions you need only to + include the first letter of the word in the string. E.g. for read and write + permissions you would provide a string "rw". + + :param str permission: Specify permissions in + the string with the first letter of the word. + :keyword callable cls: A custom type or function that will be passed the direct response + :return: A AccountSasPermissions object + :rtype: ~azure.data.tables.AccountSasPermissions + """ + p_read = 'r' in permission + p_write = 'w' in permission + p_delete = 'd' in permission + p_list = 'l' in permission + p_add = 'a' in permission + p_create = 'c' in permission + p_update = 'u' in permission + p_process = 'p' in permission + + parsed = cls( + **dict(kwargs, read=p_read, write=p_write, delete=p_delete, list=p_list, add=p_add, create=p_create, + update=p_update, process=p_process)) + parsed._str = permission # pylint: disable = protected-access + return parsed diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/policies.py b/sdk/tables/azure-data-tables/azure/data/tables/_policies.py similarity index 99% rename from sdk/tables/azure-data-tables/azure/data/tables/_shared/policies.py rename to sdk/tables/azure-data-tables/azure/data/tables/_policies.py index adc84c59dded..732779f3943b 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/policies.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_policies.py @@ -40,7 +40,7 @@ ) from azure.core.exceptions import AzureError, ServiceRequestError, ServiceResponseError -from .models import LocationMode +from ._models import LocationMode try: _unicode_type = unicode # type: ignore diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_serialize.py b/sdk/tables/azure-data-tables/azure/data/tables/_serialize.py index 32e9db432c57..f937d68f3b72 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_serialize.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_serialize.py @@ -4,21 +4,20 @@ # license information. # -------------------------------------------------------------------------- # pylint: disable=no-self-use -import sys -import uuid from uuid import UUID from datetime import datetime - -from math import ( - isnan, -) +from math import isnan +import sys +import uuid +import isodate from azure.core import MatchConditions +from azure.core.exceptions import raise_with_traceback + from ._entity import EdmType, EntityProperty from ._models import TablePayloadFormat -from ._shared._common_conversion import _to_str, _encode_base64, _to_utc_datetime -from ._shared._error import _ERROR_VALUE_TOO_LARGE, _ERROR_TYPE_NOT_SUPPORTED - +from ._common_conversion import _to_str, _encode_base64, _to_utc_datetime +from ._error import _ERROR_VALUE_TOO_LARGE, _ERROR_TYPE_NOT_SUPPORTED _SUPPORTED_API_VERSIONS = [ @@ -215,3 +214,31 @@ def _add_entity_properties(source): # generate the entity_body return properties + + +def serialize_iso(attr): + """Serialize Datetime object into ISO-8601 formatted string. + + :param Datetime attr: Object to be serialized. + :rtype: str + :raises: ValueError if format invalid. + """ + if not attr: + return None + if isinstance(attr, str): + attr = isodate.parse_datetime(attr) + try: + utc = attr.utctimetuple() + if utc.tm_year > 9999 or utc.tm_year < 1: + raise OverflowError("Hit max or min date") + + date = "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}".format( + utc.tm_year, utc.tm_mon, utc.tm_mday, + utc.tm_hour, utc.tm_min, utc.tm_sec) + return date + 'Z' + except (ValueError, OverflowError) as err: + msg = "Unable to serialize datetime object." + raise_with_traceback(ValueError, msg, err) + except AttributeError as err: + msg = "ISO-8601 object must be valid Datetime object." + raise_with_traceback(TypeError, msg, err) diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/_common_conversion.py b/sdk/tables/azure-data-tables/azure/data/tables/_shared/_common_conversion.py deleted file mode 100644 index 4de22559aba8..000000000000 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/_common_conversion.py +++ /dev/null @@ -1,107 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -import base64 -import hashlib -import hmac -from io import (SEEK_SET) -import six -from azure.data.tables._shared.parser import _str - - -from ._error import ( - _ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM, - _ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM, -) - - -def _to_str(value): - return _str(value) if value is not None else None - - -def _int_to_str(value): - return str(int(value)) if value is not None else None - - -def _bool_to_str(value): - if value is None: - return None - - if isinstance(value, bool): - if value: # pylint: disable=R1705 - return 'true' - else: - return 'false' - - return str(value) - - -def _to_utc_datetime(value): - return value.strftime('%Y-%m-%dT%H:%M:%SZ') - - -def _datetime_to_utc_string(value): - # Azure expects the date value passed in to be UTC. - # Azure will always return values as UTC. - # If a date is passed in without timezone info, it is assumed to be UTC. - if value is None: - return None - - return value.strftime('%a, %d %b %Y %H:%M:%S GMT') - - -def _encode_base64(data): - if isinstance(data, six.text_type): - data = data.encode('utf-8') - encoded = base64.b64encode(data) - return encoded.decode('utf-8') - - -def _decode_base64_to_bytes(data): - if isinstance(data, six.text_type): - data = data.encode('utf-8') - return base64.b64decode(data) - - -def _decode_base64_to_text(data): - decoded_bytes = _decode_base64_to_bytes(data) - return decoded_bytes.decode('utf-8') - - -def _sign_string(key, string_to_sign, key_is_base64=True): - if key_is_base64: - key = _decode_base64_to_bytes(key) - else: - if isinstance(key, six.text_type): - key = key.encode('utf-8') - if isinstance(string_to_sign, six.text_type): - string_to_sign = string_to_sign.encode('utf-8') - signed_hmac_sha256 = hmac.HMAC(key, string_to_sign, hashlib.sha256) - digest = signed_hmac_sha256.digest() - encoded_digest = _encode_base64(digest) - return encoded_digest - - -def _get_content_md5(data): - md5 = hashlib.md5() # nosec - if isinstance(data, bytes): - md5.update(data) - elif hasattr(data, 'read'): - pos = 0 - pos = data.tell() - for chunk in iter(lambda: data.read(4096), b""): - md5.update(chunk) - try: - data.seek(pos, SEEK_SET) - except (AttributeError, IOError): - raise ValueError(_ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM.format('data')) - else: - raise ValueError(_ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM.format('data')) - - return base64.b64encode(md5.digest()).decode('utf-8') - - -def _lower(text): - return text.lower() diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/_error.py b/sdk/tables/azure-data-tables/azure/data/tables/_shared/_error.py deleted file mode 100644 index 32d9e7705378..000000000000 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/_error.py +++ /dev/null @@ -1,230 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from sys import version_info -from re import match - -from azure.core.exceptions import HttpResponseError, ResourceExistsError, ResourceNotFoundError -from azure.data.tables._shared.parser import _str - -from ._constants import ( - _ENCRYPTION_PROTOCOL_V1, -) - -def _to_str(value): - return _str(value) if value is not None else None - - -_ERROR_ATTRIBUTE_MISSING = '\'{0}\' object has no attribute \'{1}\'' -_ERROR_BATCH_COMMIT_FAIL = 'Batch Commit Fail' -_ERROR_CANNOT_FIND_PARTITION_KEY = 'Cannot find partition key in request.' -_ERROR_CANNOT_FIND_ROW_KEY = 'Cannot find row key in request.' -_ERROR_CANNOT_SERIALIZE_VALUE_TO_ENTITY = \ - 'Cannot serialize the specified value ({0}) to an entity. Please use ' + \ - 'an EntityProperty (which can specify custom types), int, str, bool, ' + \ - 'or datetime.' -_ERROR_CANNOT_DESERIALIZE_VALUE_TO_ENTITY = \ - 'Cannot deserialize the specified value ({0}).' -_ERROR_DUPLICATE_ROW_KEY_IN_BATCH = \ - 'Row Keys should not be the same in a batch operations' -_ERROR_INCORRECT_PARTITION_KEY_IN_BATCH = \ - 'Partition Key should be the same in a batch operations' -_ERROR_INVALID_ENTITY_TYPE = 'The entity must be either in dict format or an entity object.' -_ERROR_INVALID_PROPERTY_RESOLVER = \ - 'The specified property resolver returned an invalid type. Name: {0}, Value: {1}, ' + \ - 'EdmType: {2}' -_ERROR_PROPERTY_NAME_TOO_LONG = 'The property name exceeds the maximum allowed length.' -_ERROR_TOO_MANY_ENTITIES_IN_BATCH = \ - 'Batches may only contain 100 operations' -_ERROR_TOO_MANY_PROPERTIES = 'The entity contains more properties than allowed.' -_ERROR_TYPE_NOT_SUPPORTED = 'Type not supported when sending data to the service: {0}.' -_ERROR_VALUE_TOO_LARGE = '{0} is too large to be cast to type {1}.' -_ERROR_UNSUPPORTED_TYPE_FOR_ENCRYPTION = 'Encryption is only supported for not None strings.' -_ERROR_ENTITY_NOT_ENCRYPTED = 'Entity was not encrypted.' -_ERROR_ATTRIBUTE_MISSING = '\'{0}\' object has no attribute \'{1}\'' -_ERROR_CONFLICT = 'Conflict ({0})' -_ERROR_NOT_FOUND = 'Not found ({0})' -_ERROR_UNKNOWN = 'Unknown error ({0})' -_ERROR_STORAGE_MISSING_INFO = \ - 'You need to provide an account name and either an account_key or sas_token when creating a storage service.' -_ERROR_EMULATOR_DOES_NOT_SUPPORT_FILES = \ - 'The emulator does not support the file service.' -_ERROR_ACCESS_POLICY = \ - 'share_access_policy must be either SignedIdentifier or AccessPolicy ' + \ - 'instance' -_ERROR_PARALLEL_NOT_SEEKABLE = 'Parallel operations require a seekable stream.' -_ERROR_VALUE_SHOULD_BE_BYTES = '{0} should be of type bytes.' -_ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM = '{0} should be of type bytes or a readable file-like/io.IOBase stream object.' -_ERROR_VALUE_SHOULD_BE_SEEKABLE_STREAM = '{0} should be a seekable file-like/io.IOBase type stream object.' -_ERROR_VALUE_SHOULD_BE_STREAM = '{0} should be a file-like/io.IOBase type stream object with a read method.' -_ERROR_VALUE_NONE = '{0} should not be None.' -_ERROR_VALUE_NONE_OR_EMPTY = '{0} should not be None or empty.' -_ERROR_VALUE_NEGATIVE = '{0} should not be negative.' -_ERROR_START_END_NEEDED_FOR_MD5 = \ - 'Both end_range and start_range need to be specified ' + \ - 'for getting content MD5.' -_ERROR_RANGE_TOO_LARGE_FOR_MD5 = \ - 'Getting content MD5 for a range greater than 4MB ' + \ - 'is not supported.' -_ERROR_MD5_MISMATCH = \ - 'MD5 mismatch. Expected value is \'{0}\', computed value is \'{1}\'.' -_ERROR_TOO_MANY_ACCESS_POLICIES = \ - 'Too many access policies provided. ' \ - 'The server does not support setting more than 5 access policies on a single resource.' -_ERROR_OBJECT_INVALID = \ - '{0} does not define a complete interface. Value of {1} is either missing or invalid.' -_ERROR_UNSUPPORTED_ENCRYPTION_VERSION = \ - 'Encryption version is not supported.' -_ERROR_DECRYPTION_FAILURE = \ - 'Decryption failed' -_ERROR_ENCRYPTION_REQUIRED = \ - 'Encryption required but no key was provided.' -_ERROR_DECRYPTION_REQUIRED = \ - 'Decryption required but neither key nor resolver was provided.' + \ - ' If you do not want to decypt, please do not set the require encryption flag.' -_ERROR_INVALID_KID = \ - 'Provided or resolved key-encryption-key does not match the id of key used to encrypt.' -_ERROR_UNSUPPORTED_ENCRYPTION_ALGORITHM = \ - 'Specified encryption algorithm is not supported.' -_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION = 'The require_encryption flag is set, but encryption is not supported' + \ - ' for this method.' -_ERROR_UNKNOWN_KEY_WRAP_ALGORITHM = 'Unknown key wrap algorithm.' -_ERROR_DATA_NOT_ENCRYPTED = 'Encryption required, but received data does not contain appropriate metatadata.' + \ - 'Data was either not encrypted or metadata has been lost.' - - -def _dont_fail_on_exist(error): - """ don't throw exception if the resource exists. - This is called by create_* APIs with fail_on_exist=False""" - if isinstance(error, ResourceExistsError): # pylint: disable=R1705 - return False - else: - raise error - - -def _dont_fail_not_exist(error): - """ don't throw exception if the resource doesn't exist. - This is called by create_* APIs with fail_on_exist=False""" - if isinstance(error, ResourceNotFoundError): # pylint: disable=R1705 - return False - else: - raise error - - -def _http_error_handler(http_error): - """ Simple error handler for azure.""" - message = str(http_error) - error_code = None - - if 'x-ms-error-code' in http_error.respheader: - error_code = http_error.respheader['x-ms-error-code'] - message += ' ErrorCode: ' + error_code - - if http_error.respbody is not None: - message += '\n' + http_error.respbody.decode('utf-8-sig') - - ex = HttpResponseError(message, http_error.status) - ex.error_code = error_code - - raise ex - - -def _validate_type_bytes(param_name, param): - if not isinstance(param, bytes): - raise TypeError(_ERROR_VALUE_SHOULD_BE_BYTES.format(param_name)) - - -def _validate_type_bytes_or_stream(param_name, param): - if not (isinstance(param, bytes) or hasattr(param, 'read')): - raise TypeError(_ERROR_VALUE_SHOULD_BE_BYTES_OR_STREAM.format(param_name)) - - -def _validate_not_none(param_name, param): - if param is None: - raise ValueError(_ERROR_VALUE_NONE.format(param_name)) - - -def _validate_content_match(server_md5, computed_md5): - if server_md5 != computed_md5: - raise Exception(_ERROR_MD5_MISMATCH.format(server_md5, computed_md5)) - - -def _validate_access_policies(identifiers): - if identifiers and len(identifiers) > 5: - raise Exception(_ERROR_TOO_MANY_ACCESS_POLICIES) - - -def _validate_key_encryption_key_wrap(kek): - # Note that None is not callable and so will fail the second clause of each check. - if not hasattr(kek, 'wrap_key') or not callable(kek.wrap_key): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'wrap_key')) - if not hasattr(kek, 'get_kid') or not callable(kek.get_kid): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_kid')) - if not hasattr(kek, 'get_key_wrap_algorithm') or not callable(kek.get_key_wrap_algorithm): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_key_wrap_algorithm')) - - -def _validate_key_encryption_key_unwrap(kek): - if not hasattr(kek, 'get_kid') or not callable(kek.get_kid): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'get_kid')) - if not hasattr(kek, 'unwrap_key') or not callable(kek.unwrap_key): - raise AttributeError(_ERROR_OBJECT_INVALID.format('key encryption key', 'unwrap_key')) - - -def _validate_encryption_required(require_encryption, kek): - if require_encryption and (kek is None): - raise ValueError(_ERROR_ENCRYPTION_REQUIRED) - - -def _validate_decryption_required(require_encryption, kek, resolver): - if (require_encryption and (kek is None) and - (resolver is None)): - raise ValueError(_ERROR_DECRYPTION_REQUIRED) - - -def _validate_encryption_protocol_version(encryption_protocol): - if not _ENCRYPTION_PROTOCOL_V1 == encryption_protocol: - raise ValueError(_ERROR_UNSUPPORTED_ENCRYPTION_VERSION) - - -def _validate_kek_id(kid, resolved_id): - if not kid == resolved_id: - raise ValueError(_ERROR_INVALID_KID) - - -def _validate_encryption_unsupported(require_encryption, key_encryption_key): - if require_encryption or (key_encryption_key is not None): - raise ValueError(_ERROR_UNSUPPORTED_METHOD_FOR_ENCRYPTION) - - -# wraps a given exception with the desired exception type -def _wrap_exception(ex, desired_type): - msg = "" - if len(ex.args) > 0: # pylint: disable=C1801 - msg = ex.args[0] - if version_info >= (3,): # pylint: disable=R1705 - # Automatic chaining in Python 3 means we keep the trace - return desired_type(msg) - else: - # There isn't a good solution in 2 for keeping the stack trace - # in general, or that will not result in an error in 3 - # However, we can keep the previous error type and message - # TODO: In the future we will log the trace - return desired_type('{}: {}'.format(ex.__class__.__name__, msg)) - - -def _validate_table_name(table_name): - if match("^[a-zA-Z]{1}[a-zA-Z0-9]{2,62}$", table_name) is None: - raise ValueError( - "Table names must be alphanumeric, cannot begin with a number, and must be between 3-63 characters long." - ) - - -class AzureSigningError(Exception): - """ - Represents a fatal error when attempting to sign a request. - In general, the cause of this exception is user error. For example, the given account key is not valid. - Please visit https://docs.microsoft.com/en-us/azure/storage/common/storage-create-storage-account for more info. - """ diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/constants.py b/sdk/tables/azure-data-tables/azure/data/tables/_shared/constants.py deleted file mode 100644 index 7fb05b559850..000000000000 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/constants.py +++ /dev/null @@ -1,26 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -import sys -from .._generated.version import VERSION - - -X_MS_VERSION = VERSION - -# Socket timeout in seconds -CONNECTION_TIMEOUT = 20 -READ_TIMEOUT = 20 - -# for python 3.5+, there was a change to the definition of the socket timeout (as far as socket.sendall is concerned) -# The socket timeout is now the maximum total duration to send all data. -if sys.version_info >= (3, 5): - # the timeout to connect is 20 seconds, and the read timeout is 2000 seconds - # the 2000 seconds was calculated with: 100MB (max block size)/ 50KB/s (an arbitrarily chosen minimum upload speed) - READ_TIMEOUT = 2000 - -STORAGE_OAUTH_SCOPE = "https://storage.azure.com/.default" - -SERVICE_HOST_BASE = 'core.windows.net' diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/models.py b/sdk/tables/azure-data-tables/azure/data/tables/_shared/models.py deleted file mode 100644 index c7ff9db66035..000000000000 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/models.py +++ /dev/null @@ -1,350 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- -from enum import Enum - - -def get_enum_value(value): - if value is None or value in ["None", ""]: - return None - try: - return value.value - except AttributeError: - return value - - -class TableErrorCode(str, Enum): - # Generic storage values - account_already_exists = "AccountAlreadyExists" - account_being_created = "AccountBeingCreated" - account_is_disabled = "AccountIsDisabled" - authentication_failed = "AuthenticationFailed" - authorization_failure = "AuthorizationFailure" - no_authentication_information = "NoAuthenticationInformation" - condition_headers_not_supported = "ConditionHeadersNotSupported" - condition_not_met = "ConditionNotMet" - empty_metadata_key = "EmptyMetadataKey" - insufficient_account_permissions = "InsufficientAccountPermissions" - internal_error = "InternalError" - invalid_authentication_info = "InvalidAuthenticationInfo" - invalid_header_value = "InvalidHeaderValue" - invalid_http_verb = "InvalidHttpVerb" - invalid_input = "InvalidInput" - invalid_md5 = "InvalidMd5" - invalid_metadata = "InvalidMetadata" - invalid_query_parameter_value = "InvalidQueryParameterValue" - invalid_range = "InvalidRange" - invalid_resource_name = "InvalidResourceName" - invalid_uri = "InvalidUri" - invalid_xml_document = "InvalidXmlDocument" - invalid_xml_node_value = "InvalidXmlNodeValue" - md5_mismatch = "Md5Mismatch" - metadata_too_large = "MetadataTooLarge" - missing_content_length_header = "MissingContentLengthHeader" - missing_required_query_parameter = "MissingRequiredQueryParameter" - missing_required_header = "MissingRequiredHeader" - missing_required_xml_node = "MissingRequiredXmlNode" - multiple_condition_headers_not_supported = "MultipleConditionHeadersNotSupported" - operation_timed_out = "OperationTimedOut" - out_of_range_input = "OutOfRangeInput" - out_of_range_query_parameter_value = "OutOfRangeQueryParameterValue" - request_body_too_large = "RequestBodyTooLarge" - resource_type_mismatch = "ResourceTypeMismatch" - request_url_failed_to_parse = "RequestUrlFailedToParse" - resource_already_exists = "ResourceAlreadyExists" - resource_not_found = "ResourceNotFound" - server_busy = "ServerBusy" - unsupported_header = "UnsupportedHeader" - unsupported_xml_node = "UnsupportedXmlNode" - unsupported_query_parameter = "UnsupportedQueryParameter" - unsupported_http_verb = "UnsupportedHttpVerb" - - # table error codes - duplicate_properties_specified = "DuplicatePropertiesSpecified" - entity_not_found = "EntityNotFound" - entity_already_exists = "EntityAlreadyExists" - entity_too_large = "EntityTooLarge" - host_information_not_present = "HostInformationNotPresent" - invalid_duplicate_row = "InvalidDuplicateRow" - invalid_value_type = "InvalidValueType" - json_format_not_supported = "JsonFormatNotSupported" - method_not_allowed = "MethodNotAllowed" - not_implemented = "NotImplemented" - properties_need_value = "PropertiesNeedValue" - property_name_invalid = "PropertyNameInvalid" - property_name_too_long = "PropertyNameTooLong" - property_value_too_large = "PropertyValueTooLarge" - table_already_exists = "TableAlreadyExists" - table_being_deleted = "TableBeingDeleted" - table_not_found = "TableNotFound" - too_many_properties = "TooManyProperties" - update_condition_not_satisfied = "UpdateConditionNotSatisfied" - x_method_incorrect_count = "XMethodIncorrectCount" - x_method_incorrect_value = "XMethodIncorrectValue" - x_method_not_using_post = "XMethodNotUsingPost" - - -class DictMixin(object): - - def __setitem__(self, key, item): - self.__dict__[key] = item - - def __getitem__(self, key): - return self.__dict__[key] - - def __repr__(self): - return str(self) - - def __len__(self): - return len(self.keys()) - - def __delitem__(self, key): - self.__dict__[key] = None - - def __eq__(self, other): - """Compare objects by comparing all attributes.""" - if isinstance(other, self.__class__): - return self.__dict__ == other.__dict__ - return False - - def __ne__(self, other): - """Compare objects by comparing all attributes.""" - return not self.__eq__(other) - - def __str__(self): - return str({k: v for k, v in self.__dict__.items() if not k.startswith('_')}) - - def has_key(self, k): - return k in self.__dict__ - - def update(self, *args, **kwargs): - return self.__dict__.update(*args, **kwargs) - - def keys(self): - return [k for k in self.__dict__ if not k.startswith('_')] - - def values(self): - return [v for k, v in self.__dict__.items() if not k.startswith('_')] - - def items(self): - return [(k, v) for k, v in self.__dict__.items() if not k.startswith('_')] - - def get(self, key, default=None): - if key in self.__dict__: - return self.__dict__[key] - return default - - -class LocationMode(object): - """ - Specifies the location the request should be sent to. This mode only applies - for RA-GRS accounts which allow secondary read access. All other account types - must use PRIMARY. - """ - - PRIMARY = 'primary' #: Requests should be sent to the primary location. - SECONDARY = 'secondary' #: Requests should be sent to the secondary location, if possible. - - -class ResourceTypes(object): - """ - Specifies the resource types that are accessible with the account SAS. - - :param bool service: - Access to service-level APIs (e.g., Get/Set Service Properties, - Get Service Stats, List Containers/Queues/Shares) - :param bool object: - Access to object-level APIs for blobs, queue messages, and - files(e.g. Put Blob, Query Entity, Get Messages, Create File, etc.) - """ - - def __init__(self, service=False, object=False): # pylint: disable=redefined-builtin - self.service = service - self.object = object - self._str = (('s' if self.service else '') + - ('o' if self.object else '')) - - def __str__(self): - return self._str - - @classmethod - def from_string(cls, string): - """Create a ResourceTypes from a string. - - To specify service, container, or object you need only to - include the first letter of the word in the string. E.g. service and container, - you would provide a string "sc". - - :param str string: Specify service, container, or object in - in the string with the first letter of the word. - :return: A ResourceTypes object - :rtype: ~azure.data.tables.ResourceTypes - """ - res_service = 's' in string - res_object = 'o' in string - - parsed = cls(res_service, res_object) - parsed._str = string # pylint: disable = protected-access - return parsed - - -class AccountSasPermissions(object): - """ - :class:`~ResourceTypes` class to be used with generate_account_sas - function and for the AccessPolicies used with set_*_acl. There are two types of - SAS which may be used to grant resource access. One is to grant access to a - specific resource (resource-specific). Another is to grant access to the - entire service for a specific account and allow certain operations based on - perms found here. - - :ivar bool read: - Valid for all signed resources types (Service, Container, and Object). - Permits read permissions to the specified resource type. - :ivar bool write: - Valid for all signed resources types (Service, Container, and Object). - Permits write permissions to the specified resource type. - :ivar bool delete: - Valid for Container and Object resource types, except for queue messages. - :ivar bool list: - Valid for Service and Container resource types only. - :ivar bool add: - Valid for the following Object resource types only: queue messages, and append blobs. - :ivar bool create: - Valid for the following Object resource types only: blobs and files. - Users can create new blobs or files, but may not overwrite existing - blobs or files. - :ivar bool update: - Valid for the following Object resource types only: queue messages. - :ivar bool process: - Valid for the following Object resource type only: queue messages. - """ - - def __init__(self, **kwargs): # pylint: disable=redefined-builtin - self.read = kwargs.pop('read', None) - self.write = kwargs.pop('write', None) - self.delete = kwargs.pop('delete', None) - self.list = kwargs.pop('list', None) - self.add = kwargs.pop('add', None) - self.create = kwargs.pop('create', None) - self.update = kwargs.pop('update', None) - self.process = kwargs.pop('process', None) - self._str = (('r' if self.read else '') + - ('w' if self.write else '') + - ('d' if self.delete else '') + - ('l' if self.list else '') + - ('a' if self.add else '') + - ('c' if self.create else '') + - ('u' if self.update else '') + - ('p' if self.process else '')) - - def __str__(self): - return self._str - - @classmethod - def from_string(cls, permission, **kwargs): # pylint:disable=W0613 - """Create AccountSasPermissions from a string. - - To specify read, write, delete, etc. permissions you need only to - include the first letter of the word in the string. E.g. for read and write - permissions you would provide a string "rw". - - :param str permission: Specify permissions in - the string with the first letter of the word. - :keyword callable cls: A custom type or function that will be passed the direct response - :return: A AccountSasPermissions object - :rtype: ~azure.data.tables.AccountSasPermissions - """ - p_read = 'r' in permission - p_write = 'w' in permission - p_delete = 'd' in permission - p_list = 'l' in permission - p_add = 'a' in permission - p_create = 'c' in permission - p_update = 'u' in permission - p_process = 'p' in permission - - parsed = cls( - **dict(kwargs, read=p_read, write=p_write, delete=p_delete, list=p_list, add=p_add, create=p_create, - update=p_update, process=p_process)) - parsed._str = permission # pylint: disable = protected-access - return parsed - - -class Services(object): - """Specifies the services accessible with the account SAS. - - :param bool blob: - Access for the `~azure.storage.blob.BlobServiceClient` - :param bool queue: - Access for the `~azure.data.tables.QueueServiceClient` - :param bool fileshare: - Access for the `~azure.storage.fileshare.ShareServiceClient` - """ - - def __init__(self, blob=False, queue=False, fileshare=False): - self.blob = blob - self.queue = queue - self.fileshare = fileshare - self._str = (('b' if self.blob else '') + - ('q' if self.queue else '') + - ('f' if self.fileshare else '')) - - def __str__(self): - return self._str - - @classmethod - def from_string(cls, string): - """Create Services from a string. - - To specify blob, queue, or file you need only to - include the first letter of the word in the string. E.g. for blob and queue - you would provide a string "bq". - - :param str string: Specify blob, queue, or file in - in the string with the first letter of the word. - :return: A Services object - :rtype: ~azure.data.tables.Services - """ - res_blob = 'b' in string - res_queue = 'q' in string - res_file = 'f' in string - - parsed = cls(res_blob, res_queue, res_file) - parsed._str = string # pylint: disable = protected-access - return parsed - - -class UserDelegationKey(object): - """ - Represents a user delegation key, provided to the user by Azure Storage - based on their Azure Active Directory access token. - - The fields are saved as simple strings since the user does not have to interact with this object; - to generate an identify SAS, the user can simply pass it to the right API. - - :ivar str signed_oid: - Object ID of this token. - :ivar str signed_tid: - Tenant ID of the tenant that issued this token. - :ivar str signed_start: - The datetime this token becomes valid. - :ivar str signed_expiry: - The datetime this token expires. - :ivar str signed_service: - What service this key is valid for. - :ivar str signed_version: - The version identifier of the REST service that created this token. - :ivar str value: - The user delegation key. - """ - - def __init__(self): - self.signed_oid = None - self.signed_tid = None - self.signed_start = None - self.signed_expiry = None - self.signed_service = None - self.signed_version = None - self.value = None diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/parser.py b/sdk/tables/azure-data-tables/azure/data/tables/_shared/parser.py deleted file mode 100644 index c6feba8a6393..000000000000 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/parser.py +++ /dev/null @@ -1,20 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -import sys - -if sys.version_info < (3,): - def _str(value): - if isinstance(value, unicode): # pylint: disable=undefined-variable - return value.encode('utf-8') - - return str(value) -else: - _str = str - - -def _to_utc_datetime(value): - return value.strftime('%Y-%m-%dT%H:%M:%SZ') diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/request_handlers.py b/sdk/tables/azure-data-tables/azure/data/tables/_shared/request_handlers.py deleted file mode 100644 index 2ce74d43db21..000000000000 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/request_handlers.py +++ /dev/null @@ -1,147 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -from typing import ( # pylint: disable=unused-import - Union, Optional, Any, Iterable, Dict, List, Type, Tuple, - TYPE_CHECKING -) - -import logging -from os import fstat -from io import (SEEK_END, SEEK_SET, UnsupportedOperation) - -import isodate - -from azure.core.exceptions import raise_with_traceback - - -_LOGGER = logging.getLogger(__name__) - - -def serialize_iso(attr): - """Serialize Datetime object into ISO-8601 formatted string. - - :param Datetime attr: Object to be serialized. - :rtype: str - :raises: ValueError if format invalid. - """ - if not attr: - return None - if isinstance(attr, str): - attr = isodate.parse_datetime(attr) - try: - utc = attr.utctimetuple() - if utc.tm_year > 9999 or utc.tm_year < 1: - raise OverflowError("Hit max or min date") - - date = "{:04}-{:02}-{:02}T{:02}:{:02}:{:02}".format( - utc.tm_year, utc.tm_mon, utc.tm_mday, - utc.tm_hour, utc.tm_min, utc.tm_sec) - return date + 'Z' - except (ValueError, OverflowError) as err: - msg = "Unable to serialize datetime object." - raise_with_traceback(ValueError, msg, err) - except AttributeError as err: - msg = "ISO-8601 object must be valid Datetime object." - raise_with_traceback(TypeError, msg, err) - - -def get_length(data): - length = None - # Check if object implements the __len__ method, covers most input cases such as bytearray. - try: - length = len(data) - except: # pylint: disable=bare-except - pass - - if not length: - # Check if the stream is a file-like stream object. - # If so, calculate the size using the file descriptor. - try: - fileno = data.fileno() - except (AttributeError, UnsupportedOperation): - pass - else: - try: - return fstat(fileno).st_size - except OSError: - # Not a valid fileno, may be possible requests returned - # a socket number? - pass - - # If the stream is seekable and tell() is implemented, calculate the stream size. - try: - current_position = data.tell() - data.seek(0, SEEK_END) - length = data.tell() - current_position - data.seek(current_position, SEEK_SET) - except (AttributeError, UnsupportedOperation): - pass - - return length - - -def read_length(data): - try: - if hasattr(data, 'read'): - read_data = b'' - for chunk in iter(lambda: data.read(4096), b""): - read_data += chunk - return len(read_data), read_data - if hasattr(data, '__iter__'): - read_data = b'' - for chunk in data: - read_data += chunk - return len(read_data), read_data - except: # pylint: disable=bare-except - pass - raise ValueError("Unable to calculate content length, please specify.") - - -def validate_and_format_range_headers( - start_range, end_range, start_range_required=True, - end_range_required=True, check_content_md5=False, align_to_page=False): - # If end range is provided, start range must be provided - if (start_range_required or end_range is not None) and start_range is None: - raise ValueError("start_range value cannot be None.") - if end_range_required and end_range is None: - raise ValueError("end_range value cannot be None.") - - # Page ranges must be 512 aligned - if align_to_page: - if start_range is not None and start_range % 512 != 0: - raise ValueError("Invalid page blob start_range: {0}. " - "The size must be aligned to a 512-byte boundary.".format(start_range)) - if end_range is not None and end_range % 512 != 511: - raise ValueError("Invalid page blob end_range: {0}. " - "The size must be aligned to a 512-byte boundary.".format(end_range)) - - # Format based on whether end_range is present - range_header = None - if end_range is not None: - range_header = 'bytes={0}-{1}'.format(start_range, end_range) - elif start_range is not None: - range_header = "bytes={0}-".format(start_range) - - # Content MD5 can only be provided for a complete range less than 4MB in size - range_validation = None - if check_content_md5: - if start_range is None or end_range is None: - raise ValueError("Both start and end range requied for MD5 content validation.") - if end_range - start_range > 4 * 1024 * 1024: - raise ValueError("Getting content MD5 for a range greater than 4MB is not supported.") - range_validation = 'true' - - return range_header, range_validation - - -def add_metadata_headers(metadata=None): - # type: (Optional[Dict[str, str]]) -> Dict[str, str] - headers = {} - if metadata: - for key, value in metadata.items(): - headers['x-ms-meta-{}'.format(key)] = value - return headers diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/response_handlers.py b/sdk/tables/azure-data-tables/azure/data/tables/_shared/response_handlers.py deleted file mode 100644 index 2c542ac3c7a7..000000000000 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/response_handlers.py +++ /dev/null @@ -1,159 +0,0 @@ -# ------------------------------------------------------------------------- -# Copyright (c) Microsoft Corporation. All rights reserved. -# Licensed under the MIT License. See License.txt in the project root for -# license information. -# -------------------------------------------------------------------------- - -from typing import ( # pylint: disable=unused-import - Union, Optional, Any, Iterable, Dict, List, Type, Tuple, - TYPE_CHECKING -) -import logging - -from azure.core.pipeline.policies import ContentDecodePolicy -from azure.core.exceptions import ( - HttpResponseError, - ResourceNotFoundError, - ResourceModifiedError, - ResourceExistsError, - ClientAuthenticationError, - DecodeError) - -from .parser import _to_utc_datetime -from .models import TableErrorCode, UserDelegationKey, get_enum_value - - -if TYPE_CHECKING: - from datetime import datetime - from azure.core.exceptions import AzureError - - -_LOGGER = logging.getLogger(__name__) - - -class PartialBatchErrorException(HttpResponseError): - """There is a partial failure in batch operations. - - :param str message: The message of the exception. - :param response: Server response to be deserialized. - :param list parts: A list of the parts in multipart response. - """ - - def __init__(self, message, response, parts): - self.parts = parts - super(PartialBatchErrorException, self).__init__(message=message, response=response) - - -def parse_length_from_content_range(content_range): - ''' - Parses the blob length from the content range header: bytes 1-3/65537 - ''' - if content_range is None: - return None - - # First, split in space and take the second half: '1-3/65537' - # Next, split on slash and take the second half: '65537' - # Finally, convert to an int: 65537 - return int(content_range.split(' ', 1)[1].split('/', 1)[1]) - - -def normalize_headers(headers): - normalized = {} - for key, value in headers.items(): - if key.startswith('x-ms-'): - key = key[5:] - normalized[key.lower().replace('-', '_')] = get_enum_value(value) - return normalized - - -def deserialize_metadata(response, obj, headers): # pylint: disable=unused-argument - raw_metadata = {k: v for k, v in response.headers.items() if k.startswith("x-ms-meta-")} - return {k[10:]: v for k, v in raw_metadata.items()} - - -def return_response_headers(response, deserialized, response_headers): # pylint: disable=unused-argument - return normalize_headers(response_headers) - - -def return_headers_and_deserialized(response, deserialized, response_headers): # pylint: disable=unused-argument - return normalize_headers(response_headers), deserialized - - -def return_context_and_deserialized(response, deserialized, response_headers): # pylint: disable=unused-argument - return response.http_response.location_mode, deserialized, response_headers - - -def process_table_error(storage_error): - raise_error = HttpResponseError - error_code = storage_error.response.headers.get('x-ms-error-code') - error_message = storage_error.message - additional_data = {} - try: - error_body = ContentDecodePolicy.deserialize_from_http_generics(storage_error.response) - if isinstance(error_body, dict): - for info in error_body['odata.error']: - if info == 'code': - error_code = error_body['odata.error'][info] - elif info == 'message': - error_message = error_body['odata.error'][info]['value'] - else: - additional_data[info.tag] = info.text - else: - if error_body: - for info in error_body.iter(): - if info.tag.lower().find('code') != -1: - error_code = info.text - elif info.tag.lower().find('message') != -1: - error_message = info.text - else: - additional_data[info.tag] = info.text - except DecodeError: - pass - - try: - if error_code: - error_code = TableErrorCode(error_code) - if error_code in [TableErrorCode.condition_not_met]: - raise_error = ResourceModifiedError - if error_code in [TableErrorCode.invalid_authentication_info, - TableErrorCode.authentication_failed]: - raise_error = ClientAuthenticationError - if error_code in [TableErrorCode.resource_not_found, - TableErrorCode.table_not_found, - TableErrorCode.entity_not_found, - ResourceNotFoundError]: - raise_error = ResourceNotFoundError - if error_code in [TableErrorCode.resource_already_exists, - TableErrorCode.table_already_exists, - TableErrorCode.account_already_exists, - TableErrorCode.entity_already_exists, - ResourceExistsError]: - raise_error = ResourceExistsError - except ValueError: - # Got an unknown error code - pass - - try: - - error_message += "\nErrorCode:{}".format(error_code.value) - except AttributeError: - error_message += "\nErrorCode:{}".format(error_code) - for name, info in additional_data.items(): - error_message += "\n{}:{}".format(name, info) - - error = raise_error(message=error_message, response=storage_error.response) - error.error_code = error_code - error.additional_info = additional_data - raise error - - -def parse_to_internal_user_delegation_key(service_user_delegation_key): - internal_user_delegation_key = UserDelegationKey() - internal_user_delegation_key.signed_oid = service_user_delegation_key.signed_oid - internal_user_delegation_key.signed_tid = service_user_delegation_key.signed_tid - internal_user_delegation_key.signed_start = _to_utc_datetime(service_user_delegation_key.signed_start) - internal_user_delegation_key.signed_expiry = _to_utc_datetime(service_user_delegation_key.signed_expiry) - internal_user_delegation_key.signed_service = service_user_delegation_key.signed_service - internal_user_delegation_key.signed_version = service_user_delegation_key.signed_version - internal_user_delegation_key.value = service_user_delegation_key.value - return internal_user_delegation_key diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/shared_access_signature.py b/sdk/tables/azure-data-tables/azure/data/tables/_shared_access_signature.py similarity index 99% rename from sdk/tables/azure-data-tables/azure/data/tables/_shared/shared_access_signature.py rename to sdk/tables/azure-data-tables/azure/data/tables/_shared_access_signature.py index 9ce7a0cf9f1a..4192426de914 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/shared_access_signature.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_shared_access_signature.py @@ -5,7 +5,7 @@ # -------------------------------------------------------------------------- from datetime import date -from azure.data.tables._shared import url_quote +from ._deserialize import url_quote from ._common_conversion import ( diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_table_client.py b/sdk/tables/azure-data-tables/azure/data/tables/_table_client.py index ff0918cfd051..a8ee3a33c212 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_table_client.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_table_client.py @@ -16,23 +16,20 @@ from azure.core.paging import ItemPaged from azure.core.exceptions import HttpResponseError, ResourceNotFoundError from azure.core.tracing.decorator import distributed_trace + from ._deserialize import _convert_to_entity from ._entity import TableEntity from ._generated import AzureTable from ._generated.models import AccessPolicy, SignedIdentifier, TableProperties, QueryOptions from ._serialize import _get_match_headers, _add_entity_properties -from ._shared.base_client import parse_connection_str -from ._shared._table_client_base import TableClientBase - -from ._shared.request_handlers import serialize_iso -from ._shared.response_handlers import process_table_error - +from ._base_client import parse_connection_str +from ._table_client_base import TableClientBase +from ._serialize import serialize_iso +from ._deserialize import _return_headers_and_deserialized +from ._error import _process_table_error from ._version import VERSION - from ._models import TableEntityPropertiesPaged, UpdateMode, Table -from ._shared.response_handlers import return_headers_and_deserialized - class TableClient(TableClientBase): """ :ivar str account_name: Name of the storage account (Cosmos or Azure)""" @@ -144,10 +141,10 @@ def get_table_access_policy( _, identifiers = self._client.table.get_access_policy( table=self.table_name, timeout=timeout, - cls=return_headers_and_deserialized, + cls=kwargs.pop('cls', None) or _return_headers_and_deserialized, **kwargs) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) return {s.id: s.access_policy or AccessPolicy() for s in identifiers} # pylint: disable=E1125 @distributed_trace @@ -178,7 +175,7 @@ def set_table_access_policy( table_acl=signed_identifiers or None, **kwargs) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace def create_table( @@ -197,7 +194,7 @@ def create_table( table = self._client.table.create(table_properties) return Table(table=table) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace def delete_table( @@ -213,7 +210,7 @@ def delete_table( try: self._client.table.delete(table=self.table_name, **kwargs) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace def delete_entity( @@ -247,7 +244,7 @@ def delete_entity( if_match=if_match or if_not_match or '*', **kwargs) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace def create_entity( @@ -279,7 +276,7 @@ def create_entity( properties = _convert_to_entity(inserted_entity) return properties except ResourceNotFoundError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace def update_entity( # pylint:disable=R1710 @@ -327,7 +324,7 @@ def update_entity( # pylint:disable=R1710 else: raise ValueError('Mode type is not supported') except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace def list_entities( @@ -419,7 +416,7 @@ def get_entity( properties = _convert_to_entity(entity.additional_properties) return properties except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace def upsert_entity( # pylint:disable=R1710 diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/_table_client_base.py b/sdk/tables/azure-data-tables/azure/data/tables/_table_client_base.py similarity index 95% rename from sdk/tables/azure-data-tables/azure/data/tables/_shared/_table_client_base.py rename to sdk/tables/azure-data-tables/azure/data/tables/_table_client_base.py index 54e0b360fac6..e8f2a896c87c 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/_table_client_base.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_table_client_base.py @@ -9,9 +9,9 @@ except ImportError: from urlparse import urlparse # type: ignore -from azure.data.tables._shared._error import _validate_table_name -from azure.data.tables._shared.base_client import parse_query -from .base_client import StorageAccountHostsMixin +from ._error import _validate_table_name +from ._base_client import parse_query +from ._base_client import StorageAccountHostsMixin class TableClientBase(StorageAccountHostsMixin): diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_table_service_client.py b/sdk/tables/azure-data-tables/azure/data/tables/_table_service_client.py index ce9e97c88a88..1c4c94303ef6 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_table_service_client.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_table_service_client.py @@ -15,13 +15,12 @@ from ._generated import AzureTable from ._generated.models import TableProperties, TableServiceProperties, QueryOptions from ._models import TablePropertiesPaged, service_stats_deserialize, service_properties_deserialize -from ._shared.base_client import parse_connection_str, TransportWrapper -from ._shared.models import LocationMode -from ._shared.response_handlers import process_table_error +from ._base_client import parse_connection_str, TransportWrapper +from ._models import LocationMode +from ._error import _process_table_error from ._version import VERSION - from ._table_client import TableClient -from ._shared._table_service_client_base import TableServiceClientBase +from ._table_service_client_base import TableServiceClientBase class TableServiceClient(TableServiceClientBase): @@ -85,7 +84,7 @@ def get_service_stats(self, **kwargs): timeout=timeout, use_location=LocationMode.SECONDARY, **kwargs) return service_stats_deserialize(stats) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace def get_service_properties(self, **kwargs): @@ -102,7 +101,7 @@ def get_service_properties(self, **kwargs): service_props = self._client.service.get_properties(timeout=timeout, **kwargs) # type: ignore return service_properties_deserialize(service_props) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace def set_service_properties( @@ -138,7 +137,7 @@ def set_service_properties( try: return self._client.service.set_properties(props, **kwargs) # type: ignore except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace def create_table( diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/_table_service_client_base.py b/sdk/tables/azure-data-tables/azure/data/tables/_table_service_client_base.py similarity index 96% rename from sdk/tables/azure-data-tables/azure/data/tables/_shared/_table_service_client_base.py rename to sdk/tables/azure-data-tables/azure/data/tables/_table_service_client_base.py index 99967f590e94..138fac3f0ceb 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/_table_service_client_base.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_table_service_client_base.py @@ -9,8 +9,7 @@ except ImportError: from urlparse import urlparse # type: ignore -from azure.data.tables._shared.base_client import parse_query -from .base_client import StorageAccountHostsMixin +from ._base_client import parse_query, StorageAccountHostsMixin class TableServiceClientBase(StorageAccountHostsMixin): diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/table_shared_access_signature.py b/sdk/tables/azure-data-tables/azure/data/tables/_table_shared_access_signature.py similarity index 96% rename from sdk/tables/azure-data-tables/azure/data/tables/_shared/table_shared_access_signature.py rename to sdk/tables/azure-data-tables/azure/data/tables/_table_shared_access_signature.py index 02e392faf8c3..6cf152dc8340 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/table_shared_access_signature.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/_table_shared_access_signature.py @@ -5,12 +5,11 @@ # -------------------------------------------------------------------------- from typing import Union -from azure.data.tables._shared.models import AccountSasPermissions -from azure.data.tables._models import TableServices -from azure.data.tables._shared._common_conversion import _sign_string -from azure.data.tables._shared._error import _validate_not_none -from azure.data.tables._shared.constants import X_MS_VERSION -from azure.data.tables._shared.shared_access_signature import _SharedAccessHelper,\ +from ._models import AccountSasPermissions +from ._common_conversion import _sign_string +from ._error import _validate_not_none +from ._constants import X_MS_VERSION +from ._shared_access_signature import _SharedAccessHelper,\ SharedAccessSignature, QueryStringConstants @@ -71,7 +70,7 @@ def generate_account_sas( if permission is str: permission = AccountSasPermissions.from_string(permission=permission) sas = TableSharedAccessSignature(account_name, account_key) - return sas.generate_account(TableServices(), resource_types, permission, + return sas.generate_account("t", resource_types, permission, expiry, start=kwargs.pop('start', None), ip_address_or_range=kwargs.pop('ip_address_or_range', None), protocol=kwargs.pop('protocol', None)) diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/base_client_async.py b/sdk/tables/azure-data-tables/azure/data/tables/aio/_base_client_async.py similarity index 93% rename from sdk/tables/azure-data-tables/azure/data/tables/_shared/base_client_async.py rename to sdk/tables/azure-data-tables/azure/data/tables/aio/_base_client_async.py index 18597c9a5a29..5e8020b84e95 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/base_client_async.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/aio/_base_client_async.py @@ -21,18 +21,18 @@ ) from azure.core.pipeline.transport import AsyncHttpTransport -from .constants import STORAGE_OAUTH_SCOPE, CONNECTION_TIMEOUT, READ_TIMEOUT -from .authentication import SharedKeyCredentialPolicy -from .base_client import create_configuration -from .policies import ( +from .._constants import STORAGE_OAUTH_SCOPE, CONNECTION_TIMEOUT, READ_TIMEOUT +from .._authentication import SharedKeyCredentialPolicy +from .._base_client import create_configuration +from .._policies import ( StorageContentValidation, StorageRequestHook, StorageHosts, StorageHeadersPolicy ) -from .policies_async import AsyncStorageResponseHook - -from .response_handlers import process_table_error, PartialBatchErrorException +from ._policies_async import AsyncStorageResponseHook +from .._error import _process_table_error +from .._models import PartialBatchErrorException if TYPE_CHECKING: from azure.core.pipeline import Pipeline @@ -147,7 +147,7 @@ async def _batch_send( return AsyncList(parts_list) return parts except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) class AsyncTransportWrapper(AsyncHttpTransport): diff --git a/sdk/tables/azure-data-tables/azure/data/tables/aio/_models.py b/sdk/tables/azure-data-tables/azure/data/tables/aio/_models.py index d41bfa187b06..fa19a7a1a4fd 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/aio/_models.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/aio/_models.py @@ -3,12 +3,15 @@ # Licensed under the MIT License. See License.txt in the project root for # license information. # -------------------------------------------------------------------------- -from azure.data.tables._models import Table -from azure.data.tables._deserialize import _convert_to_entity -from azure.data.tables._shared.response_handlers import return_context_and_deserialized, process_table_error from azure.core.exceptions import HttpResponseError from azure.core.async_paging import AsyncPageIterator +from .._deserialize import ( + _return_context_and_deserialized, + _convert_to_entity +) +from .._models import Table +from .._error import _process_table_error class TablePropertiesPaged(AsyncPageIterator): """An iterable of Table properties. @@ -36,11 +39,11 @@ async def _get_next_cb(self, continuation_token, **kwargs): try: return await self._command( next_table_name=continuation_token or None, - cls=kwargs.pop('cls', return_context_and_deserialized), + cls=kwargs.pop('cls', None) or _return_context_and_deserialized, use_location=self.location_mode ) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) async def _extract_data_cb(self, get_next_return): self.location_mode, self._response, self._headers = get_next_return @@ -74,7 +77,7 @@ def __init__(self, command, results_per_page=None, table=None, self.table = table self.location_mode = None - async def _get_next_cb(self, continuation_token): + async def _get_next_cb(self, continuation_token, **kwargs): row_key = "" partition_key = "" for key, value in continuation_token.items(): @@ -88,11 +91,11 @@ async def _get_next_cb(self, continuation_token): next_row_key=row_key or None, next_partition_key=partition_key or None, table=self.table, - cls=return_context_and_deserialized, + cls=kwargs.pop("cls", _return_context_and_deserialized), use_location=self.location_mode ) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) async def _extract_data_cb(self, get_next_return): self.location_mode, self._response, self._headers = get_next_return diff --git a/sdk/tables/azure-data-tables/azure/data/tables/_shared/policies_async.py b/sdk/tables/azure-data-tables/azure/data/tables/aio/_policies_async.py similarity index 99% rename from sdk/tables/azure-data-tables/azure/data/tables/_shared/policies_async.py rename to sdk/tables/azure-data-tables/azure/data/tables/aio/_policies_async.py index f627df43f9e4..88695af888aa 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/_shared/policies_async.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/aio/_policies_async.py @@ -12,7 +12,7 @@ from azure.core.pipeline.policies import AsyncHTTPPolicy from azure.core.exceptions import AzureError -from .policies import is_retry, StorageRetryPolicy +from .._policies import is_retry, StorageRetryPolicy if TYPE_CHECKING: from azure.core.pipeline import PipelineRequest, PipelineResponse diff --git a/sdk/tables/azure-data-tables/azure/data/tables/aio/_table_client_async.py b/sdk/tables/azure-data-tables/azure/data/tables/aio/_table_client_async.py index 00c3acd2c5ec..5adb7e22b7b6 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/aio/_table_client_async.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/aio/_table_client_async.py @@ -13,21 +13,22 @@ from azure.core.exceptions import ResourceNotFoundError, HttpResponseError from azure.core.tracing.decorator import distributed_trace from azure.core.tracing.decorator_async import distributed_trace_async -from azure.data.tables import VERSION -from azure.data.tables._entity import TableEntity -from azure.data.tables._generated.aio._azure_table_async import AzureTable -from azure.data.tables._generated.models import SignedIdentifier, TableProperties, QueryOptions -from azure.data.tables._models import AccessPolicy, Table -from azure.data.tables._shared.base_client_async import AsyncStorageAccountHostsMixin -from azure.data.tables._shared.policies_async import ExponentialRetry -from azure.data.tables._shared.request_handlers import serialize_iso -from azure.data.tables._shared.response_handlers import return_headers_and_deserialized, process_table_error +from .. import VERSION +from .._entity import TableEntity +from .._generated.aio import AzureTable +from .._generated.models import SignedIdentifier, TableProperties, QueryOptions +from .._models import AccessPolicy, Table +from .._serialize import serialize_iso +from .._deserialize import _return_headers_and_deserialized +from .._error import _process_table_error from .._models import UpdateMode -from ._models import TableEntityPropertiesPaged from .._deserialize import _convert_to_entity from .._serialize import _add_entity_properties, _get_match_headers -from .._shared._table_client_base import TableClientBase +from .._table_client_base import TableClientBase +from ._base_client_async import AsyncStorageAccountHostsMixin +from ._models import TableEntityPropertiesPaged +from ._policies_async import ExponentialRetry class TableClient(AsyncStorageAccountHostsMixin, TableClientBase): @@ -84,10 +85,10 @@ async def get_table_access_policy( _, identifiers = await self._client.table.get_access_policy( table=self.table_name, timeout=timeout, - cls=return_headers_and_deserialized, + cls=kwargs.pop('cls', None) or _return_headers_and_deserialized, **kwargs) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) return {s.id: s.access_policy or AccessPolicy() for s in identifiers} @distributed_trace_async @@ -118,7 +119,7 @@ async def set_table_access_policy( table_acl=signed_identifiers or None, **kwargs) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace_async async def create_table( @@ -136,7 +137,7 @@ async def create_table( table = await self._client.table.create(table_properties) return Table(table) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace_async async def delete_table( @@ -151,7 +152,7 @@ async def delete_table( try: await self._client.table.delete(table=self.table_name, **kwargs) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace_async async def delete_entity( @@ -183,7 +184,7 @@ async def delete_entity( if_match=if_match or if_not_match or '*', **kwargs) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace_async async def create_entity( @@ -214,7 +215,7 @@ async def create_entity( properties = _convert_to_entity(inserted_entity) return properties except ResourceNotFoundError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace_async async def update_entity( @@ -264,7 +265,7 @@ async def update_entity( else: raise ValueError('Mode type is not supported') except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace def list_entities( @@ -355,7 +356,7 @@ async def get_entity( properties = _convert_to_entity(entity.additional_properties) return properties except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace_async async def upsert_entity( diff --git a/sdk/tables/azure-data-tables/azure/data/tables/aio/_table_service_client_async.py b/sdk/tables/azure-data-tables/azure/data/tables/aio/_table_service_client_async.py index 7c58d21a8a94..1901b0f1903a 100644 --- a/sdk/tables/azure-data-tables/azure/data/tables/aio/_table_service_client_async.py +++ b/sdk/tables/azure-data-tables/azure/data/tables/aio/_table_service_client_async.py @@ -15,18 +15,18 @@ from azure.core.pipeline import AsyncPipeline from azure.core.tracing.decorator import distributed_trace from azure.core.tracing.decorator_async import distributed_trace_async -from azure.data.tables import VERSION, LocationMode -from azure.data.tables._generated.aio._azure_table_async import AzureTable -from azure.data.tables._generated.models import TableServiceProperties, TableProperties, QueryOptions -from azure.data.tables._models import service_stats_deserialize, service_properties_deserialize -from azure.data.tables._shared.base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper -from azure.data.tables._shared.policies_async import ExponentialRetry -from azure.data.tables._shared.response_handlers import process_table_error -from azure.data.tables.aio._table_client_async import TableClient -from ._models import TablePropertiesPaged -from .._shared._error import _validate_table_name -from .._shared._table_service_client_base import TableServiceClientBase + +from .. import VERSION, LocationMode +from .._generated.aio._azure_table_async import AzureTable +from .._generated.models import TableServiceProperties, TableProperties, QueryOptions +from .._models import service_stats_deserialize, service_properties_deserialize +from .._error import _validate_table_name, _process_table_error +from .._table_service_client_base import TableServiceClientBase from .._models import Table +from ._policies_async import ExponentialRetry +from ._table_client_async import TableClient +from ._base_client_async import AsyncStorageAccountHostsMixin, AsyncTransportWrapper +from ._models import TablePropertiesPaged class TableServiceClient(AsyncStorageAccountHostsMixin, TableServiceClientBase): @@ -103,7 +103,7 @@ async def get_service_stats(self, **kwargs): timeout=timeout, use_location=LocationMode.SECONDARY, **kwargs) return service_stats_deserialize(stats) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace_async async def get_service_properties(self, **kwargs): @@ -121,7 +121,7 @@ async def get_service_properties(self, **kwargs): service_props = await self._client.service.get_properties(timeout=timeout, **kwargs) # type: ignore return service_properties_deserialize(service_props) except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace_async async def set_service_properties( @@ -157,7 +157,7 @@ async def set_service_properties( try: return await self._client.service.set_properties(props, **kwargs) # type: ignore except HttpResponseError as error: - process_table_error(error) + _process_table_error(error) @distributed_trace_async async def create_table( diff --git a/sdk/tables/azure-data-tables/samples/creation_deletion_of_table.py b/sdk/tables/azure-data-tables/samples/creation_deletion_of_table.py index e9a614968af4..018c703b5894 100644 --- a/sdk/tables/azure-data-tables/samples/creation_deletion_of_table.py +++ b/sdk/tables/azure-data-tables/samples/creation_deletion_of_table.py @@ -16,10 +16,12 @@ def connection_string_auth(self): table_service_client = TableServiceClient.from_connection_string(conn_str=self.connection_string) def sas_token_auth(self): - from azure.data.tables import TableServiceClient - from azure.data.tables._shared.table_shared_access_signature import generate_account_sas - from azure.data.tables import ResourceTypes - from azure.data.tables import AccountSasPermissions + from azure.data.tables import ( + TableServiceClient, + ResourceTypes, + AccountSasPermissions, + generate_account_sas + ) import datetime import timedelta diff --git a/sdk/tables/azure-data-tables/tests/_shared/testcase.py b/sdk/tables/azure-data-tables/tests/_shared/testcase.py index 69d43165658d..3dcfde01bf48 100644 --- a/sdk/tables/azure-data-tables/tests/_shared/testcase.py +++ b/sdk/tables/azure-data-tables/tests/_shared/testcase.py @@ -14,7 +14,7 @@ from datetime import datetime, timedelta from azure.data.tables import ResourceTypes, AccountSasPermissions -from azure.data.tables._shared.table_shared_access_signature import generate_account_sas +from azure.data.tables._table_shared_access_signature import generate_account_sas try: import unittest.mock as mock diff --git a/sdk/tables/azure-data-tables/tests/test_table.py b/sdk/tables/azure-data-tables/tests/test_table.py index 63a76c8dc315..bb6acef68892 100644 --- a/sdk/tables/azure-data-tables/tests/test_table.py +++ b/sdk/tables/azure-data-tables/tests/test_table.py @@ -15,9 +15,18 @@ timedelta, ) -from azure.data.tables._models import TableSasPermissions, UpdateMode, AccessPolicy, TableAnalyticsLogging, Metrics, CorsRule, \ - RetentionPolicy -from azure.data.tables._shared.models import ResourceTypes, AccountSasPermissions + +from azure.data.tables import ( + ResourceTypes, + AccountSasPermissions, + TableSasPermissions, + CorsRule, + RetentionPolicy, + UpdateMode, + AccessPolicy, + TableAnalyticsLogging, + Metrics +) from azure.core.pipeline import Pipeline from azure.core.pipeline.policies import ( HeadersPolicy, @@ -25,7 +34,7 @@ ) from _shared.testcase import TableTestCase, GlobalStorageAccountPreparer -from azure.data.tables._shared.authentication import SharedKeyCredentialPolicy +from azure.data.tables._authentication import SharedKeyCredentialPolicy from azure.core.pipeline.transport import RequestsTransport from azure.core.exceptions import ( HttpResponseError, @@ -33,7 +42,7 @@ ResourceExistsError) # ------------------------------------------------------------------------------ -from azure.data.tables._shared.table_shared_access_signature import generate_account_sas +from azure.data.tables._table_shared_access_signature import generate_account_sas TEST_TABLE_PREFIX = 'pytablesync' diff --git a/sdk/tables/azure-data-tables/tests/test_table_async.py b/sdk/tables/azure-data-tables/tests/test_table_async.py index 5b6de7387b22..061122548b9a 100644 --- a/sdk/tables/azure-data-tables/tests/test_table_async.py +++ b/sdk/tables/azure-data-tables/tests/test_table_async.py @@ -10,7 +10,7 @@ from azure.data.tables import AccessPolicy, TableSasPermissions, ResourceTypes, AccountSasPermissions from azure.data.tables.aio import TableServiceClient from azure.data.tables._generated.models import QueryOptions -from azure.data.tables._shared.table_shared_access_signature import generate_account_sas +from azure.data.tables._table_shared_access_signature import generate_account_sas TEST_TABLE_PREFIX = 'pytableasync' diff --git a/sdk/tables/azure-data-tables/tests/test_table_encryption.py b/sdk/tables/azure-data-tables/tests/test_table_encryption.py deleted file mode 100644 index 008f0ea3743c..000000000000 --- a/sdk/tables/azure-data-tables/tests/test_table_encryption.py +++ /dev/null @@ -1,977 +0,0 @@ -# # coding: utf-8 -# -# # ------------------------------------------------------------------------- -# # Copyright (c) Microsoft Corporation. All rights reserved. -# # Licensed under the MIT License. See License.txt in the project root for -# # license information. -# # -------------------------------------------------------------------------- -# -# import unittest -# -# import pytest -# from datetime import datetime -# -# from azure.common import AzureException -# from azure.core.exceptions import ResourceExistsError -# from azure.data.tables import TableServiceClient -# from azure.data.tables._entity import EntityProperty, EdmType, Entity -# from azure.data.tables._models import TablePayloadFormat, AccessPolicy, TableSasPermissions, TableServices -# from azure.data.tables._shared._common_conversion import _encode_base64 -# from azure.data.tables._shared.encryption import _dict_to_encryption_data, _generate_AES_CBC_cipher -# from dateutil.tz import tzutc -# from os import urandom -# from json import loads -# from copy import deepcopy -# -# # from encryption_test_helper import KeyWrapper, KeyResolver, RSAKeyWrapper -# from testutils.common_recordingtestcase import TestMode -# -# pytestmark = pytest.mark.skip -# -# # from testcase import ( -# # TableTestCase, -# # TestMode, -# # record, -# # ) -# # from azure.data.tables import ( -# # Entity, -# # EntityProperty, -# # TableService, -# # EdmType, -# # TableBatch, -# # ) -# # from azure.storage.models import( -# # AccessPolicy, -# # ) -# # from tests.test_encryption_helper import( -# # KeyWrapper, -# # KeyResolver, -# # RSAKeyWrapper, -# # ) -# # from azure.storage.table.models import( -# # TablePayloadFormat, -# # TablePermissions, -# # ) -# from azure.data.tables._shared._error import ( -# _ERROR_UNSUPPORTED_TYPE_FOR_ENCRYPTION, -# ) -# from azure.data.tables._shared._error import ( -# _ERROR_OBJECT_INVALID, -# _ERROR_DECRYPTION_FAILURE, -# ) -# #Encyption not supported yet -# # from cryptography.hazmat.backends import default_backend -# # from cryptography.hazmat.primitives.ciphers.algorithms import AES -# # from cryptography.hazmat.primitives.ciphers.modes import CBC -# # from cryptography.hazmat.primitives.padding import PKCS7 -# # from cryptography.hazmat.primitives.ciphers import Cipher -# # from cryptography.hazmat.primitives.hashes import ( -# # Hash, -# # SHA256, -# # ) -# -# from _shared.testcase import GlobalStorageAccountPreparer, TableTestCase, LogCaptured -# -# -# class StorageTableEncryptionTest(TableTestCase): -# -# def _set_up(self, storage_account, storage_account_key): -# self.ts = TableServiceClient(self.account_url(storage_account, "table"), storage_account_key) -# self.table_name = self.get_resource_name('uttable') -# self.table = self.ts.get_table_client(self.table_name) -# if self.is_live: -# try: -# self.ts.create_table(table_name=self.table_name) -# except ResourceExistsError: -# pass -# -# self.query_tables = [] -# -# def _tear_down(self): -# if self.is_live: -# try: -# self.ts.delete_table(self.table_name) -# except: -# pass -# -# for table_name in self.query_tables: -# try: -# self.ts.delete_table(table_name) -# except: -# pass -# -# # --Helpers----------------------------------------------------------------- -# -# def _create_query_table_encrypted(self, entity_count): -# ''' -# Creates a table with the specified name and adds entities with the -# default set of values. PartitionKey is set to 'MyPartition' and RowKey -# is set to a unique counter value starting at 1 (as a string). The -# 'sex' attribute is set to be encrypted. -# ''' -# table_name = self.get_resource_name('querytable') -# self.ts.create_table(table_name, True) -# self.query_tables.append(table_name) -# self.ts.require_encryption = True -# -# entity = self._create_default_entity_for_encryption() -# self.table.create_entity(table_entity_properties=entity) -# # with self.ts.batch(table_name) as batch: -# # for i in range(1, entity_count + 1): -# # entity['RowKey'] = entity['RowKey'] + str(i) -# # batch.insert_entity(entity) -# return table_name -# -# def _create_random_base_entity_class(self): -# ''' -# Creates a class-based entity with only pk and rk. -# ''' -# partition = self.get_resource_name('pk') -# row = self.get_resource_name('rk') -# entity = Entity() -# entity.PartitionKey = partition -# entity.RowKey = row -# return entity -# -# def _create_random_base_entity_dict(self): -# ''' -# Creates a dict-based entity with only pk and rk. -# ''' -# partition = self.get_resource_name('pk') -# row = self.get_resource_name('rk') -# return {'PartitionKey': partition, -# 'RowKey': row, -# } -# -# def _create_random_entity_class(self, pk=None, rk=None): -# ''' -# Creates a class-based entity with fixed values, using all -# of the supported data types. -# ''' -# partition = pk if pk is not None else self.get_resource_name('pk') -# row = rk if rk is not None else self.get_resource_name('rk') -# entity = Entity() -# entity.PartitionKey = partition -# entity.RowKey = row -# entity.age = 39 -# entity.sex = 'male' -# entity.name = 'John Doe' -# entity.married = True -# entity.deceased = False -# entity.optional = None -# entity.evenratio = 3.0 -# entity.ratio = 3.1 -# entity.large = 933311100 -# entity.Birthday = datetime(1973, 10, 4) -# entity.birthday = datetime(1970, 10, 4) -# entity.binary = EntityProperty(EdmType.BINARY, b'binary') -# entity.other = EntityProperty(EdmType.INT32, 20) -# entity.clsid = EntityProperty( -# EdmType.GUID, 'c9da6455-213d-42c9-9a79-3e9149a57833') -# return entity -# -# def _create_default_entity_for_encryption(self): -# entity = self._create_random_entity_class() -# entity['sex'] = EntityProperty(EdmType.STRING, entity['sex'], True) -# entity['name'] = EntityProperty(EdmType.STRING, entity['name'], True) -# return entity -# -# def _create_default_entity_dict(self, pk=None, rk=None): -# ''' -# Creates a dictionary-based entity with fixed values, using all -# of the supported data types. -# ''' -# partition = pk if pk is not None else self.get_resource_name('pk') -# row = rk if rk is not None else self.get_resource_name('rk') -# return {'PartitionKey': partition, -# 'RowKey': row, -# 'age': 39, -# 'sex': 'male', -# 'name': 'John Doe', -# 'married': True, -# 'deceased': False, -# 'optional': None, -# 'ratio': 3.1, -# 'evenratio': 3.0, -# 'large': 933311100, -# 'Birthday': datetime(1973, 10, 4), -# 'birthday': datetime(1970, 10, 4), -# 'binary': EntityProperty(EdmType.BINARY, b'binary'), -# 'other': EntityProperty(EdmType.INT32, 20), -# 'clsid': EntityProperty( -# EdmType.GUID, -# 'c9da6455-213d-42c9-9a79-3e9149a57833')} -# -# def _assert_default_entity(self, entity): -# ''' -# Asserts that the entity passed in matches the default entity. -# ''' -# self.assertEqual(entity.age, 39) -# self.assertEqual(entity.sex, 'male') -# self.assertEqual(entity.name, 'John Doe') -# self.assertEqual(entity.married, True) -# self.assertEqual(entity.deceased, False) -# self.assertFalse(hasattr(entity, "optional")) -# self.assertFalse(hasattr(entity, "aquarius")) -# self.assertEqual(entity.ratio, 3.1) -# self.assertEqual(entity.evenratio, 3.0) -# self.assertEqual(entity.large, 933311100) -# self.assertEqual(entity.Birthday, datetime(1973, 10, 4, tzinfo=tzutc())) -# self.assertEqual(entity.birthday, datetime(1970, 10, 4, tzinfo=tzutc())) -# self.assertIsInstance(entity.binary, EntityProperty) -# self.assertEqual(entity.binary.type, EdmType.BINARY) -# self.assertEqual(entity.binary.value, b'binary') -# self.assertIsInstance(entity.other, EntityProperty) -# self.assertEqual(entity.other.type, EdmType.INT32) -# self.assertEqual(entity.other.value, 20) -# self.assertIsInstance(entity.clsid, EntityProperty) -# self.assertEqual(entity.clsid.type, EdmType.GUID) -# self.assertEqual(entity.clsid.value, -# 'c9da6455-213d-42c9-9a79-3e9149a57833') -# self.assertTrue(hasattr(entity, "Timestamp")) -# self.assertIsInstance(entity.Timestamp, datetime) -# self.assertIsNotNone(entity.etag) -# -# def _assert_default_entity_json_no_metadata(self, entity): -# ''' -# Asserts that the entity passed in matches the default entity. -# ''' -# self.assertEqual(entity.age, '39') -# self.assertEqual(entity.sex, 'male') -# self.assertEqual(entity.name, 'John Doe') -# self.assertEqual(entity.married, True) -# self.assertEqual(entity.deceased, False) -# self.assertFalse(hasattr(entity, "optional")) -# self.assertFalse(hasattr(entity, "aquarius")) -# self.assertEqual(entity.ratio, 3.1) -# self.assertEqual(entity.evenratio, 3.0) -# self.assertEqual(entity.large, '933311100') -# self.assertEqual(entity.Birthday, '1973-10-04T00:00:00Z') -# self.assertEqual(entity.birthday, '1970-10-04T00:00:00Z') -# self.assertEqual(entity.binary, _encode_base64(b'binary')) -# self.assertIsInstance(entity.other, EntityProperty) -# self.assertEqual(entity.other.type, EdmType.INT32) -# self.assertEqual(entity.other.value, 20) -# self.assertEqual(entity.clsid, 'c9da6455-213d-42c9-9a79-3e9149a57833') -# self.assertTrue(hasattr(entity, "Timestamp")) -# self.assertIsInstance(entity.Timestamp, datetime) -# self.assertIsNotNone(entity.etag) -# -# def _default_encryption_resolver(self, x, y, property): -# return (property == 'sex' or property == 'name') -# -# # @record -# def test_get_encrypted_dict(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_dict() -# entity['sex'] = EntityProperty(EdmType.STRING, entity['sex'], True) -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.table.create_entity(table_entity_properties=entity) -# -# # Act -# new_entity = self.table.get_entity(entity['PartitionKey'], entity['RowKey']) -# -# # Assert -# self._assert_default_entity(new_entity) -# -# # @record -# def test_get_encrypted_entity(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_for_encryption() -# # Only want to encrypt one property in this test -# entity['name'] = 'John Doe' -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.table.insert_entity(self.table_name, entity) -# -# # Act -# new_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # Assert -# self._assert_default_entity(new_entity) -# -# # @record -# def test_get_encrypt_multiple_properties(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_for_encryption() -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.create_entity(table_entity_properties=entity) -# -# # Act -# new_entity = self.ts.get_entity(entity['PartitionKey'], entity['RowKey']) -# -# # Assert -# self._assert_default_entity(new_entity) -# -# # @record -# def test_get_encrypted_entity_key_resolver(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_for_encryption() -# self.ts.key_encryption_key = KeyWrapper('key1') -# key_resolver = KeyResolver() -# key_resolver.put_key(self.ts.key_encryption_key) -# self.ts.key_resolver_function = key_resolver.resolve_key -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# self.ts.key_encryption_key = None -# new_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # Assert -# self._assert_default_entity(new_entity) -# -# # @record -# def test_get_encrypted_entity_encryption_resolver(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_random_entity_class() -# self.ts.encryption_resolver_function = self._default_encryption_resolver -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# new_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# self.ts.key_encryption_key = None -# self.ts.require_encryption = False -# # Retrive a second copy without decrypting to ensure properties were encrypted. -# new_entity2 = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # Assert -# self._assert_default_entity(new_entity) -# self.assertEqual(EdmType.BINARY, new_entity2['sex'].type) -# self.assertEqual(EdmType.BINARY, new_entity2['name'].type) -# -# # @record -# def test_get_encrypted_entity_properties_and_resolver(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_for_encryption() -# self.ts.encryption_resolver_function = self._default_encryption_resolver -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# new_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # Assert -# self._assert_default_entity(new_entity) -# -# def _get_with_payload_format(self, format): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_for_encryption() -# entity['RowKey'] = entity['RowKey'] + format[len('application/json;odata='):] -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# new_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey'], -# accept=format) -# -# # Assert -# if format == TablePayloadFormat.JSON_NO_METADATA: -# self._assert_default_entity_json_no_metadata(new_entity) -# else: -# self._assert_default_entity(new_entity) -# -# # @record -# def test_get_payload_formats(self): -# self._get_with_payload_format(TablePayloadFormat.JSON_FULL_METADATA) -# self._get_with_payload_format(TablePayloadFormat.JSON_MINIMAL_METADATA) -# self._get_with_payload_format(TablePayloadFormat.JSON_NO_METADATA) -# -# def test_get_entity_kek_RSA(self): -# # We can only generate random RSA keys, so this must be run live or -# # the playback test will fail due to a change in kek values. -# if TestMode.need_recording_file(self.test_mode): -# return -# -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_for_encryption() -# self.ts.key_encryption_key = RSAKeyWrapper('key2') -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# new_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # Assert -# self._assert_default_entity(new_entity) -# -# # @record -# def test_get_entity_nonmatching_kid(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_random_entity_class() -# self.ts.encryption_resolver_function = self._default_encryption_resolver -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# self.ts.key_encryption_key.kid = 'Invalid' -# -# # Assert -# try: -# self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# self.fail() -# except AzureException as e: -# self.assertEqual(str(e), _ERROR_DECRYPTION_FAILURE) -# -# # @record -# def test_get_entity_invalid_value_kek_wrap(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_for_encryption() -# self.ts.key_encryption_key = KeyWrapper('key1') -# -# self.ts.key_encryption_key.get_key_wrap_algorithm = None -# try: -# self.ts.insert_entity(self.table_name, entity) -# self.fail() -# except AttributeError as e: -# self.assertEqual(str(e), _ERROR_OBJECT_INVALID.format('key encryption key', 'get_key_wrap_algorithm')) -# -# self.ts.key_encryption_key = KeyWrapper('key1') -# -# self.ts.key_encryption_key.get_kid = None -# with self.assertRaises(AttributeError): -# self.ts.insert_entity(self.table_name, entity) -# -# self.ts.key_encryption_key = KeyWrapper('key1') -# -# self.ts.key_encryption_key.wrap_key = None -# with self.assertRaises(AttributeError): -# self.ts.insert_entity(self.table_name, entity) -# -# # @record -# def test_get_entity_invalid_value_kek_unwrap(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_for_encryption() -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.insert_entity(self.table_name, entity) -# -# self.ts.key_encryption_key.unwrap_key = None -# try: -# self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# self.fail() -# except AzureException as e: -# self.assertEqual(str(e), _ERROR_DECRYPTION_FAILURE) -# -# self.ts.key_encryption_key = KeyWrapper('key1') -# -# self.ts.key_encryption_key.get_kid = None -# with self.assertRaises(AzureException): -# self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # @record -# def test_insert_entity_missing_attribute_kek_wrap(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_for_encryption() -# valid_key = KeyWrapper('key1') -# -# # Act -# invalid_key_1 = lambda: None # functions are objects, so this effectively creates an empty object -# invalid_key_1.get_key_wrap_algorithm = valid_key.get_key_wrap_algorithm -# invalid_key_1.get_kid = valid_key.get_kid -# # No attribute wrap_key -# self.ts.key_encryption_key = invalid_key_1 -# with self.assertRaises(AttributeError): -# self.ts.insert_entity(self.table_name, entity) -# -# invalid_key_2 = lambda: None # functions are objects, so this effectively creates an empty object -# invalid_key_2.wrap_key = valid_key.wrap_key -# invalid_key_2.get_kid = valid_key.get_kid -# # No attribute get_key_wrap_algorithm -# self.ts.key_encryption_key = invalid_key_2 -# with self.assertRaises(AttributeError): -# self.ts.insert_entity(self.table_name, entity) -# -# invalid_key_3 = lambda: None # functions are objects, so this effectively creates an empty object -# invalid_key_3.get_key_wrap_algorithm = valid_key.get_key_wrap_algorithm -# invalid_key_3.wrap_key = valid_key.wrap_key -# # No attribute get_kid -# self.ts.key_encryption_key = invalid_key_3 -# with self.assertRaises(AttributeError): -# self.ts.insert_entity(self.table_name, entity) -# -# # @record -# def test_get_entity_missing_attribute_kek_unwrap(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_for_encryption() -# valid_key = KeyWrapper('key1') -# self.ts.key_encryption_key = valid_key -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# invalid_key_1 = lambda: None # functions are objects, so this effectively creates an empty object -# invalid_key_1.get_kid = valid_key.get_kid -# # No attribute unwrap_key -# self.ts.key_encryption_key = invalid_key_1 -# with self.assertRaises(AzureException): -# self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# invalid_key_2 = lambda: None # functions are objects, so this effectively creates an empty object -# invalid_key_2.unwrap_key = valid_key.unwrap_key -# # No attribute get_kid -# self.ts.key_encryption_key = invalid_key_2 -# with self.assertRaises(AzureException): -# self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # @record -# def test_get_entity_no_decryption(self): -# # Arrange -# entity = self._create_default_entity_for_encryption() -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# self.ts.key_encryption_key = None -# new_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # Assert -# # Access the properties to ensure they are still on the entity -# new_entity['_ClientEncryptionMetadata1'] -# new_entity['_ClientEncryptionMetadata2'] -# -# value = new_entity['sex'] -# self.assertEqual(value.type, EdmType.BINARY) -# -# # @record -# def test_replace_entity(self): -# # Arrange -# entity = self._create_random_entity_class() -# self.ts.insert_entity(self.table_name, entity) -# entity['sex'] = EntityProperty(EdmType.STRING, 'female', True) -# self.ts.key_encryption_key = KeyWrapper('key1') -# -# # Act -# self.ts.require_encryption = True -# self.ts.update_entity(self.table_name, entity) -# new_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # Assert -# self.assertEqual(new_entity['sex'], entity['sex'].value) -# -# # @record -# def test_insert_strict_mode(self): -# # Arrange -# entity = self._create_default_entity_for_encryption() -# self.ts.require_encryption = True -# -# # Assert -# with self.assertRaises(ValueError): -# self.ts.insert_entity(self.table_name, entity) -# -# # @record -# def test_strict_mode_policy_no_encrypted_properties(self): -# # Arrange -# entity = self._create_random_entity_class() -# self.ts.require_encryption = True -# self.ts.key_encryption_key = KeyWrapper('key1') -# -# # Act -# # Even when require encryption is true, it should be possilbe to insert -# # an entity that happens to not have any properties marked for encyrption. -# self.ts.insert_entity(self.table_name, entity) -# new_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # Assert -# self._assert_default_entity(new_entity) -# -# # @record -# def test_get_strict_mode_no_key(self): -# # Arrange -# entity = self._create_default_entity_for_encryption() -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# self.ts.key_encryption_key = None -# self.ts.require_encryption = True -# -# # Assert -# with self.assertRaises(AzureException): -# self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # @record -# def test_get_strict_mode_unencrypted_entity(self): -# # Arrange -# entity = self._create_random_base_entity_class() -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# self.ts.require_encryption = True -# self.ts.key_encryption_key = KeyWrapper('key1') -# -# # Assert -# with self.assertRaises(AzureException): -# self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # @record -# @pytest.mark.skip("pending") -# def test_batch_entity_inserts_context_manager(self): -# # Arrange -# self.ts.require_encryption = True -# entity1 = self._create_random_entity_class() -# entity2 = self._create_random_entity_class(rk='Entity2') -# entity3 = self._create_random_entity_class(rk='Entity3') -# entity2['PartitionKey'] = entity1['PartitionKey'] -# entity3['PartitionKey'] = entity1['PartitionKey'] -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.require_encryption = True -# self.ts.encryption_resolver_function = self._default_encryption_resolver -# self.ts.insert_entity(self.table_name, entity3) -# entity3['sex'] = 'female' -# -# # Act -# with self.ts.batch(self.table_name) as batch: -# batch.insert_entity(entity1) -# batch.insert_or_replace_entity(entity2) -# batch.update_entity(entity3) -# -# new_entity1 = self.ts.get_entity(self.table_name, entity1['PartitionKey'], entity1['RowKey']) -# new_entity2 = self.ts.get_entity(self.table_name, entity2['PartitionKey'], entity2['RowKey']) -# new_entity3 = self.ts.get_entity(self.table_name, entity3['PartitionKey'], entity3['RowKey']) -# -# # Assert -# self.assertEqual(new_entity1['sex'], entity1['sex']) -# self.assertEqual(new_entity2['sex'], entity2['sex']) -# self.assertEqual(new_entity3['sex'], entity3['sex']) -# -# # @record -# @pytest.mark.skip("pending") -# def test_batch_strict_mode(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_for_encryption() -# -# # Act -# batch = TableBatch(require_encryption=True) -# -# # Assert -# with self.assertRaises(ValueError): -# batch.insert_entity(entity) -# -# # @record -# def test_property_resolver_decrypt_conflict(self): -# # Tests that the encrypted properties list is given priorty -# # over the property resolver when deserializng (i.e. the -# # EdmType should be binary, not the result of the resolver) -# -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_default_entity_for_encryption() -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.insert_entity(self.table_name, entity) -# -# property_resolver = lambda x, y, name, a, b: EdmType.STRING if name == 'sex' else None -# -# # Act -# new_entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey'], -# property_resolver=property_resolver) -# -# # Assert -# # If the encrypted property list correctly took priority, this field will have been -# # properly decrypted -# self.assertEqual(new_entity['sex'], 'male') -# -# # @record -# def test_validate_encryption(self): -# # Arrange -# entity = self._create_default_entity_for_encryption() -# key_encryption_key = KeyWrapper('key1') -# self.ts.key_encryption_key = key_encryption_key -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# self.ts.key_encryption_key = None -# entity = self.ts.get_entity(self.table_name, entity['PartitionKey'], entity['RowKey']) -# -# # Note the minor discrepancy from the normal decryption process: because the entity was retrieved -# # without being decrypted, the encrypted_properties list is now stored in an EntityProperty object -# # and is already raw bytes. -# encrypted_properties_list = entity['_ClientEncryptionMetadata2'].value -# encryption_data = entity['_ClientEncryptionMetadata1'] -# encryption_data = _dict_to_encryption_data(loads(encryption_data)) -# -# content_encryption_key = key_encryption_key.unwrap_key(encryption_data.wrapped_content_key.encrypted_key, -# encryption_data.wrapped_content_key.algorithm) -# -# digest = Hash(SHA256(), default_backend()) -# digest.update(encryption_data.content_encryption_IV + -# (entity['RowKey'] + entity['PartitionKey'] + '_ClientEncryptionMetadata2').encode('utf-8')) -# metadataIV = digest.finalize() -# metadataIV = metadataIV[:16] -# -# cipher = _generate_AES_CBC_cipher(content_encryption_key, metadataIV) -# -# # Decrypt the data. -# decryptor = cipher.decryptor() -# encrypted_properties_list = decryptor.update(encrypted_properties_list) + decryptor.finalize() -# -# # Unpad the data. -# unpadder = PKCS7(128).unpadder() -# encrypted_properties_list = unpadder.update(encrypted_properties_list) + unpadder.finalize() -# -# encrypted_properties_list = encrypted_properties_list.decode('utf-8') -# -# # Strip the square braces from the ends and split string into list. -# encrypted_properties_list = loads(encrypted_properties_list) -# -# entity_iv, encrypted_properties, content_encryption_key = \ -# (encryption_data.content_encryption_IV, encrypted_properties_list, content_encryption_key) -# -# decrypted_entity = deepcopy(entity) -# -# for property in encrypted_properties_list: -# value = entity[property] -# -# digest = Hash(SHA256(), default_backend()) -# digest.update(entity_iv + -# (entity['RowKey'] + entity['PartitionKey'] + property).encode('utf-8')) -# propertyIV = digest.finalize() -# propertyIV = propertyIV[:16] -# -# cipher = _generate_AES_CBC_cipher(content_encryption_key, -# propertyIV) -# -# # Decrypt the property. -# decryptor = cipher.decryptor() -# decrypted_data = (decryptor.update(value.value) + decryptor.finalize()) -# -# # Unpad the data. -# unpadder = PKCS7(128).unpadder() -# decrypted_data = (unpadder.update(decrypted_data) + unpadder.finalize()) -# -# decrypted_data = decrypted_data.decode('utf-8') -# -# decrypted_entity[property] = decrypted_data -# -# decrypted_entity.pop('_ClientEncryptionMetadata1') -# decrypted_entity.pop('_ClientEncryptionMetadata2') -# -# # Assert -# self.assertEqual(decrypted_entity['sex'], 'male') -# -# # @record -# def test_insert_encrypt_invalid_types(self): -# # Arrange -# self.ts.require_encryption = True -# entity_binary = self._create_random_entity_class() -# entity_binary['bytes'] = EntityProperty(EdmType.BINARY, urandom(10), True) -# entity_boolean = self._create_random_entity_class() -# entity_boolean['married'] = EntityProperty(EdmType.BOOLEAN, True, True) -# entity_date_time = self._create_random_entity_class() -# entity_date_time['birthday'] = EntityProperty(EdmType.DATETIME, entity_date_time['birthday'], True) -# entity_double = self._create_random_entity_class() -# entity_double['ratio'] = EntityProperty(EdmType.DATETIME, entity_double['ratio'], True) -# entity_guid = self._create_random_entity_class() -# entity_guid['clsid'].encrypt = True -# entity_int32 = self._create_random_entity_class() -# entity_int32['other'].encrypt = True -# entity_int64 = self._create_random_entity_class() -# entity_int64['large'] = EntityProperty(EdmType.INT64, entity_int64['large'], True) -# self.ts.key_encryption_key = KeyWrapper('key1') -# entity_none_str = self._create_random_entity_class() -# entity_none_str['none_str'] = EntityProperty(EdmType.STRING, None, True) -# -# # Act -# -# # Assert -# try: -# self.ts.insert_entity(self.table_name, entity_binary) -# self.fail() -# except ValueError as e: -# self.assertEqual(str(e), _ERROR_UNSUPPORTED_TYPE_FOR_ENCRYPTION) -# with self.assertRaises(ValueError): -# self.ts.insert_entity(self.table_name, entity_boolean) -# with self.assertRaises(ValueError): -# self.ts.insert_entity(self.table_name, entity_date_time) -# with self.assertRaises(ValueError): -# self.ts.insert_entity(self.table_name, entity_double) -# with self.assertRaises(ValueError): -# self.ts.insert_entity(self.table_name, entity_guid) -# with self.assertRaises(ValueError): -# self.ts.insert_entity(self.table_name, entity_int32) -# with self.assertRaises(ValueError): -# self.ts.insert_entity(self.table_name, entity_int64) -# with self.assertRaises(ValueError): -# self.ts.insert_entity(self.table_name, entity_none_str) -# -# # @record -# def test_invalid_encryption_operations_fail(self): -# # Arrange -# entity = self._create_default_entity_for_encryption() -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.insert_entity(self.table_name, entity) -# -# # Assert -# with self.assertRaises(ValueError): -# self.ts.merge_entity(self.table_name, entity) -# -# with self.assertRaises(ValueError): -# self.ts.insert_or_merge_entity(self.table_name, entity) -# -# self.ts.require_encryption = True -# self.ts.key_encryption_key = None -# -# with self.assertRaises(ValueError): -# self.ts.merge_entity(self.table_name, entity) -# -# with self.assertRaises(ValueError): -# self.ts.insert_or_merge_entity(self.table_name, entity) -# -# # @record -# @pytest.mark.skip("pending") -# def test_invalid_encryption_operations_fail_batch(self): -# # Arrange -# entity = self._create_default_entity_for_encryption() -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# batch = TableBatch(require_encryption=True, key_encryption_key=self.ts.key_encryption_key) -# -# # Assert -# with self.assertRaises(ValueError): -# batch.merge_entity(entity) -# -# with self.assertRaises(ValueError): -# batch.insert_or_merge_entity(entity) -# -# # @record -# def test_query_entities_all_properties(self): -# # Arrange -# self.ts.require_encryption = True -# self.ts.key_encryption_key = KeyWrapper('key1') -# table_name = self._create_query_table_encrypted(5) -# default_entity = self._create_random_entity_class() -# -# # Act -# resp = self.ts.query_entities(table_name, num_results=5) -# -# # Assert -# self.assertEqual(len(resp.items), 5) -# for entity in resp.items: -# self.assertEqual(default_entity['sex'], entity['sex']) -# -# # @record -# def test_query_entities_projection(self): -# # Arrange -# self.ts.require_encryption = True -# self.ts.key_encryption_key = KeyWrapper('key1') -# table_name = self._create_query_table_encrypted(5) -# default_entity = self._create_random_entity_class() -# -# # Act -# resp = self.ts.query_entities(table_name, num_results=5, select='PartitionKey,RowKey,sex') -# -# # Assert -# for entity in resp.items: -# self.assertEqual(default_entity['sex'], entity['sex']) -# self.assertFalse(hasattr(entity, '_ClientEncryptionMetadata1')) -# self.assertFalse(hasattr(entity, '_ClientEncryptionMetadata2')) -# -# # @record -# def test_query_entities_mixed_mode(self): -# # Arrange -# entity = self._create_random_entity_class(rk='unencrypted') -# entity['RowKey'] += 'unencrypted' -# self.ts.insert_entity(self.table_name, entity) -# entity = self._create_default_entity_for_encryption() -# self.ts.key_encryption_key = KeyWrapper('key1') -# self.ts.insert_entity(self.table_name, entity) -# -# # Act -# # Pass with out encryption_required -# self.ts.query_entities(self.table_name) -# -# # Assert -# # Fail with encryption_required because not all returned entities -# # will be encrypted. -# self.ts.require_encryption = True -# with self.assertRaises(AzureException): -# self.ts.query_entities(self.table_name) -# -# # @record -# def test_insert_entity_too_many_properties(self): -# # Arrange -# self.ts.require_encryption = True -# entity = self._create_random_base_entity_dict() -# self.ts.key_encryption_key = KeyWrapper('key1') -# for i in range(251): -# entity['key{0}'.format(i)] = 'value{0}'.format(i) -# -# # Act -# with self.assertRaises(ValueError): -# resp = self.ts.insert_entity(self.table_name, entity) -# -# # @record -# def test_validate_swapping_properties_fails(self): -# # Arrange -# entity1 = self._create_random_entity_class(rk='entity1') -# entity2 = self._create_random_entity_class(rk='entity2') -# kek = KeyWrapper('key1') -# self.ts.key_encryption_key = kek -# self.ts.encryption_resolver_function = self._default_encryption_resolver -# self.ts.insert_entity(self.table_name, entity1) -# self.ts.insert_entity(self.table_name, entity2) -# -# # Act -# self.ts.key_encryption_key = None -# new_entity1 = self.ts.get_entity(self.table_name, entity1['PartitionKey'], entity1['RowKey']) -# new_entity2 = deepcopy(new_entity1) -# new_entity2['PartitionKey'] = entity2['PartitionKey'] -# new_entity2['RowKey'] = entity2['RowKey'] -# self.ts.update_entity(self.table_name, new_entity2) -# self.ts.key_encryption_key = kek -# -# # Assert -# with self.assertRaises(AzureException): -# self.ts.get_entity(self.table_name, new_entity2['PartitionKey'], new_entity2['RowKey']) -# -# # @record -# def test_table_ops_ignore_encryption(self): -# table_name = self.get_resource_name('EncryptionTableOps') -# try: -# # Arrange -# self.ts.require_encryption = True -# self.ts.key_encryption_key = KeyWrapper('key1') -# -# # Act -# self.assertTrue(self.ts.create_table(table_name)) -# -# self.assertTrue(self.ts.exists(table_name)) -# -# list_tables = self.ts.list_tables() -# test_table_exists = False -# for table in list_tables: -# if table.name == table_name: -# test_table_exists = True -# self.assertTrue(test_table_exists) -# -# permissions = self.ts.get_table_acl(table_name) -# new_policy = AccessPolicy(TableSasPermissions(_str='r'), expiry=datetime(2017, 9, 9)) -# permissions['samplePolicy'] = new_policy -# self.ts.set_table_acl(table_name, permissions) -# permissions = self.ts.get_table_acl(table_name) -# permissions['samplePolicy'] -# self.ts.key_encryption_key = None -# permissions = self.ts.get_table_acl(table_name) -# permissions['samplePolicy'] -# -# self.ts.delete_table(table_name) -# self.assertFalse(self.ts.exists(table_name)) -# finally: -# self.ts.delete_table(table_name) -# -# -# # ------------------------------------------------------------------------------ -# if __name__ == '__main__': -# unittest.main() diff --git a/sdk/tables/tests.yml b/sdk/tables/tests.yml index ab37d5ceaa85..f94c7a72f029 100644 --- a/sdk/tables/tests.yml +++ b/sdk/tables/tests.yml @@ -8,9 +8,8 @@ resources: endpoint: azure jobs: - - template: ./tests_invoke.yml + - template: ../../eng/pipelines/templates/jobs/archetype-sdk-tests.yml parameters: - InjectedPackages: $(InjectedPackages) EnvVars: STORAGE_ACCOUNT_NAME: $(python-storage-storage-account-name) STORAGE_ACCOUNT_KEY: $(python-storage-storage-account-key)