diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index 189f5185c71e..93801d6836a4 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -334,7 +334,7 @@ try { $serviceName = if (Split-Path $ServiceDirectory) { Split-Path -Leaf $ServiceDirectory } else { - $ServiceDirectory + $ServiceDirectory.Trim('/') } $ResourceGroupName = if ($ResourceGroupName) { @@ -600,7 +600,7 @@ try { $outputFile = "$($templateFile.originalFilePath).env" $environmentText = $deploymentOutputs | ConvertTo-Json; - $bytes = ([System.Text.Encoding]::UTF8).GetBytes($environmentText) + $bytes = [System.Text.Encoding]::UTF8.GetBytes($environmentText) $protectedBytes = [Security.Cryptography.ProtectedData]::Protect($bytes, $null, [Security.Cryptography.DataProtectionScope]::CurrentUser) Set-Content $outputFile -Value $protectedBytes -AsByteStream -Force diff --git a/eng/scripts/Language-Settings.ps1 b/eng/scripts/Language-Settings.ps1 index 6db2d2c150a6..972163597fcf 100644 --- a/eng/scripts/Language-Settings.ps1 +++ b/eng/scripts/Language-Settings.ps1 @@ -429,14 +429,15 @@ function Find-python-Artifacts-For-Apireview($artifactDir, $artifactName) return $packages } -function SetPackageVersion ($PackageName, $Version, $ServiceDirectory, $ReleaseDate) +function SetPackageVersion ($PackageName, $Version, $ServiceDirectory, $ReleaseDate, $ReplaceLatestEntryTitle) { if($null -eq $ReleaseDate) { $ReleaseDate = Get-Date -Format "yyyy-MM-dd" } pip install -r "$EngDir/versioning/requirements.txt" -q -I - python "$EngDir/versioning/version_set.py" --package-name $PackageName --new-version $Version --service $ServiceDirectory --release-date $ReleaseDate + python "$EngDir/versioning/version_set.py" --package-name $PackageName --new-version $Version ` + --service $ServiceDirectory --release-date $ReleaseDate --replace-latest-entry-title $ReplaceLatestEntryTitle } function GetExistingPackageVersions ($PackageName, $GroupId=$null) diff --git a/eng/versioning/version_set.py b/eng/versioning/version_set.py index 0bdfedd58e8b..e8900d52d0d2 100644 --- a/eng/versioning/version_set.py +++ b/eng/versioning/version_set.py @@ -11,6 +11,7 @@ parser.add_argument('--new-version', required=True, help='new package version') parser.add_argument('--service', required=True, help='name of the service for which to set the dev build id (e.g. keyvault)') parser.add_argument('--release-date', help='date in the format "yyyy-MM-dd"') + parser.add_argument('--replace-latest-entry-title', help='indicate if to replace the latest changelog entry"') parser.add_argument( dest="glob_string", nargs="?", @@ -38,4 +39,4 @@ set_version_py(target_package[0], new_version) set_dev_classifier(target_package[0], new_version) - update_change_log(target_package[0], new_version, args.service, args.package_name, False, True, args.release_date) \ No newline at end of file + update_change_log(target_package[0], new_version, args.service, args.package_name, False, args.replace_latest_entry_title, args.release_date) \ No newline at end of file diff --git a/sdk/cognitivelanguage/azure-ai-language-conversations/CHANGELOG.md b/sdk/cognitivelanguage/azure-ai-language-conversations/CHANGELOG.md index f0d9f184e050..a136c4c50b05 100644 --- a/sdk/cognitivelanguage/azure-ai-language-conversations/CHANGELOG.md +++ b/sdk/cognitivelanguage/azure-ai-language-conversations/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.0b1 (unreleased) +## 1.0.0b1 (2021-11-03) ### Features Added * Initial release diff --git a/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_conversation_app_language_parm_async.py b/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_conversation_app_language_parm_async.py index 28ce5f347867..835035df43f3 100644 --- a/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_conversation_app_language_parm_async.py +++ b/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_conversation_app_language_parm_async.py @@ -30,7 +30,7 @@ async def sample_analyze_conversation_app_language_parm_async(): import os from azure.core.credentials import AzureKeyCredential - from azure.ai.language.conversations import ConversationAnalysisClient + from azure.ai.language.conversations.aio import ConversationAnalysisClient from azure.ai.language.conversations.models import ConversationAnalysisOptions # get secrets @@ -48,7 +48,7 @@ async def sample_analyze_conversation_app_language_parm_async(): # analyze quey client = ConversationAnalysisClient(conv_endpoint, AzureKeyCredential(conv_key)) async with client: - result = client.analyze_conversations( + result = await client.analyze_conversations( input, project_name=conv_project, deployment_name='production' diff --git a/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_conversation_response_async.py b/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_conversation_response_async.py index 5b0acd521099..dd34d5352c02 100644 --- a/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_conversation_response_async.py +++ b/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_conversation_response_async.py @@ -30,7 +30,7 @@ async def sample_analyze_orchestration_app_conversation_response_async(): import os from azure.core.credentials import AzureKeyCredential - from azure.ai.language.conversations import ConversationAnalysisClient + from azure.ai.language.conversations.aio import ConversationAnalysisClient from azure.ai.language.conversations.models import ConversationAnalysisOptions # get secrets @@ -47,7 +47,7 @@ async def sample_analyze_orchestration_app_conversation_response_async(): # analyze query client = ConversationAnalysisClient(conv_endpoint, AzureKeyCredential(conv_key)) async with client: - result = client.analyze_conversations( + result = await client.analyze_conversations( input, project_name=orchestration_project, deployment_name='production', diff --git a/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_luis_response_async.py b/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_luis_response_async.py index c767930f43b8..428c32769edd 100644 --- a/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_luis_response_async.py +++ b/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_luis_response_async.py @@ -30,7 +30,7 @@ async def sample_analyze_orchestration_app_luis_response_async(): import os from azure.core.credentials import AzureKeyCredential - from azure.ai.language.conversations import ConversationAnalysisClient + from azure.ai.language.conversations.aio import ConversationAnalysisClient from azure.ai.language.conversations.models import ConversationAnalysisOptions # get secrets @@ -47,7 +47,7 @@ async def sample_analyze_orchestration_app_luis_response_async(): # analyze query client = ConversationAnalysisClient(conv_endpoint, AzureKeyCredential(conv_key)) async with client: - result = client.analyze_conversations( + result = await client.analyze_conversations( input, project_name=orchestration_project, deployment_name='production', diff --git a/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_qna_response_async.py b/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_qna_response_async.py index af197a1df031..017a63b3d9fb 100644 --- a/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_qna_response_async.py +++ b/sdk/cognitivelanguage/azure-ai-language-conversations/samples/async/sample_analyze_orchestration_app_qna_response_async.py @@ -30,7 +30,7 @@ async def sample_analyze_orchestration_app_qna_response_async(): import os from azure.core.credentials import AzureKeyCredential - from azure.ai.language.conversations import ConversationAnalysisClient + from azure.ai.language.conversations.aio import ConversationAnalysisClient from azure.ai.language.conversations.models import ConversationAnalysisOptions # get secrets @@ -47,7 +47,7 @@ async def sample_analyze_orchestration_app_qna_response_async(): # analyze query client = ConversationAnalysisClient(conv_endpoint, AzureKeyCredential(conv_key)) async with client: - result = client.analyze_conversations( + result = await client.analyze_conversations( input, project_name=orchestration_project, deployment_name='production', diff --git a/sdk/cognitivelanguage/azure-ai-language-conversations/setup.py b/sdk/cognitivelanguage/azure-ai-language-conversations/setup.py index 42ec6f386a01..a8c490e51d13 100644 --- a/sdk/cognitivelanguage/azure-ai-language-conversations/setup.py +++ b/sdk/cognitivelanguage/azure-ai-language-conversations/setup.py @@ -76,11 +76,15 @@ packages=find_packages(exclude=[ 'tests', # Exclude packages that will be covered by PEP420 or nspkg + # This means any folder structure that only consists of a __init__.py. + # For example, for storage, this would mean adding 'azure.storage' + # in addition to the default 'azure' that is seen here. + 'azure', 'azure.ai', 'azure.ai.language', ]), install_requires=[ - "azure-core<2.0.0,>=1.19.0", + "azure-core<2.0.0,>=1.19.1", "msrest>=0.6.21", 'azure-common~=1.1', 'six>=1.11.0', diff --git a/sdk/cognitivelanguage/azure-ai-language-questionanswering/CHANGELOG.md b/sdk/cognitivelanguage/azure-ai-language-questionanswering/CHANGELOG.md index b0ac4a4e7f86..f01da3820557 100644 --- a/sdk/cognitivelanguage/azure-ai-language-questionanswering/CHANGELOG.md +++ b/sdk/cognitivelanguage/azure-ai-language-questionanswering/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.0.0 (Unreleased) +## 1.0.0 (2021-11-03) * We are now targeting service version `2021-10-01` diff --git a/sdk/cognitivelanguage/azure-ai-language-questionanswering/setup.py b/sdk/cognitivelanguage/azure-ai-language-questionanswering/setup.py index 2f338c1f28c5..35fff0b16aa7 100644 --- a/sdk/cognitivelanguage/azure-ai-language-questionanswering/setup.py +++ b/sdk/cognitivelanguage/azure-ai-language-questionanswering/setup.py @@ -67,7 +67,7 @@ 'azure.ai.language', ]), install_requires=[ - 'azure-core<2.0.0,>=1.19.0', + 'azure-core<2.0.0,>=1.19.1', 'msrest>=0.6.21', ], extras_require={ diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 67982abcddb5..1db0e79329cb 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -10,6 +10,7 @@ requests and sends them through our pipelines. - GA errors `StreamConsumedError`, `StreamClosedError`, and `ResponseNotReadError` to `azure.core.exceptions`. These errors are thrown if you mishandle streamed responses from the `azure.core.rest` module - add kwargs to the methods for `iter_raw` and `iter_bytes` #21529 +- Added new error type `IncompleteReadError` which is raised if peer closes the connection before we have received the complete message body. ### Breaking Changes @@ -17,9 +18,11 @@ are thrown if you mishandle streamed responses from the `azure.core.rest` module ### Bugs Fixed +- The `Content-Length` header in a http response is strictly checked against the actual number of bytes in the body, + rather than silently truncating data in case the underlying tcp connection is closed prematurely. + (thanks to @jochen-ott-by for the contribution) #20412 - UnboundLocalError when SansIOHTTPPolicy handles an exception #15222 - -### Other Changes +- Add default content type header of `text/plain` and content length header for users who pass unicode strings to the `content` kwarg of `HttpRequest` in 2.7 #21550 ## 1.19.1 (2021-11-01) diff --git a/sdk/core/azure-core/azure/core/exceptions.py b/sdk/core/azure-core/azure/core/exceptions.py index 324d374a6620..1c77f6693497 100644 --- a/sdk/core/azure-core/azure/core/exceptions.py +++ b/sdk/core/azure-core/azure/core/exceptions.py @@ -338,6 +338,10 @@ class DecodeError(HttpResponseError): """Error raised during response deserialization.""" +class IncompleteReadError(DecodeError): + """Error raised if peer closes the connection before we have received the complete message body.""" + + class ResourceExistsError(HttpResponseError): """An error response with status code 4xx. This will not be raised directly by the Azure core pipeline.""" diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py index 0ade4b7a6f4e..2d18e6bd91c3 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_aiohttp.py @@ -36,7 +36,7 @@ from multidict import CIMultiDict from azure.core.configuration import ConnectionConfiguration -from azure.core.exceptions import ServiceRequestError, ServiceResponseError +from azure.core.exceptions import ServiceRequestError, ServiceResponseError, IncompleteReadError from azure.core.pipeline import Pipeline from ._base import HttpRequest @@ -300,6 +300,12 @@ async def __anext__(self): except _ResponseStopIteration: internal_response.close() raise StopAsyncIteration() + except aiohttp.client_exceptions.ClientPayloadError as err: + # This is the case that server closes connection before we finish the reading. aiohttp library + # raises ClientPayloadError. + _LOGGER.warning("Incomplete download: %s", err) + internal_response.close() + raise IncompleteReadError(err, error=err) except Exception as err: _LOGGER.warning("Unable to stream download: %s", err) internal_response.close() @@ -384,7 +390,12 @@ def text(self, encoding: Optional[str] = None) -> str: async def load_body(self) -> None: """Load in memory the body, so it could be accessible from sync methods.""" - self._content = await self.internal_response.read() + try: + self._content = await self.internal_response.read() + except aiohttp.client_exceptions.ClientPayloadError as err: + # This is the case that server closes connection before we finish the reading. aiohttp library + # raises ClientPayloadError. + raise IncompleteReadError(err, error=err) def stream_download(self, pipeline, **kwargs) -> AsyncIteratorType[bytes]: """Generator for streaming response body data. diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py index b5d61aeff474..d0adf93efca0 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_asyncio.py @@ -28,7 +28,7 @@ import functools import logging from typing import ( - Any, Union, Optional, AsyncIterator as AsyncIteratorType, TYPE_CHECKING, overload + Any, Optional, AsyncIterator as AsyncIteratorType, TYPE_CHECKING, overload ) import urllib3 # type: ignore @@ -36,7 +36,9 @@ from azure.core.exceptions import ( ServiceRequestError, - ServiceResponseError + ServiceResponseError, + IncompleteReadError, + HttpResponseError, ) from azure.core.pipeline import Pipeline from ._base import HttpRequest @@ -44,7 +46,7 @@ AsyncHttpResponse, _ResponseStopIteration, _iterate_response_content) -from ._requests_basic import RequestsTransportResponse, _read_raw_stream +from ._requests_basic import RequestsTransportResponse, _read_raw_stream, AzureErrorUnion from ._base_requests_async import RequestsAsyncTransportBase from .._tools import is_rest as _is_rest from .._tools_async import handle_no_stream_rest_response as _handle_no_stream_rest_response @@ -134,7 +136,7 @@ async def send(self, request, **kwargs): # pylint:disable=invalid-overridden-me self.open() loop = kwargs.get("loop", _get_running_loop()) response = None - error = None # type: Optional[Union[ServiceRequestError, ServiceResponseError]] + error = None # type: Optional[AzureErrorUnion] data_to_send = await self._retrieve_request_data(request) try: response = await loop.run_in_executor( @@ -151,6 +153,7 @@ async def send(self, request, **kwargs): # pylint:disable=invalid-overridden-me cert=kwargs.pop('connection_cert', self.connection_config.cert), allow_redirects=False, **kwargs)) + response.raw.enforce_content_length = True except urllib3.exceptions.NewConnectionError as err: error = ServiceRequestError(err, error=err) @@ -161,6 +164,14 @@ async def send(self, request, **kwargs): # pylint:disable=invalid-overridden-me error = ServiceResponseError(err, error=err) else: error = ServiceRequestError(err, error=err) + except requests.exceptions.ChunkedEncodingError as err: + msg = err.__str__() + if 'IncompleteRead' in msg: + _LOGGER.warning("Incomplete download: %s", err) + error = IncompleteReadError(err, error=err) + else: + _LOGGER.warning("Unable to stream download: %s", err) + error = HttpResponseError(err, error=err) except requests.RequestException as err: error = ServiceRequestError(err, error=err) @@ -223,6 +234,15 @@ async def __anext__(self): raise StopAsyncIteration() except requests.exceptions.StreamConsumedError: raise + except requests.exceptions.ChunkedEncodingError as err: + msg = err.__str__() + if 'IncompleteRead' in msg: + _LOGGER.warning("Incomplete download: %s", err) + internal_response.close() + raise IncompleteReadError(err, error=err) + _LOGGER.warning("Unable to stream download: %s", err) + internal_response.close() + raise HttpResponseError(err, error=err) except Exception as err: _LOGGER.warning("Unable to stream download: %s", err) internal_response.close() diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py index 728ae0ad8566..ab9807e7bdc0 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_basic.py @@ -36,7 +36,9 @@ from azure.core.configuration import ConnectionConfiguration from azure.core.exceptions import ( ServiceRequestError, - ServiceResponseError + ServiceResponseError, + IncompleteReadError, + HttpResponseError, ) from . import HttpRequest # pylint: disable=unused-import @@ -51,6 +53,13 @@ if TYPE_CHECKING: from ...rest import HttpRequest as RestHttpRequest, HttpResponse as RestHttpResponse +AzureErrorUnion = Union[ + ServiceRequestError, + ServiceResponseError, + IncompleteReadError, + HttpResponseError, +] + PipelineType = TypeVar("PipelineType") _LOGGER = logging.getLogger(__name__) @@ -79,6 +88,7 @@ def _read_raw_stream(response, chunk_size=1): # https://github.com/psf/requests/blob/master/requests/models.py#L774 response._content_consumed = True # pylint: disable=protected-access + class _RequestsTransportResponseBase(_HttpResponseBase): """Base class for accessing response data. @@ -164,6 +174,15 @@ def __next__(self): raise StopIteration() except requests.exceptions.StreamConsumedError: raise + except requests.exceptions.ChunkedEncodingError as err: + msg = err.__str__() + if 'IncompleteRead' in msg: + _LOGGER.warning("Incomplete download: %s", err) + internal_response.close() + raise IncompleteReadError(err, error=err) + _LOGGER.warning("Unable to stream download: %s", err) + internal_response.close() + raise HttpResponseError(err, error=err) except Exception as err: _LOGGER.warning("Unable to stream download: %s", err) internal_response.close() @@ -289,7 +308,7 @@ def send(self, request, **kwargs): # type: ignore """ self.open() response = None - error = None # type: Optional[Union[ServiceRequestError, ServiceResponseError]] + error = None # type: Optional[AzureErrorUnion] try: connection_timeout = kwargs.pop('connection_timeout', self.connection_config.timeout) @@ -313,6 +332,7 @@ def send(self, request, **kwargs): # type: ignore cert=kwargs.pop('connection_cert', self.connection_config.cert), allow_redirects=False, **kwargs) + response.raw.enforce_content_length = True except (urllib3.exceptions.NewConnectionError, urllib3.exceptions.ConnectTimeoutError) as err: error = ServiceRequestError(err, error=err) @@ -323,6 +343,14 @@ def send(self, request, **kwargs): # type: ignore error = ServiceResponseError(err, error=err) else: error = ServiceRequestError(err, error=err) + except requests.exceptions.ChunkedEncodingError as err: + msg = err.__str__() + if 'IncompleteRead' in msg: + _LOGGER.warning("Incomplete download: %s", err) + error = IncompleteReadError(err, error=err) + else: + _LOGGER.warning("Unable to stream download: %s", err) + error = HttpResponseError(err, error=err) except requests.RequestException as err: error = ServiceRequestError(err, error=err) diff --git a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py index 5d2b4dfa6285..1fce4048318f 100644 --- a/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py +++ b/sdk/core/azure-core/azure/core/pipeline/transport/_requests_trio.py @@ -27,7 +27,7 @@ import functools import logging from typing import ( - Any, Callable, Union, Optional, AsyncIterator as AsyncIteratorType, TYPE_CHECKING, overload + Any, Optional, AsyncIterator as AsyncIteratorType, TYPE_CHECKING, overload ) import trio import urllib3 @@ -36,7 +36,9 @@ from azure.core.exceptions import ( ServiceRequestError, - ServiceResponseError + ServiceResponseError, + IncompleteReadError, + HttpResponseError, ) from azure.core.pipeline import Pipeline from ._base import HttpRequest @@ -44,7 +46,7 @@ AsyncHttpResponse, _ResponseStopIteration, _iterate_response_content) -from ._requests_basic import RequestsTransportResponse, _read_raw_stream +from ._requests_basic import RequestsTransportResponse, _read_raw_stream, AzureErrorUnion from ._base_requests_async import RequestsAsyncTransportBase from .._tools import is_rest as _is_rest from .._tools_async import handle_no_stream_rest_response as _handle_no_stream_rest_response @@ -105,6 +107,15 @@ async def __anext__(self): raise StopAsyncIteration() except requests.exceptions.StreamConsumedError: raise + except requests.exceptions.ChunkedEncodingError as err: + msg = err.__str__() + if 'IncompleteRead' in msg: + _LOGGER.warning("Incomplete download: %s", err) + internal_response.close() + raise IncompleteReadError(err, error=err) + _LOGGER.warning("Unable to stream download: %s", err) + internal_response.close() + raise HttpResponseError(err, error=err) except Exception as err: _LOGGER.warning("Unable to stream download: %s", err) internal_response.close() @@ -184,7 +195,7 @@ async def send(self, request, **kwargs: Any): # pylint:disable=invalid-overridd self.open() trio_limiter = kwargs.get("trio_limiter", None) response = None - error = None # type: Optional[Union[ServiceRequestError, ServiceResponseError]] + error = None # type: Optional[AzureErrorUnion] data_to_send = await self._retrieve_request_data(request) try: try: @@ -217,6 +228,7 @@ async def send(self, request, **kwargs: Any): # pylint:disable=invalid-overridd allow_redirects=False, **kwargs), limiter=trio_limiter) + response.raw.enforce_content_length = True except urllib3.exceptions.NewConnectionError as err: error = ServiceRequestError(err, error=err) @@ -227,6 +239,14 @@ async def send(self, request, **kwargs: Any): # pylint:disable=invalid-overridd error = ServiceResponseError(err, error=err) else: error = ServiceRequestError(err, error=err) + except requests.exceptions.ChunkedEncodingError as err: + msg = err.__str__() + if 'IncompleteRead' in msg: + _LOGGER.warning("Incomplete download: %s", err) + error = IncompleteReadError(err, error=err) + else: + _LOGGER.warning("Unable to stream download: %s", err) + error = HttpResponseError(err, error=err) except requests.RequestException as err: error = ServiceRequestError(err, error=err) diff --git a/sdk/core/azure-core/azure/core/rest/_helpers.py b/sdk/core/azure-core/azure/core/rest/_helpers.py index 934bfa613c76..aa5b05bd9e10 100644 --- a/sdk/core/azure-core/azure/core/rest/_helpers.py +++ b/sdk/core/azure-core/azure/core/rest/_helpers.py @@ -134,7 +134,7 @@ def _shared_set_content_body(content): if isinstance(content, ET.Element): # XML body return set_xml_body(content) - if isinstance(content, (str, bytes)): + if isinstance(content, (six.string_types, bytes)): headers = {} body = content if isinstance(content, six.string_types): diff --git a/sdk/core/azure-core/tests/async_tests/test_content_length_checking_async.py b/sdk/core/azure-core/tests/async_tests/test_content_length_checking_async.py new file mode 100644 index 000000000000..726c4baf32ed --- /dev/null +++ b/sdk/core/azure-core/tests/async_tests/test_content_length_checking_async.py @@ -0,0 +1,28 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from azure.core.pipeline import AsyncPipeline +from azure.core.pipeline.transport import ( + HttpRequest, +) +from azure.core import AsyncPipelineClient +from azure.core.exceptions import IncompleteReadError +import pytest + + +@pytest.mark.asyncio +async def test_aio_transport_short_read_download_stream(port): + url = "http://localhost:{}/errors/short-data".format(port) + client = AsyncPipelineClient(url) + with pytest.raises(IncompleteReadError): + async with client: + request = HttpRequest("GET", url) + pipeline_response = await client._pipeline.run(request, stream=True) + response = pipeline_response.http_response + data = response.stream_download(client._pipeline) + content = b"" + async for d in data: + content += d diff --git a/sdk/core/azure-core/tests/test_content_length_checking.py b/sdk/core/azure-core/tests/test_content_length_checking.py new file mode 100644 index 000000000000..7cdbd8f70c2a --- /dev/null +++ b/sdk/core/azure-core/tests/test_content_length_checking.py @@ -0,0 +1,27 @@ +# coding: utf-8 +# ------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See LICENSE.txt in the project root for +# license information. +# ------------------------------------------------------------------------- +from azure.core import PipelineClient +from azure.core.pipeline import Pipeline +from azure.core.pipeline.transport import ( + HttpRequest, + RequestsTransport, +) +from azure.core.exceptions import IncompleteReadError +import pytest + + +def test_sync_transport_short_read_download_stream(port): + url = "http://localhost:{}/errors/short-data".format(port) + client = PipelineClient(url) + request = HttpRequest("GET", url) + with pytest.raises(IncompleteReadError): + pipeline_response = client._pipeline.run(request, stream=True) + response = pipeline_response.http_response + data = response.stream_download(client._pipeline) + content = b"" + for d in data: + content += d diff --git a/sdk/core/azure-core/tests/test_rest_http_request.py b/sdk/core/azure-core/tests/test_rest_http_request.py index 70c184b5cc3d..c56871dcd477 100644 --- a/sdk/core/azure-core/tests/test_rest_http_request.py +++ b/sdk/core/azure-core/tests/test_rest_http_request.py @@ -255,6 +255,17 @@ def test_data_str_input(): assert len(request.headers) == 1 assert request.headers['Content-Type'] == 'application/x-www-form-urlencoded' +def test_content_str_input(): + requests = [ + HttpRequest("POST", "/fake", content="hello, world!"), + HttpRequest("POST", "/fake", content=u"hello, world!"), + ] + for request in requests: + assert len(request.headers) == 2 + assert request.headers["Content-Type"] == "text/plain" + assert request.headers["Content-Length"] == "13" + assert request.content == "hello, world!" + @pytest.mark.parametrize(("value"), (object(), {"key": "value"})) def test_multipart_invalid_value(value): diff --git a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py index 221f598e063a..bab6ef3c4913 100644 --- a/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py +++ b/sdk/core/azure-core/tests/testserver_tests/coretestserver/coretestserver/test_routes/errors.py @@ -26,3 +26,10 @@ def __iter__(self): yield b"Hello, " yield b"world!" return Response(StreamingBody(), status=500) + +@errors_api.route('/short-data', methods=['GET']) +def get_short_data(): + response = Response(b"X" * 4, status=200) + response.automatically_set_content_length = False + response.headers["Content-Length"] = "8" + return response diff --git a/sdk/monitor/azure-monitor-query/CHANGELOG.md b/sdk/monitor/azure-monitor-query/CHANGELOG.md index f1043932106b..eb45cefa2f2c 100644 --- a/sdk/monitor/azure-monitor-query/CHANGELOG.md +++ b/sdk/monitor/azure-monitor-query/CHANGELOG.md @@ -8,6 +8,8 @@ ### Bugs Fixed +- Fixed a bug where Metadata values in timestamp don't show up sometimes. + ### Other Changes ## 1.0.0 (2021-10-06) diff --git a/sdk/monitor/azure-monitor-query/azure/monitor/query/_models.py b/sdk/monitor/azure-monitor-query/azure/monitor/query/_models.py index 1b7b30da04de..fc0462f9b1be 100644 --- a/sdk/monitor/azure-monitor-query/azure/monitor/query/_models.py +++ b/sdk/monitor/azure-monitor-query/azure/monitor/query/_models.py @@ -509,7 +509,7 @@ class TimeSeriesElement(object): """ def __init__(self, **kwargs): # type: (Any) -> None - self.metadata_values = kwargs.get("metadatavalues", None) + self.metadata_values = kwargs.get("metadata_values", None) self.data = kwargs.get("data", None) @classmethod diff --git a/sdk/monitor/azure-monitor-query/tests/async/test_metrics_client_async.py b/sdk/monitor/azure-monitor-query/tests/async/test_metrics_client_async.py index 89757e75e37d..cd8360596064 100644 --- a/sdk/monitor/azure-monitor-query/tests/async/test_metrics_client_async.py +++ b/sdk/monitor/azure-monitor-query/tests/async/test_metrics_client_async.py @@ -42,6 +42,23 @@ async def test_metrics_granularity(): assert response assert response.granularity == timedelta(minutes=5) +@pytest.mark.live_test_only +@pytest.mark.asyncio +async def test_metrics_filter(): + credential = _credential() + client = MetricsQueryClient(credential) + response = await client.query_resource( + os.environ['METRICS_RESOURCE_URI'], + metric_names=["MatchedEventCount"], + timespan=timedelta(days=1), + granularity=timedelta(minutes=5), + filter="EventSubscriptionName eq '*'", + aggregations=[MetricAggregationType.COUNT] + ) + assert response + metric = response.metrics['MatchedEventCount'] + for t in metric.timeseries: + assert t.metadata_values is not None @pytest.mark.live_test_only @pytest.mark.asyncio diff --git a/sdk/monitor/azure-monitor-query/tests/test_metrics_client.py b/sdk/monitor/azure-monitor-query/tests/test_metrics_client.py index f9f5da4db012..54656b1cb3cf 100644 --- a/sdk/monitor/azure-monitor-query/tests/test_metrics_client.py +++ b/sdk/monitor/azure-monitor-query/tests/test_metrics_client.py @@ -39,6 +39,23 @@ def test_metrics_granularity(): assert response assert response.granularity == timedelta(minutes=5) +@pytest.mark.live_test_only +def test_metrics_filter(): + credential = _credential() + client = MetricsQueryClient(credential) + response = client.query_resource( + os.environ['METRICS_RESOURCE_URI'], + metric_names=["MatchedEventCount"], + timespan=timedelta(days=1), + granularity=timedelta(minutes=5), + filter="EventSubscriptionName eq '*'", + aggregations=[MetricAggregationType.COUNT] + ) + assert response + metric = response.metrics['MatchedEventCount'] + for t in metric.timeseries: + assert t.metadata_values is not None + @pytest.mark.live_test_only def test_metrics_list(): credential = _credential() diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py index 25e5fed4899a..3c1bdfb448e3 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_schema_registry_client.py @@ -25,6 +25,7 @@ # -------------------------------------------------------------------------- from typing import Any, TYPE_CHECKING, Union +from ._utils import get_http_request_kwargs from ._common._constants import SchemaFormat from ._common._schema import Schema, SchemaProperties from ._common._response_handlers import ( @@ -114,16 +115,17 @@ def register_schema( except AttributeError: pass + http_request_kwargs = get_http_request_kwargs(kwargs) request = schema_rest.build_register_request( group_name=group_name, schema_name=name, content=schema_definition, serialization_type=format, content_type=kwargs.pop("content_type", "application/json"), - **kwargs + **http_request_kwargs ) - response = self._generated_client.send_request(request) + response = self._generated_client.send_request(request, **kwargs) response.raise_for_status() return _parse_response_schema_properties(response) @@ -147,7 +149,8 @@ def get_schema(self, id, **kwargs): # pylint:disable=redefined-builtin :caption: Get schema by id. """ - request = schema_rest.build_get_by_id_request(schema_id=id) + http_request_kwargs = get_http_request_kwargs(kwargs) + request = schema_rest.build_get_by_id_request(schema_id=id, **http_request_kwargs) response = self._generated_client.send_request(request, **kwargs) response.raise_for_status() return _parse_response_schema(response) @@ -183,13 +186,14 @@ def get_schema_properties( except AttributeError: pass + http_request_kwargs = get_http_request_kwargs(kwargs) request = schema_rest.build_query_id_by_content_request( group_name=group_name, schema_name=name, content=schema_definition, serialization_type=format, content_type=kwargs.pop("content_type", "application/json"), - **kwargs + **http_request_kwargs ) response = self._generated_client.send_request(request, **kwargs) diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_utils.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_utils.py new file mode 100644 index 000000000000..d041e5c75f69 --- /dev/null +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/_utils.py @@ -0,0 +1,9 @@ +# -------------------------------------------------------------------------------------------- +# Copyright (c) Microsoft Corporation. All rights reserved. +# Licensed under the MIT License. See License.txt in the project root for license information. +# -------------------------------------------------------------------------------------------- + +def get_http_request_kwargs(kwargs): + http_request_keywords = ["params", "headers", "json", "data", "files"] + http_request_kwargs = {key: kwargs.pop(key, None) for key in http_request_keywords if key in kwargs} + return http_request_kwargs diff --git a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py index 26cdcf219023..b42e132f55ec 100644 --- a/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/azure/schemaregistry/aio/_schema_registry_client_async.py @@ -25,6 +25,7 @@ # -------------------------------------------------------------------------- from typing import Any, TYPE_CHECKING, Union +from .._utils import get_http_request_kwargs from .._common._constants import SchemaFormat from .._common._schema import Schema, SchemaProperties from .._common._response_handlers import ( @@ -116,16 +117,17 @@ async def register_schema( except AttributeError: pass + http_request_kwargs = get_http_request_kwargs(kwargs) request = schema_rest.build_register_request( group_name=group_name, schema_name=name, content=schema_definition, serialization_type=format, content_type=kwargs.pop("content_type", "application/json"), - **kwargs + **http_request_kwargs ) - response = await self._generated_client.send_request(request) + response = await self._generated_client.send_request(request, **kwargs) response.raise_for_status() return _parse_response_schema_properties(response) @@ -152,7 +154,8 @@ async def get_schema( :caption: Get schema by id. """ - request = schema_rest.build_get_by_id_request(schema_id=id) + http_request_kwargs = get_http_request_kwargs(kwargs) + request = schema_rest.build_get_by_id_request(schema_id=id, **http_request_kwargs) response = await self._generated_client.send_request(request, **kwargs) response.raise_for_status() return _parse_response_schema(response) @@ -192,13 +195,14 @@ async def get_schema_properties( except AttributeError: pass + http_request_kwargs = get_http_request_kwargs(kwargs) request = schema_rest.build_query_id_by_content_request( group_name=group_name, schema_name=name, content=schema_definition, serialization_type=format, content_type=kwargs.pop("content_type", "application/json"), - **kwargs + **http_request_kwargs ) response = await self._generated_client.send_request(request, **kwargs) diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py index 01b08ff2b6e4..66e165b04c51 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/async_tests/test_schema_registry_async.py @@ -47,20 +47,20 @@ async def test_schema_basic_async(self, schemaregistry_fully_qualified_namespace schema_name = self.get_resource_name('test-schema-basic-async') schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" format = "Avro" - schema_properties = await client.register_schema(schemaregistry_group, schema_name, schema_str, format) + schema_properties = await client.register_schema(schemaregistry_group, schema_name, schema_str, format, logging_enable=True) assert schema_properties.id is not None assert schema_properties.version is 1 assert schema_properties.format == "Avro" - returned_schema = await client.get_schema(id=schema_properties.id) + returned_schema = await client.get_schema(id=schema_properties.id, logging_enable=True) assert returned_schema.properties.id == schema_properties.id assert returned_schema.properties.version == 1 assert returned_schema.properties.format == "Avro" assert returned_schema.schema_definition == schema_str - returned_schema_properties = await client.get_schema_properties(schemaregistry_group, schema_name, schema_str, format) + returned_schema_properties = await client.get_schema_properties(schemaregistry_group, schema_name, schema_str, format, logging_enable=True) assert returned_schema_properties.id == schema_properties.id assert returned_schema_properties.version == 1 diff --git a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py index 907617a311bf..26a3ff971aff 100644 --- a/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py +++ b/sdk/schemaregistry/azure-schemaregistry/tests/test_schema_registry.py @@ -42,20 +42,20 @@ def test_schema_basic(self, schemaregistry_fully_qualified_namespace, schemaregi schema_name = self.get_resource_name('test-schema-basic') schema_str = """{"namespace":"example.avro","type":"record","name":"User","fields":[{"name":"name","type":"string"},{"name":"favorite_number","type":["int","null"]},{"name":"favorite_color","type":["string","null"]}]}""" format = "Avro" - schema_properties = client.register_schema(schemaregistry_group, schema_name, schema_str, format) + schema_properties = client.register_schema(schemaregistry_group, schema_name, schema_str, format, logging_enable=True) assert schema_properties.id is not None assert schema_properties.version is 1 assert schema_properties.format == "Avro" - returned_schema = client.get_schema(id=schema_properties.id) + returned_schema = client.get_schema(id=schema_properties.id, logging_enable=True) assert returned_schema.properties.id == schema_properties.id assert returned_schema.properties.version == 1 assert returned_schema.properties.format == "Avro" assert returned_schema.schema_definition == schema_str - returned_schema_properties = client.get_schema_properties(schemaregistry_group, schema_name, schema_str, format) + returned_schema_properties = client.get_schema_properties(schemaregistry_group, schema_name, schema_str, format, logging_enable=True) assert returned_schema_properties.id == schema_properties.id assert returned_schema_properties.version == 1 diff --git a/sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md b/sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md index fe86a940fecb..fbc6d121f627 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md +++ b/sdk/textanalytics/azure-ai-textanalytics/CHANGELOG.md @@ -1,5 +1,15 @@ # Release History +## 5.2.0b3 (Unreleased) + +### Features Added + +### Breaking Changes + +### Bugs Fixed + +### Other Changes + ## 5.2.0b2 (2021-11-02) This version of the SDK defaults to the latest supported API version, which currently is `v3.2-preview.2`. diff --git a/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_version.py b/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_version.py index 9faba3e954c6..164878621003 100644 --- a/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_version.py +++ b/sdk/textanalytics/azure-ai-textanalytics/azure/ai/textanalytics/_version.py @@ -4,5 +4,5 @@ # Licensed under the MIT License. # ------------------------------------ -VERSION = "5.2.0b2" +VERSION = "5.2.0b3" DEFAULT_API_VERSION = "v3.2-preview.2" diff --git a/shared_requirements.txt b/shared_requirements.txt index 09fd92bdc815..78f9ac27ffc1 100644 --- a/shared_requirements.txt +++ b/shared_requirements.txt @@ -151,7 +151,7 @@ backports.functools-lru-cache >= 1.6.4; python_version == "2.7" #override azure-keyvault-secrets azure-core<2.0.0,>=1.7.0 #override azure-ai-textanalytics msrest>=0.6.21 #override azure-ai-textanalytics azure-core<2.0.0,>=1.19.1 -#override azure-ai-language-questionanswering azure-core<2.0.0,>=1.19.0 +#override azure-ai-language-questionanswering azure-core<2.0.0,>=1.19.1 #override azure-ai-language-questionanswering msrest>=0.6.21 #override azure-search-documents azure-core<2.0.0,>=1.19.0 #override azure-ai-formrecognizer msrest>=0.6.21 @@ -353,7 +353,7 @@ opentelemetry-sdk<2.0.0,>=1.5.0,!=1.10a0 #override azure-mgmt-azurearcdata msrest>=0.6.21 #override azure-mgmt-servicelinker msrest>=0.6.21 #override azure-mgmt-fluidrelay msrest>=0.6.21 -#override azure-ai-language-conversations azure-core<2.0.0,>=1.19.0 +#override azure-ai-language-conversations azure-core<2.0.0,>=1.19.1 #override azure-ai-language-conversations msrest>=0.6.21 #override azure-mgmt-dataprotection msrest>=0.6.21 #override azure-mgmt-chaos msrest>=0.6.21