From 38a1385342ff4960256619d0e74ecdda4a99bf2a Mon Sep 17 00:00:00 2001 From: iscai-msft <43154838+iscai-msft@users.noreply.github.com> Date: Mon, 30 Aug 2021 20:12:54 -0400 Subject: [PATCH 1/5] remove iter_text and iter_lines (#20460) --- sdk/core/azure-core/CHANGELOG.md | 3 +- sdk/core/azure-core/azure/core/_version.py | 2 +- .../azure-core/azure/core/rest/_helpers.py | 34 --------------- sdk/core/azure-core/azure/core/rest/_rest.py | 16 ------- .../azure-core/azure/core/rest/_rest_py3.py | 43 ------------------- .../test_rest_stream_responses_async.py | 20 +++++---- .../tests/test_rest_stream_responses.py | 28 ++++++------ 7 files changed, 29 insertions(+), 117 deletions(-) diff --git a/sdk/core/azure-core/CHANGELOG.md b/sdk/core/azure-core/CHANGELOG.md index 811230155bbb..acf86443a780 100644 --- a/sdk/core/azure-core/CHANGELOG.md +++ b/sdk/core/azure-core/CHANGELOG.md @@ -1,6 +1,6 @@ # Release History -## 1.17.1 (Unreleased) +## 1.18.0 (Unreleased) ### Features Added @@ -10,6 +10,7 @@ - The `text` property on `azure.core.rest.HttpResponse` and `azure.core.rest.AsyncHttpResponse` has changed to a method, which also takes an `encoding` parameter. +- Removed `iter_text` and `iter_lines` from `azure.core.rest.HttpResponse` and `azure.core.rest.AsyncHttpResponse` ### Bugs Fixed diff --git a/sdk/core/azure-core/azure/core/_version.py b/sdk/core/azure-core/azure/core/_version.py index 691adad6d1be..8b847f69ad1b 100644 --- a/sdk/core/azure-core/azure/core/_version.py +++ b/sdk/core/azure-core/azure/core/_version.py @@ -9,4 +9,4 @@ # regenerated. # -------------------------------------------------------------------------- -VERSION = "1.17.1" +VERSION = "1.18.0" diff --git a/sdk/core/azure-core/azure/core/rest/_helpers.py b/sdk/core/azure-core/azure/core/rest/_helpers.py index 1a011689a238..072d9f70992d 100644 --- a/sdk/core/azure-core/azure/core/rest/_helpers.py +++ b/sdk/core/azure-core/azure/core/rest/_helpers.py @@ -227,40 +227,6 @@ def lookup_encoding(encoding): except LookupError: return False -def parse_lines_from_text(text): - # largely taken from httpx's LineDecoder code - lines = [] - last_chunk_of_text = "" - while text: - text_length = len(text) - for idx in range(text_length): - curr_char = text[idx] - next_char = None if idx == len(text) - 1 else text[idx + 1] - if curr_char == "\n": - lines.append(text[: idx + 1]) - text = text[idx + 1: ] - break - if curr_char == "\r" and next_char == "\n": - # if it ends with \r\n, we only do \n - lines.append(text[:idx] + "\n") - text = text[idx + 2:] - break - if curr_char == "\r" and next_char is not None: - # if it's \r then a normal character, we switch \r to \n - lines.append(text[:idx] + "\n") - text = text[idx + 1:] - break - if next_char is None: - last_chunk_of_text += text - text = "" - break - if last_chunk_of_text.endswith("\r"): - # if ends with \r, we switch \r to \n - lines.append(last_chunk_of_text[:-1] + "\n") - elif last_chunk_of_text: - lines.append(last_chunk_of_text) - return lines - def to_pipeline_transport_request_helper(rest_request): from ..pipeline.transport import HttpRequest as PipelineTransportHttpRequest return PipelineTransportHttpRequest( diff --git a/sdk/core/azure-core/azure/core/rest/_rest.py b/sdk/core/azure-core/azure/core/rest/_rest.py index 10a8486a2c64..1603a754b840 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest.py +++ b/sdk/core/azure-core/azure/core/rest/_rest.py @@ -33,7 +33,6 @@ from ..utils._utils import _case_insensitive_dict from ._helpers import ( FilesType, - parse_lines_from_text, set_content_body, set_json_body, set_multipart_body, @@ -357,21 +356,6 @@ def iter_bytes(self): """ raise NotImplementedError() - def iter_text(self): - # type: () -> Iterator[str] - """Iterate over the response text - """ - for byte in self.iter_bytes(): - text = byte.decode(self.encoding or "utf-8") - yield text - - def iter_lines(self): - # type: () -> Iterator[str] - for text in self.iter_text(): - lines = parse_lines_from_text(text) - for line in lines: - yield line - def _close_stream(self): # type: (...) -> None self.is_stream_consumed = True diff --git a/sdk/core/azure-core/azure/core/rest/_rest_py3.py b/sdk/core/azure-core/azure/core/rest/_rest_py3.py index 21e42f46b044..5bb49a4a0cf7 100644 --- a/sdk/core/azure-core/azure/core/rest/_rest_py3.py +++ b/sdk/core/azure-core/azure/core/rest/_rest_py3.py @@ -48,7 +48,6 @@ FilesType, HeadersType, cast, - parse_lines_from_text, set_json_body, set_multipart_body, set_urlencoded_body, @@ -377,27 +376,6 @@ def iter_bytes(self) -> Iterator[bytes]: """ raise NotImplementedError() - def iter_text(self) -> Iterator[str]: - """Iterates over the text in the response. - - :return: An iterator of string. Each string chunk will be a text from the response - :rtype: Iterator[str] - """ - for byte in self.iter_bytes(): - text = byte.decode(self.encoding or "utf-8") - yield text - - def iter_lines(self) -> Iterator[str]: - """Iterates over the lines in the response. - - :return: An iterator of string. Each string chunk will be a line from the response - :rtype: Iterator[str] - """ - for text in self.iter_text(): - lines = parse_lines_from_text(text) - for line in lines: - yield line - def __repr__(self) -> str: content_type_str = ( ", Content-Type: {}".format(self.content_type) if self.content_type else "" @@ -471,27 +449,6 @@ async def iter_bytes(self) -> AsyncIterator[bytes]: # getting around mypy behavior, see https://github.com/python/mypy/issues/10732 yield # pylint: disable=unreachable - async def iter_text(self) -> AsyncIterator[str]: - """Asynchronously iterates over the text in the response. - - :return: An async iterator of string. Each string chunk will be a text from the response - :rtype: AsyncIterator[str] - """ - async for byte in self.iter_bytes(): # type: ignore - text = byte.decode(self.encoding or "utf-8") - yield text - - async def iter_lines(self) -> AsyncIterator[str]: - """Asynchronously iterates over the lines in the response. - - :return: An async iterator of string. Each string chunk will be a line from the response - :rtype: AsyncIterator[str] - """ - async for text in self.iter_text(): - lines = parse_lines_from_text(text) - for line in lines: - yield line - async def close(self) -> None: """Close the response. diff --git a/sdk/core/azure-core/tests/async_tests/test_rest_stream_responses_async.py b/sdk/core/azure-core/tests/async_tests/test_rest_stream_responses_async.py index d94f4748f90c..2b8a68cb7bd1 100644 --- a/sdk/core/azure-core/tests/async_tests/test_rest_stream_responses_async.py +++ b/sdk/core/azure-core/tests/async_tests/test_rest_stream_responses_async.py @@ -66,6 +66,7 @@ async def test_iter_bytes(client): assert response.is_closed assert raw == b"Hello, world!" +@pytest.mark.skip(reason="We've gotten rid of iter_text for now") @pytest.mark.asyncio async def test_iter_text(client): request = HttpRequest("GET", "/basic/string") @@ -76,14 +77,15 @@ async def test_iter_text(client): content += part assert content == "Hello, world!" +@pytest.mark.skip(reason="We've gotten rid of iter_lines for now") @pytest.mark.asyncio async def test_iter_lines(client): request = HttpRequest("GET", "/basic/lines") async with client.send_request(request, stream=True) as response: content = [] - async for line in response.iter_lines(): - content.append(line) + async for part in response.iter_lines(): + content.append(part) assert content == ["Hello,\n", "world!"] @@ -161,11 +163,11 @@ async def test_iter_read_back_and_forth(client): # the reason why the code flow is like this, is because the 'iter_x' functions don't # actually read the contents into the response, the output them. Once they're yielded, # the stream is closed, so you have to catch the output when you iterate through it - request = HttpRequest("GET", "/basic/lines") + request = HttpRequest("GET", "/basic/string") async with client.send_request(request, stream=True) as response: - async for line in response.iter_lines(): - assert line + async for part in response.iter_bytes(): + assert part with pytest.raises(ResponseNotReadError): response.text() with pytest.raises(StreamConsumedError): @@ -175,16 +177,16 @@ async def test_iter_read_back_and_forth(client): @pytest.mark.asyncio async def test_stream_with_return_pipeline_response(client): - request = HttpRequest("GET", "/basic/lines") + request = HttpRequest("GET", "/basic/string") pipeline_response = await client.send_request(request, stream=True, _return_pipeline_response=True) assert hasattr(pipeline_response, "http_request") assert hasattr(pipeline_response.http_request, "content") assert hasattr(pipeline_response, "http_response") assert hasattr(pipeline_response, "context") parts = [] - async for line in pipeline_response.http_response.iter_lines(): - parts.append(line) - assert parts == ['Hello,\n', 'world!'] + async for part in pipeline_response.http_response.iter_bytes(): + parts.append(part) + assert parts == [b'Hello, world!'] await client.close() @pytest.mark.asyncio diff --git a/sdk/core/azure-core/tests/test_rest_stream_responses.py b/sdk/core/azure-core/tests/test_rest_stream_responses.py index cf547d8f750e..a62a64e9b121 100644 --- a/sdk/core/azure-core/tests/test_rest_stream_responses.py +++ b/sdk/core/azure-core/tests/test_rest_stream_responses.py @@ -79,6 +79,7 @@ def test_iter_bytes(client): assert response.is_stream_consumed assert raw == b"Hello, world!" +@pytest.mark.skip(reason="We've gotten rid of iter_text for now") def test_iter_text(client): request = HttpRequest("GET", "/basic/string") @@ -88,6 +89,7 @@ def test_iter_text(client): content += part assert content == "Hello, world!" +@pytest.mark.skip(reason="We've gotten rid of iter_lines for now") def test_iter_lines(client): request = HttpRequest("GET", "/basic/lines") @@ -175,18 +177,18 @@ def test_decompress_compressed_header(client): url = "https://{}.blob.core.windows.net/tests/test_with_header.tar.gz".format(account_name) request = HttpRequest("GET", url) response = client.send_request(request, stream=True) - iter = response.iter_text() - data = "".join(list(iter)) - assert data == "test" + iter = response.iter_bytes() + data = b"".join(list(iter)) + assert data == b"test" def test_iter_read(client): # thanks to McCoy Patiño for this test! - request = HttpRequest("GET", "/basic/lines") + request = HttpRequest("GET", "/basic/string") response = client.send_request(request, stream=True) response.read() - iterator = response.iter_lines() - for line in iterator: - assert line + iterator = response.iter_bytes() + for part in iterator: + assert part assert response.text() def test_iter_read_back_and_forth(client): @@ -196,11 +198,11 @@ def test_iter_read_back_and_forth(client): # the reason why the code flow is like this, is because the 'iter_x' functions don't # actually read the contents into the response, the output them. Once they're yielded, # the stream is closed, so you have to catch the output when you iterate through it - request = HttpRequest("GET", "/basic/lines") + request = HttpRequest("GET", "/basic/string") response = client.send_request(request, stream=True) - iterator = response.iter_lines() - for line in iterator: - assert line + iterator = response.iter_bytes() + for part in iterator: + assert part with pytest.raises(ResponseNotReadError): response.text() with pytest.raises(StreamConsumedError): @@ -209,12 +211,12 @@ def test_iter_read_back_and_forth(client): response.text() def test_stream_with_return_pipeline_response(client): - request = HttpRequest("GET", "/basic/lines") + request = HttpRequest("GET", "/basic/string") pipeline_response = client.send_request(request, stream=True, _return_pipeline_response=True) assert hasattr(pipeline_response, "http_request") assert hasattr(pipeline_response, "http_response") assert hasattr(pipeline_response, "context") - assert list(pipeline_response.http_response.iter_lines()) == ['Hello,\n', 'world!'] + assert list(pipeline_response.http_response.iter_bytes()) == [b'Hello, world!'] def test_error_reading(client): request = HttpRequest("GET", "/errors/403") From e595715863da780bbbbe874e07bf3f2c2f87272d Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Mon, 30 Aug 2021 17:21:11 -0700 Subject: [PATCH 2/5] Skip eng common workflow enforcer for private repos (#20462) Co-authored-by: Chidozie Ononiwu --- .../pipelines/templates/steps/eng-common-workflow-enforcer.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml b/eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml index b68e1cd0c27a..58a8b5b487b9 100644 --- a/eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml +++ b/eng/common/pipelines/templates/steps/eng-common-workflow-enforcer.yml @@ -21,4 +21,4 @@ steps: } } displayName: Prevent changes to eng/common outside of azure-sdk-tools repo - condition: and(succeeded(), ne(variables['Skip.EngCommonWorkflowEnforcer'], 'true')) + condition: and(succeeded(), ne(variables['Skip.EngCommonWorkflowEnforcer'], 'true'), not(endsWith(variables['Build.Repository.Name'], '-pr'))) \ No newline at end of file From 946b2903d4f8bb12e9219fea79eebd9aa713a548 Mon Sep 17 00:00:00 2001 From: Azure SDK Bot <53356347+azure-sdk@users.noreply.github.com> Date: Mon, 30 Aug 2021 17:46:00 -0700 Subject: [PATCH 3/5] Sync eng/common directory with azure-sdk-tools for PR 1953 (#20466) * Add random elements to default base name * Fix principal * tab Co-authored-by: JoshLove-msft <54595583+JoshLove-msft@users.noreply.github.com> --- eng/common/TestResources/New-TestResources.ps1 | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/eng/common/TestResources/New-TestResources.ps1 b/eng/common/TestResources/New-TestResources.ps1 index fa75730dcd2d..508b9b693976 100644 --- a/eng/common/TestResources/New-TestResources.ps1 +++ b/eng/common/TestResources/New-TestResources.ps1 @@ -289,7 +289,8 @@ try { $AzureTestPrincipal } else { Log "TestApplicationId was not specified; creating a new service principal in subscription '$SubscriptionId'" - $global:AzureTestPrincipal = New-AzADServicePrincipal -Role Owner -Scope "/subscriptions/$SubscriptionId" -DisplayName "test-resources-$($baseName).microsoft.com" + $suffix = (New-Guid).ToString('n').Substring(0, 4) + $global:AzureTestPrincipal = New-AzADServicePrincipal -Role Owner -Scope "/subscriptions/$SubscriptionId" -DisplayName "test-resources-$($baseName)$suffix.microsoft.com" $global:AzureTestSubscription = $SubscriptionId Log "Created service principal '$($AzureTestPrincipal.ApplicationId)'" From a2a5bbcec78f69fea3cf511e9e897fe2f4c85152 Mon Sep 17 00:00:00 2001 From: Scott Addie <10702007+scottaddie@users.noreply.github.com> Date: Mon, 30 Aug 2021 20:45:41 -0500 Subject: [PATCH 4/5] Clarify LogsQueryClient query parameter description (#20467) --- .../azure/monitor/query/_logs_query_client.py | 6 +++--- .../azure/monitor/query/aio/_logs_query_client_async.py | 8 ++++---- 2 files changed, 7 insertions(+), 7 deletions(-) diff --git a/sdk/monitor/azure-monitor-query/azure/monitor/query/_logs_query_client.py b/sdk/monitor/azure-monitor-query/azure/monitor/query/_logs_query_client.py index d2d04abb7d77..b555985d44aa 100644 --- a/sdk/monitor/azure-monitor-query/azure/monitor/query/_logs_query_client.py +++ b/sdk/monitor/azure-monitor-query/azure/monitor/query/_logs_query_client.py @@ -60,8 +60,8 @@ def query(self, workspace_id, query, **kwargs): :param workspace_id: ID of the workspace. This is Workspace ID from the Properties blade in the Azure portal. :type workspace_id: str - :param query: The Analytics query. Learn more about the `Analytics query syntax - `_. + :param query: The Kusto query. Learn more about the `Kusto query syntax + `_. :type query: str :keyword timespan: The timespan for which to query the data. This can be a timedelta, a timedelta and a start datetime, or a start datetime/end datetime. @@ -134,7 +134,7 @@ def query_batch(self, queries, **kwargs): The response is returned in the same order as that of the requests sent. - :param queries: The list of queries that should be processed + :param queries: The list of Kusto queries to execute. :type queries: list[dict] or list[~azure.monitor.query.LogsBatchQuery] :return: List of LogsQueryResult, or the result of cls(response) :rtype: list[~azure.monitor.query.LogsQueryResult] diff --git a/sdk/monitor/azure-monitor-query/azure/monitor/query/aio/_logs_query_client_async.py b/sdk/monitor/azure-monitor-query/azure/monitor/query/aio/_logs_query_client_async.py index 19994423c947..090fcceb9ed8 100644 --- a/sdk/monitor/azure-monitor-query/azure/monitor/query/aio/_logs_query_client_async.py +++ b/sdk/monitor/azure-monitor-query/azure/monitor/query/aio/_logs_query_client_async.py @@ -55,8 +55,8 @@ async def query( :param workspace_id: ID of the workspace. This is Workspace ID from the Properties blade in the Azure portal. :type workspace_id: str - :param query: The Analytics query. Learn more about the `Analytics query syntax - `_. + :param query: The Kusto query. Learn more about the `Kusto query syntax + `_. :type query: str :param timespan: The timespan for which to query the data. This can be a timedelta, a timedelta and a start datetime, or a start datetime/end datetime. @@ -69,7 +69,7 @@ async def query( visualization options. By default, the API does not return information regarding the type of visualization to show. If your client requires this information, specify the preference :keyword additional_workspaces: A list of workspaces that are included in the query. - These can be qualified workspace names, workspsce Ids or Azure resource Ids. + These can be qualified workspace names, workspace Ids or Azure resource Ids. :paramtype additional_workspaces: list[str] :return: QueryResults, or the result of cls(response) :rtype: ~azure.monitor.query.LogsQueryResult @@ -121,7 +121,7 @@ async def query_batch( The response is returned in the same order as that of the requests sent. - :param queries: The list of queries that should be processed + :param queries: The list of Kusto queries to execute. :type queries: list[dict] or list[~azure.monitor.query.LogsBatchQuery] :return: list of LogsQueryResult objects, or the result of cls(response) :rtype: list[~azure.monitor.query.LogsQueryResult] From 22d59ed4851126a71c9a9d2fd3dceb5a87fb6948 Mon Sep 17 00:00:00 2001 From: msyyc <70930885+msyyc@users.noreply.github.com> Date: Tue, 31 Aug 2021 14:41:02 +0800 Subject: [PATCH 5/5] release_iseus_status_auto_reply (#20441) * release_iseus_status_auto_reply * issue_aoto_close_revert * Update main.py * Update main.py * Update update_issue_body.py * Update reply_generator.py * Update reply_generator.py * Update update_issue_body.py * Update main.py * Update update_issue_body.py * Update update_issue_body.py * Update main.py * Update reply_generator.py * Update main.py * Update update_issue_body.py * Update main.py * Update scripts/release_issue_status/update_issue_body.py * Update update_issue_body.py * Update main.py * Update reply_generator.py * Update update_issue_body.py * Update main.py * Update scripts/release_issue_status/main.py * Update main.py * Update scripts/release_issue_status/main.py * Update scripts/release_issue_status/main.py * Update scripts/release_issue_status/main.py * Update scripts/release_issue_status/update_issue_body.py * Update scripts/release_issue_status/main.py * Update scripts/release_issue_status/update_issue_body.py * Update reply_generator.py * Update main.py * Update scripts/release_issue_status/update_issue_body.py Co-authored-by: msyyc <70930885+msyyc@users.noreply.github.com> * Update scripts/release_issue_status/update_issue_body.py Co-authored-by: msyyc <70930885+msyyc@users.noreply.github.com> * Update reply_generator.py * Update update_issue_body.py * Update main.py * Update main.py * Update update_issue_body.py * Update main.py * Update scripts/release_issue_status/main.py * Update main.py * Update reply_generator.py * Update main.py * Update main.py * Update main.py * Update main.py * Update main.py * Update main.py * Update main.py * Update main.py * Update main.py * Update update_issue_body.py * Update main.py * Update update_issue_body.py * Update reply_generator.py * Update update_issue_body.py * Update main.py * Update update_issue_body.py * Update update_issue_body.py * Update update_issue_body.py * Update update_issue_body.py * Update update_issue_body.py * Update main.py * Update main.py Co-authored-by: Zed <601306339@qq.com> Co-authored-by: Zed Lei <59104634+RAY-316@users.noreply.github.com> Co-authored-by: Jiefeng Chen <51037443+BigCat20196@users.noreply.github.com> --- scripts/release_issue_status/main.py | 108 +++++++++++++---- .../release_issue_status/reply_generator.py | 111 ++++++++++++++++++ .../release_issue_status/update_issue_body.py | 96 +++++++++++++++ 3 files changed, 295 insertions(+), 20 deletions(-) create mode 100644 scripts/release_issue_status/reply_generator.py create mode 100644 scripts/release_issue_status/update_issue_body.py diff --git a/scripts/release_issue_status/main.py b/scripts/release_issue_status/main.py index 9f9e5a35145d..6bdf38279c4f 100644 --- a/scripts/release_issue_status/main.py +++ b/scripts/release_issue_status/main.py @@ -5,12 +5,15 @@ from datetime import date, datetime import subprocess as sp from azure.storage.blob import BlobClient +import reply_generator as rg +from update_issue_body import update_issue_body, find_readme_link +import traceback _NULL = ' ' _FILE_OUT = 'release_issue_status.csv' +_FILE_OUT_PYTHON = 'release_python_status.md' _PYTHON_SDK_ADMINISTRATORS = {'msyyc', 'RAY-316', 'BigCat20196'} - def my_print(cmd): print('==' + cmd + ' ==\n') @@ -20,6 +23,20 @@ def print_check(cmd): sp.check_call(cmd, shell=True) +def output_python_md(issue_status_python): + with open(_FILE_OUT_PYTHON, 'w') as file_out: + file_out.write('| issue | author | package | assignee | bot advice | created date of issue | delay from created date |\n') + file_out.write('| ------ | ------ | ------ | ------ | ------ | ------ | :-----: |\n') + file_out.writelines([item.output_python() for item in sorted(issue_status_python, key=_key_select)]) + + +def output_csv(issue_status): + with open(_FILE_OUT, 'w') as file_out: + file_out.write('language,issue,author,package,created date,delay from created date,latest update time,' + 'delay from latest update,status,bot advice\n') + file_out.writelines([item.output() for item in sorted(issue_status, key=_key_select)]) + + class IssueStatus: link = _NULL author = _NULL @@ -36,6 +53,7 @@ class IssueStatus: whether_author_comment = True issue_object = _NULL labels = _NULL + assignee = _NULL def output(self): return '{},{},{},{},{},{},{},{},{},{}\n'.format(self.language, self.link, self.author, @@ -46,6 +64,13 @@ def output(self): self.delay_from_latest_update, self.status, self.bot_advice) + + def output_python(self): + return '| [#{}]({}) | {} | {} | {} | {} | {} | {} |\n'.format(self.link.split('/')[-1], self.link, self.author, + self.package, self.assignee, self.bot_advice, + str(date.fromtimestamp(self.create_date)), + self.delay_from_create_date) + def _extract(str_list, key_word): for item in str_list: @@ -86,8 +111,7 @@ def _extract_author_latest_comment(comments): def _whether_author_comment(comments): q = set(comment.user.login for comment in comments) diff = q.difference(_PYTHON_SDK_ADMINISTRATORS) - - return len(diff) > 0 + return len(diff) > 0 def _latest_comment_time(comments, delay_from_create_date): q = [(comment.updated_at.timestamp(), comment.user.login) @@ -96,13 +120,50 @@ def _latest_comment_time(comments, delay_from_create_date): return delay_from_create_date if not q else int((time.time() - q[-1][0]) / 3600 / 24) + +def auto_reply(item, sdk_repo, rest_repo, duplicated_issue): + print("==========new issue number: {}".format(item.issue_object.number)) + if 'auto-link' not in item.labels: + try: + package_name, readme_link = update_issue_body(sdk_repo, rest_repo, item.issue_object.number) + print("pkname, readme", package_name, readme_link) + item.package = package_name + key = ('Python', item.package) + duplicated_issue[key] = duplicated_issue.get(key, 0) + 1 + except Exception as e: + item.bot_advice = 'failed to modify the body of the new issue. Please modify manually' + item.labels.append('attention') + print(e) + raise + item.labels.append('auto-link') + item.issue_object.set_labels(*item.labels) + else: + try: + readme_link = find_readme_link(sdk_repo, item.issue_object.number) + except Exception as e: + print('Issue: {} updates body failed'.format(item.issue_object.number)) + item.bot_advice = 'failed to find Readme link, Please check !!' + item.labels.append('attention') + raise + try: + reply = rg.begin_reply_generate(item=item, rest_repo=rest_repo, readme_link=readme_link) + except Exception as e: + item.bot_advice = 'auto reply failed, Please intervene manually !!' + print('Error from auto reply ========================') + print('Issue:{}'.format(item.issue_object.number)) + print(traceback.format_exc()) + print('==============================================') + + def main(): # get latest issue status g = Github(os.getenv('TOKEN')) # please fill user_token - repo = g.get_repo('Azure/sdk-release-request') - label1 = repo.get_label('ManagementPlane') - open_issues = repo.get_issues(state='open', labels=[label1]) + sdk_repo = g.get_repo('Azure/sdk-release-request') + rest_repo = g.get_repo('Azure/azure-rest-api-specs') + label1 = sdk_repo.get_label('ManagementPlane') + open_issues = sdk_repo.get_issues(state='open', labels=[label1]) issue_status = [] + issue_status_python = [] duplicated_issue = dict() start_time = time.time() for item in open_issues: @@ -124,7 +185,9 @@ def main(): issue.issue_object = item issue.labels = [label.name for label in item.labels] issue.days_from_latest_commit = _latest_comment_time(item.get_comments(), issue.delay_from_create_date) - + if item.assignee: + issue.assignee = item.assignee.login + issue_status.append(issue) key = (issue.language, issue.package) duplicated_issue[key] = duplicated_issue.get(key, 0) + 1 @@ -135,18 +198,23 @@ def main(): # rule2: if latest comment is from author, need response asap # rule3: if comment num is 0, it is new issue, better to deal with it asap # rule4: if delay from latest update is over 7 days, better to deal with it soon. - # rule5: if delay from created date is over 30 days and owner never reply, close it. - # rule6: if delay from created date is over 15 days and owner never reply, remind owner to handle it. + # rule5: if delay from created date is over 30 days, better to close. + # rule6: if delay from created date is over 30 days and owner never reply, close it. + # rule7: if delay from created date is over 15 days and owner never reply, remind owner to handle it. for item in issue_status: if item.status == 'release': item.bot_advice = 'better to release asap.' - elif item.author == item.author_latest_comment: - item.bot_advice = 'new comment for author.' - elif item.comment_num == 0: + elif item.comment_num == 0 and 'Python' in item.labels: item.bot_advice = 'new issue and better to confirm quickly.' + try: + auto_reply(item, sdk_repo, rest_repo, duplicated_issue) + except Exception as e: + continue + elif not item.author_latest_comment in _PYTHON_SDK_ADMINISTRATORS: + item.bot_advice = 'new comment for author.' elif item.delay_from_latest_update >= 7: item.bot_advice = 'delay for a long time and better to handle now.' - + if item.days_from_latest_commit >= 30 and item.language == 'Python' and '30days attention' not in item.labels: item.labels.append('30days attention') item.issue_object.set_labels(*item.labels) @@ -157,17 +225,17 @@ def main(): ' please deal with it ASAP. We will close the issue if there is still no response after 15 days!') item.labels.append('15days attention') item.issue_object.set_labels(*item.labels) - - + # judge whether there is duplicated issue for same package if item.package != _NULL and duplicated_issue.get((item.language, item.package)) > 1: item.bot_advice = f'Warning:There is duplicated issue for {item.package}. ' + item.bot_advice + + if item.language == 'Python': + issue_status_python.append(item) # output result - with open(_FILE_OUT, 'w') as file_out: - file_out.write('language,issue,author,package,created date,delay from created date,latest update time,' - 'delay from latest update,status,bot advice\n') - file_out.writelines([item.output() for item in sorted(issue_status, key=_key_select)]) + output_python_md(issue_status_python) + output_csv(issue_status) # commit to github print_check('git add .') @@ -179,7 +247,7 @@ def main(): blob_name=_FILE_OUT) with open(_FILE_OUT, 'rb') as data: blob.upload_blob(data, overwrite=True) - + if __name__ == '__main__': main() diff --git a/scripts/release_issue_status/reply_generator.py b/scripts/release_issue_status/reply_generator.py new file mode 100644 index 000000000000..dbe6494bd7d1 --- /dev/null +++ b/scripts/release_issue_status/reply_generator.py @@ -0,0 +1,111 @@ +import re + +issue_object_rg = None + + +def weather_change_readme(rest_repo, link_dict, labels): + # to see whether need change readme + contents = str(rest_repo.get_contents(link_dict['readme_path']).decoded_content) + pattern_tag = re.compile(r'tag: package-[\w+-.]+') + package_tag = pattern_tag.search(contents).group() + package_tag = package_tag.split(':')[1].strip() + readme_python_contents = str(rest_repo.get_contents(link_dict['readme_python_path']).decoded_content) + whether_multi_api = 'multi-api' in readme_python_contents + whether_same_tag = package_tag == link_dict['readme_tag'] + whether_change_readme = not whether_same_tag or whether_multi_api and not 'MultiAPI' in labels + return whether_change_readme + + +# parse owner's comment and get links +def get_links(readme_link): + link_dict = {} + comment_body = issue_object_rg.body + pattern_readme = re.compile(r'/specification/([\w-]+/)+readme.md') + pattern_resource_manager = re.compile(r'/specification/([\w-]+/)+resource-manager') + pattern_tag = re.compile(r'package-[\w+-.]+') + readme_path = pattern_readme.search(readme_link).group() + readme_tag = pattern_tag.search(comment_body).group() + resource_manager = pattern_resource_manager.search(readme_link).group() + link_dict['readme_path'] = readme_path + link_dict['readme_python_path'] = readme_path[:readme_path.rfind('/')] + '/readme.python.md' + link_dict['readme_tag'] = readme_tag + link_dict['resource_manager'] = resource_manager + return link_dict + + +def get_latest_pr_from_readme(rest_repo, link_dict): + commits = rest_repo.get_commits(path=link_dict['resource_manager']) + latest_commit = [commit for commit in commits][0] + latest_pr_brief = latest_commit.commit.message + latest_pr_number = re.findall('\(\#[0-9]+\)', latest_pr_brief) + latest_pr_number_int = [] + for number in latest_pr_number: + number = int(re.search('\d+', number).group()) + latest_pr_number_int.append(number) + latest_pr_number_int.sort() + + return latest_pr_number_int[-1] + + +def latest_pr_parse(rest_repo, latest_pr_number): + latest_pr = rest_repo.get_issue(latest_pr_number) + latest_pr_comments = latest_pr.get_comments() + b = [i for i in latest_pr_comments] + for comment in latest_pr_comments: + if '

Swagger Generation Artifacts

' in comment.body: + return swagger_generator_parse(comment.body, latest_pr_number) + + +def swagger_generator_parse(context, latest_pr_number): + track1_info_model = '' + try: + if ' azure-sdk-for-python' in context: + pattern_python_t1 = re.compile(' azure-sdk-for-python.+?', re.DOTALL) + python_t1 = re.search(pattern_python_t1, context).group() + prttern_python_track1 = re.compile('
    \s+?
  • \s+?', re.DOTALL) + python_track1_info = re.search(prttern_python_track1, python_t1).group() + track1_info_model = '
    python-track1{}
    '.format( + python_track1_info) + except Exception as e: + print('track1 generate error') + pattern_python = re.compile(' azure-sdk-for-python-track2.+?', re.DOTALL) + python = re.search(pattern_python, context).group() + # the way that reply not contains [Release SDK Changes] + # pattern_python_track2 = re.compile('
      \s*?
    • \s*?', re.DOTALL) + pattern_python_track2 = re.compile('track2_.*
    ', re.DOTALL) + python_track2_info = re.search(pattern_python_track2, python).group() + track2_info_model = '
    python-track2{}
    '.format( + python_track2_info) + info_model = 'hi @{} Please check the package whether works well and the changelog info is as below:\n' \ + '{}\n{}\n' \ + '\n* (The version of the package is only a temporary version for testing)\n' \ + '\nhttps://github.com/Azure/azure-rest-api-specs/pull/{}\n' \ + .format(issue_object_rg.user.login, track1_info_model, track2_info_model, str(latest_pr_number)) + + return info_model + + +def reply_owner(reply_content): + issue_object_rg.create_comment(reply_content) + + +def add_label(label_name, labels): + if label_name not in labels: + labels.append(label_name) + issue_object_rg.set_labels(*labels) + + +def begin_reply_generate(item, rest_repo, readme_link): + global issue_object_rg + issue_object_rg = item.issue_object + link_dict = get_links(readme_link) + labels = item.labels + whether_change_readme = weather_change_readme(rest_repo, link_dict, labels) + + if not whether_change_readme: + latest_pr_number = get_latest_pr_from_readme(rest_repo,link_dict) + reply_content = latest_pr_parse(rest_repo, latest_pr_number) + reply_owner(reply_content) + add_label('auto-ask-check', labels) + else: + print('issue {} need config readme***********'.format(issue_object_rg.number)) diff --git a/scripts/release_issue_status/update_issue_body.py b/scripts/release_issue_status/update_issue_body.py new file mode 100644 index 000000000000..718bc906fc80 --- /dev/null +++ b/scripts/release_issue_status/update_issue_body.py @@ -0,0 +1,96 @@ +import re + + +def update_issue_body(sdk_repo, rest_repo, issue_number): + # Get Issue Number + issue_info = sdk_repo.get_issue(number=issue_number) + issue_body = issue_info.body + issue_body_list = [i for i in issue_body.split("\n") if i] + # Get the link and readme tag in issue body + link, readme_tag = '', '' + for row in issue_body_list: + if 'link' in row.lower(): + link = row.split(":", 1)[-1].strip() + if 'readme tag' in row.lower(): + readme_tag = row.split(":", 1)[-1].strip() + if link and readme_tag: + break + + if link.count('https') > 1: + link = link.split(']')[0] + link = link.replace('[', "").replace(']', "").replace('(', "").replace(')', "") + + package_name, readme_link = get_pkname_and_readme_link(rest_repo, link) + + # Check readme tag format + if 'package' not in readme_tag: + readme_tag = 'package-{}'.format(readme_tag) + issue_body_list.insert(0, f'Readme Tag: {readme_tag}') + + issue_body_list.insert(0, f'\n{readme_link.replace("/readme.md", "")}') + issue_body_list.insert(1, package_name) + issue_body_up = '' + for raw in issue_body_list: + if raw == '---\r' or raw == '---': + issue_body_up += '\n' + issue_body_up += raw + '\n' + + issue_info.edit(body=issue_body_up) + return package_name, readme_link + + +def get_pkname_and_readme_link(rest_repo, link): + # change commit link to pull json link(i.e. https://github.com/Azure/azure-rest-api-specs/commit/77f5d3b5d2fbae17621ea124485788f496786758#diff-708c2fb843b022cac4af8c6f996a527440c1e0d328abb81f54670747bf14ab1a) + pk_name = '' + if 'commit' in link: + commit_sha = link.split('commit/')[-1] + commit = rest_repo.get_commit(commit_sha) + link = commit.files[0].blob_url + + # if link is a pr, it can get both pakeage name and readme link. + if 'pull' in link: + pr_number = int(link.replace("https://github.com/Azure/azure-rest-api-specs/pull/", "").strip('/')) + + # Get Readme link + pr_info = rest_repo.get_pull(number=pr_number) + pk_url_name = set() + for pr_changed_file in pr_info.get_files(): + contents_url = pr_changed_file.contents_url + if '/resource-manager' in contents_url: + try: + pk_url_name.add(re.findall(r'/specification/(.*?)/resource-manager/', contents_url)[0]) + except Exception as e: + continue + if len(pk_url_name) > 1: + print("\nexists multiple package names: {}, {} \n".format(pk_url_name, pk_url_name1)) + raise Exception('Not find readme link, because it exists multiple package names') + + readme_link = 'https://github.com/Azure/azure-rest-api-specs/blob/main/specification/{}/' \ + 'resource-manager/readme.python.md'.format(list(pk_url_name)[0]) + + + # if link is a rest url(i.e. https://github.com/Azure/azure-rest-api-specs/blob/main/specification/xxx/resource-manager/readme.python.md) + elif '/resource-manager' not in link: + # (i.e. https://github.com/Azure/azure-rest-api-specs/tree/main/specification/xxxx) + readme_link = link + '/resource-manager/readme.python.md' + else: + readme_link = link.split('/resource-manager')[0] + '/resource-manager/readme.python.md' + # get the package name by readme link + readme_link_part = '/specification' + readme_link.split('/specification')[-1] + readme_contents = str(rest_repo.get_contents(readme_link_part).decoded_content) + pk_name = re.findall(r'package-name: (.*?)\\n', readme_contents)[0] + readme_link = readme_link.replace('python.', '') + + return pk_name, readme_link + + +def find_readme_link(sdk_repo, issue_number): + # Get Issue Number + issue_info = sdk_repo.get_issue(number=issue_number) + issue_body = issue_info.body + issue_body_list = issue_body.split("\n") + for row in issue_body_list: + if 'resource-manager' in row: + readme_link = row + '/readme.md' + return readme_link + raise Exception('Not find readme link,please check')