From 2a7b0f82ecd9c616b6ee1cf1fa4f249d0cbeda34 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen Date: Tue, 11 Apr 2023 15:53:27 -0700 Subject: [PATCH 01/11] No cluster environment Signed-off-by: Cuong Nguyen --- .../air_tests/air_benchmarks/app_config.yaml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/release/air_tests/air_benchmarks/app_config.yaml b/release/air_tests/air_benchmarks/app_config.yaml index 72118540c061..4e03f43b6b83 100644 --- a/release/air_tests/air_benchmarks/app_config.yaml +++ b/release/air_tests/air_benchmarks/app_config.yaml @@ -1,13 +1,13 @@ base_image: {{ env["RAY_IMAGE_ML_NIGHTLY_GPU"] | default("anyscale/ray-ml:nightly-py37-gpu") }} -env_vars: {} -debian_packages: - - curl - -python: - pip_packages: - - pytest - conda_packages: [] - -post_build_cmds: - - pip3 uninstall ray -y || true && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }} - - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }} +#env_vars: {} +#debian_packages: +# - curl +# +#python: +# pip_packages: +# - pytest +# conda_packages: [] +# +#post_build_cmds: +# - pip3 uninstall ray -y || true && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }} +# - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }} \ No newline at end of file From 81758dcfec0f450db17dd8d9ff56f6c98f7550f0 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen Date: Tue, 11 Apr 2023 16:28:42 -0700 Subject: [PATCH 02/11] Bisect Signed-off-by: Cuong Nguyen --- release/ray_release/scripts/bisect.py | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) create mode 100644 release/ray_release/scripts/bisect.py diff --git a/release/ray_release/scripts/bisect.py b/release/ray_release/scripts/bisect.py new file mode 100644 index 000000000000..29eb6add38ca --- /dev/null +++ b/release/ray_release/scripts/bisect.py @@ -0,0 +1,22 @@ +import click +import logger +import subprocess +from typing import List + +@click.command() +@click.argument("test_name", required=True, type=str) +@click.argument("passing_commit", required=True, type=str) +@click.argument("failing_commit", required=True, type=str) + +def _get_commit_lists(passing_commit: str, failing_commit: str) -> List[str]: + try + subprocess.check_output( + ) + except Exception as e: + logger.info(f'Failed to get commit list: {e}') + +def main(test_name: str, passing_commit: str, failing_commit: str) -> None: + return None + +if __name__ == "__main__": + main() From d0d1eaf3e756b7eaf9185625c2d04788976a4e80 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen Date: Tue, 11 Apr 2023 16:57:16 -0700 Subject: [PATCH 03/11] Skeleton for bisect job Signed-off-by: Cuong Nguyen --- release/ray_release/scripts/bisect.py | 33 +++++++++++++++++++-------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/release/ray_release/scripts/bisect.py b/release/ray_release/scripts/bisect.py index 29eb6add38ca..87f1e7a04919 100644 --- a/release/ray_release/scripts/bisect.py +++ b/release/ray_release/scripts/bisect.py @@ -1,5 +1,5 @@ import click -import logger +import math import subprocess from typing import List @@ -7,16 +7,29 @@ @click.argument("test_name", required=True, type=str) @click.argument("passing_commit", required=True, type=str) @click.argument("failing_commit", required=True, type=str) +def main(test_name: str, passing_commit: str, failing_commit: str) -> None: + lists = _get_commit_lists(passing_commit, failing_commit) + while len(lists) > 1: + middle_commit = lists[math.floor(len(lists)/2)] + is_passing = _run_test(test_name, middle_commit) + if is_passing: + passing_commit = middle_commit + else: + failing_commit = middle_commit + lists = _get_commit_lists(passing_commit, failing_commit) + break + + print(f'Blamed commit found: {failing_commit}') -def _get_commit_lists(passing_commit: str, failing_commit: str) -> List[str]: - try - subprocess.check_output( - ) - except Exception as e: - logger.info(f'Failed to get commit list: {e}') +def _run_test(test_name: str, failing_commit: str) -> bool: + return True -def main(test_name: str, passing_commit: str, failing_commit: str) -> None: - return None +def _get_commit_lists(passing_commit: str, failing_commit: str) -> List[str]: + commit_lists = subprocess.check_output( + f'git rev-list --ancestry-path {passing_commit}..{failing_commit}', + shell=True, + ) + return commit_lists.decode('utf-8').split("\n") if __name__ == "__main__": - main() + main() \ No newline at end of file From 86fef4a99e564446b15c827d1f375ce077e529d4 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen Date: Wed, 12 Apr 2023 09:28:09 -0700 Subject: [PATCH 04/11] Rename script Signed-off-by: Cuong Nguyen --- release/ray_release/scripts/{bisect.py => ray_bisect.py} | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename release/ray_release/scripts/{bisect.py => ray_bisect.py} (100%) diff --git a/release/ray_release/scripts/bisect.py b/release/ray_release/scripts/ray_bisect.py similarity index 100% rename from release/ray_release/scripts/bisect.py rename to release/ray_release/scripts/ray_bisect.py From ee110e63361c9811f6b3a9a82ebac4cc14c71e08 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen Date: Wed, 12 Apr 2023 10:23:43 -0700 Subject: [PATCH 05/11] Add unit tests Signed-off-by: Cuong Nguyen --- release/ray_release/scripts/ray_bisect.py | 10 ++++++---- release/ray_release/tests/test_bisect.py | 16 ++++++++++++++++ 2 files changed, 22 insertions(+), 4 deletions(-) create mode 100644 release/ray_release/tests/test_bisect.py diff --git a/release/ray_release/scripts/ray_bisect.py b/release/ray_release/scripts/ray_bisect.py index 87f1e7a04919..0ca9b5c1614a 100644 --- a/release/ray_release/scripts/ray_bisect.py +++ b/release/ray_release/scripts/ray_bisect.py @@ -8,6 +8,10 @@ @click.argument("passing_commit", required=True, type=str) @click.argument("failing_commit", required=True, type=str) def main(test_name: str, passing_commit: str, failing_commit: str) -> None: + blamed_commit = _bisect(test_name, passing_commit, failing_commit) + print(f'Blamed commit found: {failing_commit}') + +def _bisect(test_name: str, passing_commit: str, failing_commit: str) -> str: lists = _get_commit_lists(passing_commit, failing_commit) while len(lists) > 1: middle_commit = lists[math.floor(len(lists)/2)] @@ -17,11 +21,9 @@ def main(test_name: str, passing_commit: str, failing_commit: str) -> None: else: failing_commit = middle_commit lists = _get_commit_lists(passing_commit, failing_commit) - break - - print(f'Blamed commit found: {failing_commit}') + return failing_commit -def _run_test(test_name: str, failing_commit: str) -> bool: +def _run_test(test_name: str, commit: str) -> bool: return True def _get_commit_lists(passing_commit: str, failing_commit: str) -> List[str]: diff --git a/release/ray_release/tests/test_bisect.py b/release/ray_release/tests/test_bisect.py new file mode 100644 index 000000000000..d5cd0747005c --- /dev/null +++ b/release/ray_release/tests/test_bisect.py @@ -0,0 +1,16 @@ +from typing import List +from ray_release.scripts.ray_bisect import ( + _bisect +) + +def test_bisect(): + commit_to_test_result = { + 'c0': True, + 'c1': True, + 'c2': True, + 'c3': False, + 'c4': False, + } + def _mock_get_commit_lists(passing_commit: str, failing_commit: str) -> List[str]: + blamed_commit = _bisect('test', 'c0', 'c4') + assert blamed_commit == 'c3' \ No newline at end of file From 28dd8f96044843cbefac5ccbaf9b48c58d5ed8f2 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen Date: Wed, 12 Apr 2023 10:56:49 -0700 Subject: [PATCH 06/11] Add unit tests Signed-off-by: Cuong Nguyen --- release/ray_release/scripts/ray_bisect.py | 2 +- release/ray_release/tests/test_bisect.py | 30 +++++++++++++++++++---- 2 files changed, 26 insertions(+), 6 deletions(-) diff --git a/release/ray_release/scripts/ray_bisect.py b/release/ray_release/scripts/ray_bisect.py index 0ca9b5c1614a..23db473daecf 100644 --- a/release/ray_release/scripts/ray_bisect.py +++ b/release/ray_release/scripts/ray_bisect.py @@ -9,7 +9,7 @@ @click.argument("failing_commit", required=True, type=str) def main(test_name: str, passing_commit: str, failing_commit: str) -> None: blamed_commit = _bisect(test_name, passing_commit, failing_commit) - print(f'Blamed commit found: {failing_commit}') + print(f'Blamed commit found: {blamed_commit}') def _bisect(test_name: str, passing_commit: str, failing_commit: str) -> str: lists = _get_commit_lists(passing_commit, failing_commit) diff --git a/release/ray_release/tests/test_bisect.py b/release/ray_release/tests/test_bisect.py index d5cd0747005c..14e22b70a151 100644 --- a/release/ray_release/tests/test_bisect.py +++ b/release/ray_release/tests/test_bisect.py @@ -1,7 +1,7 @@ from typing import List -from ray_release.scripts.ray_bisect import ( - _bisect -) +from unittest import mock +from ray_release.scripts.ray_bisect import _bisect + def test_bisect(): commit_to_test_result = { @@ -12,5 +12,25 @@ def test_bisect(): 'c4': False, } def _mock_get_commit_lists(passing_commit: str, failing_commit: str) -> List[str]: - blamed_commit = _bisect('test', 'c0', 'c4') - assert blamed_commit == 'c3' \ No newline at end of file + commits = [] + in_range = False + for commit in commit_to_test_result: + if commit == failing_commit: + break + if in_range: + commits.append(commit) + if commit == passing_commit: + commits.append(commit) + in_range = True + return commits + def _mock_run_test(test_name: str, commit: str) -> bool: + return commit_to_test_result[commit] + with mock.patch( + 'ray_release.scripts.ray_bisect._get_commit_lists', + side_effect=_mock_get_commit_lists, + ), mock.patch( + 'ray_release.scripts.ray_bisect._run_test', + side_effect=_mock_run_test, + ): + blamed_commit = _bisect('test', 'c0', 'c4') + assert blamed_commit == 'c3' From a93db72bdeed73240b2c59cf27924b1cdea639f5 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen Date: Wed, 12 Apr 2023 11:12:01 -0700 Subject: [PATCH 07/11] Lints Signed-off-by: Cuong Nguyen --- release/ray_release/scripts/ray_bisect.py | 15 ++++++++++----- release/ray_release/tests/test_bisect.py | 21 ++++++++++++--------- 2 files changed, 22 insertions(+), 14 deletions(-) diff --git a/release/ray_release/scripts/ray_bisect.py b/release/ray_release/scripts/ray_bisect.py index 23db473daecf..5eb7b5dbe98e 100644 --- a/release/ray_release/scripts/ray_bisect.py +++ b/release/ray_release/scripts/ray_bisect.py @@ -3,18 +3,20 @@ import subprocess from typing import List + @click.command() @click.argument("test_name", required=True, type=str) @click.argument("passing_commit", required=True, type=str) @click.argument("failing_commit", required=True, type=str) def main(test_name: str, passing_commit: str, failing_commit: str) -> None: blamed_commit = _bisect(test_name, passing_commit, failing_commit) - print(f'Blamed commit found: {blamed_commit}') + print(f"Blamed commit found: {blamed_commit}") + def _bisect(test_name: str, passing_commit: str, failing_commit: str) -> str: lists = _get_commit_lists(passing_commit, failing_commit) while len(lists) > 1: - middle_commit = lists[math.floor(len(lists)/2)] + middle_commit = lists[math.floor(len(lists) / 2)] is_passing = _run_test(test_name, middle_commit) if is_passing: passing_commit = middle_commit @@ -23,15 +25,18 @@ def _bisect(test_name: str, passing_commit: str, failing_commit: str) -> str: lists = _get_commit_lists(passing_commit, failing_commit) return failing_commit + def _run_test(test_name: str, commit: str) -> bool: return True + def _get_commit_lists(passing_commit: str, failing_commit: str) -> List[str]: commit_lists = subprocess.check_output( - f'git rev-list --ancestry-path {passing_commit}..{failing_commit}', + f"git rev-list --ancestry-path {passing_commit}..{failing_commit}", shell=True, ) - return commit_lists.decode('utf-8').split("\n") + return commit_lists.decode("utf-8").split("\n") + if __name__ == "__main__": - main() \ No newline at end of file + main() diff --git a/release/ray_release/tests/test_bisect.py b/release/ray_release/tests/test_bisect.py index 14e22b70a151..e9954372a069 100644 --- a/release/ray_release/tests/test_bisect.py +++ b/release/ray_release/tests/test_bisect.py @@ -5,12 +5,13 @@ def test_bisect(): commit_to_test_result = { - 'c0': True, - 'c1': True, - 'c2': True, - 'c3': False, - 'c4': False, + "c0": True, + "c1": True, + "c2": True, + "c3": False, + "c4": False, } + def _mock_get_commit_lists(passing_commit: str, failing_commit: str) -> List[str]: commits = [] in_range = False @@ -23,14 +24,16 @@ def _mock_get_commit_lists(passing_commit: str, failing_commit: str) -> List[str commits.append(commit) in_range = True return commits + def _mock_run_test(test_name: str, commit: str) -> bool: return commit_to_test_result[commit] + with mock.patch( - 'ray_release.scripts.ray_bisect._get_commit_lists', + "ray_release.scripts.ray_bisect._get_commit_lists", side_effect=_mock_get_commit_lists, ), mock.patch( - 'ray_release.scripts.ray_bisect._run_test', + "ray_release.scripts.ray_bisect._run_test", side_effect=_mock_run_test, ): - blamed_commit = _bisect('test', 'c0', 'c4') - assert blamed_commit == 'c3' + blamed_commit = _bisect("test", "c0", "c4") + assert blamed_commit == "c3" From f4c45193ce64ec169c989b3c7877b429240cb8b4 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen Date: Wed, 12 Apr 2023 11:13:29 -0700 Subject: [PATCH 08/11] Undo irrelevant changes Signed-off-by: Cuong Nguyen --- .../air_tests/air_benchmarks/app_config.yaml | 24 +++++++++---------- 1 file changed, 12 insertions(+), 12 deletions(-) diff --git a/release/air_tests/air_benchmarks/app_config.yaml b/release/air_tests/air_benchmarks/app_config.yaml index 4e03f43b6b83..29db6e63a49f 100644 --- a/release/air_tests/air_benchmarks/app_config.yaml +++ b/release/air_tests/air_benchmarks/app_config.yaml @@ -1,13 +1,13 @@ base_image: {{ env["RAY_IMAGE_ML_NIGHTLY_GPU"] | default("anyscale/ray-ml:nightly-py37-gpu") }} -#env_vars: {} -#debian_packages: -# - curl -# -#python: -# pip_packages: -# - pytest -# conda_packages: [] -# -#post_build_cmds: -# - pip3 uninstall ray -y || true && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }} -# - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }} \ No newline at end of file +env_vars: {} +debian_packages: + - curl + +python: + pip_packages: + - pytest + conda_packages: [] + +post_build_cmds: + - pip3 uninstall ray -y || true && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }} + - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }} \ No newline at end of file From f3779c53d1725d5da704886a1f2e760c45e20937 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen Date: Wed, 12 Apr 2023 11:13:53 -0700 Subject: [PATCH 09/11] Remove irrelevant changes Signed-off-by: Cuong Nguyen --- release/air_tests/air_benchmarks/app_config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/release/air_tests/air_benchmarks/app_config.yaml b/release/air_tests/air_benchmarks/app_config.yaml index 29db6e63a49f..72118540c061 100644 --- a/release/air_tests/air_benchmarks/app_config.yaml +++ b/release/air_tests/air_benchmarks/app_config.yaml @@ -10,4 +10,4 @@ python: post_build_cmds: - pip3 uninstall ray -y || true && pip3 install -U {{ env["RAY_WHEELS"] | default("ray") }} - - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }} \ No newline at end of file + - {{ env["RAY_WHEELS_SANITY_CHECK"] | default("echo No Ray wheels sanity check") }} From 43f720f6747bc6a72c7300660a649abc6e30251f Mon Sep 17 00:00:00 2001 From: Cuong Nguyen Date: Thu, 13 Apr 2023 09:47:16 -0700 Subject: [PATCH 10/11] @krfricke's comments Signed-off-by: Cuong Nguyen --- release/ray_release/scripts/ray_bisect.py | 21 ++++++++++----------- release/ray_release/tests/test_bisect.py | 18 +----------------- 2 files changed, 11 insertions(+), 28 deletions(-) diff --git a/release/ray_release/scripts/ray_bisect.py b/release/ray_release/scripts/ray_bisect.py index 5eb7b5dbe98e..55219474c012 100644 --- a/release/ray_release/scripts/ray_bisect.py +++ b/release/ray_release/scripts/ray_bisect.py @@ -1,5 +1,4 @@ import click -import math import subprocess from typing import List @@ -9,21 +8,21 @@ @click.argument("passing_commit", required=True, type=str) @click.argument("failing_commit", required=True, type=str) def main(test_name: str, passing_commit: str, failing_commit: str) -> None: - blamed_commit = _bisect(test_name, passing_commit, failing_commit) + commit_lists = _get_commit_lists(passing_commit, failing_commit) + blamed_commit = _bisect(test_name, commit_lists) print(f"Blamed commit found: {blamed_commit}") -def _bisect(test_name: str, passing_commit: str, failing_commit: str) -> str: - lists = _get_commit_lists(passing_commit, failing_commit) - while len(lists) > 1: - middle_commit = lists[math.floor(len(lists) / 2)] +def _bisect(test_name: str, commit_list: List[str]) -> str: + while len(commit_list) > 1: + middle_commit_idx = len(commit_list) // 2 + middle_commit = commit_list[middle_commit_idx] is_passing = _run_test(test_name, middle_commit) if is_passing: - passing_commit = middle_commit - else: - failing_commit = middle_commit - lists = _get_commit_lists(passing_commit, failing_commit) - return failing_commit + commit_list = commit_list[middle_commit_idx + 1:] + else: + commit_list = commit_list[:middle_commit_idx] + return commit_list[-1] def _run_test(test_name: str, commit: str) -> bool: diff --git a/release/ray_release/tests/test_bisect.py b/release/ray_release/tests/test_bisect.py index e9954372a069..4f61cefd6ba2 100644 --- a/release/ray_release/tests/test_bisect.py +++ b/release/ray_release/tests/test_bisect.py @@ -12,28 +12,12 @@ def test_bisect(): "c4": False, } - def _mock_get_commit_lists(passing_commit: str, failing_commit: str) -> List[str]: - commits = [] - in_range = False - for commit in commit_to_test_result: - if commit == failing_commit: - break - if in_range: - commits.append(commit) - if commit == passing_commit: - commits.append(commit) - in_range = True - return commits - def _mock_run_test(test_name: str, commit: str) -> bool: return commit_to_test_result[commit] with mock.patch( - "ray_release.scripts.ray_bisect._get_commit_lists", - side_effect=_mock_get_commit_lists, - ), mock.patch( "ray_release.scripts.ray_bisect._run_test", side_effect=_mock_run_test, ): - blamed_commit = _bisect("test", "c0", "c4") + blamed_commit = _bisect("test", list(commit_to_test_result.keys())) assert blamed_commit == "c3" From bb32fd74854329ae589b379720a40dd0748804e6 Mon Sep 17 00:00:00 2001 From: Cuong Nguyen Date: Fri, 14 Apr 2023 08:19:20 -0700 Subject: [PATCH 11/11] Fix lint Signed-off-by: Cuong Nguyen --- release/ray_release/scripts/ray_bisect.py | 4 ++-- release/ray_release/tests/test_bisect.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/release/ray_release/scripts/ray_bisect.py b/release/ray_release/scripts/ray_bisect.py index 55219474c012..f0b2bb2edd62 100644 --- a/release/ray_release/scripts/ray_bisect.py +++ b/release/ray_release/scripts/ray_bisect.py @@ -19,8 +19,8 @@ def _bisect(test_name: str, commit_list: List[str]) -> str: middle_commit = commit_list[middle_commit_idx] is_passing = _run_test(test_name, middle_commit) if is_passing: - commit_list = commit_list[middle_commit_idx + 1:] - else: + commit_list = commit_list[middle_commit_idx + 1 :] + else: commit_list = commit_list[:middle_commit_idx] return commit_list[-1] diff --git a/release/ray_release/tests/test_bisect.py b/release/ray_release/tests/test_bisect.py index 4f61cefd6ba2..134f746260e6 100644 --- a/release/ray_release/tests/test_bisect.py +++ b/release/ray_release/tests/test_bisect.py @@ -1,4 +1,3 @@ -from typing import List from unittest import mock from ray_release.scripts.ray_bisect import _bisect