From 9d616ea95761193c8daa3e03b8311d856f95d687 Mon Sep 17 00:00:00 2001 From: IAmAlife Date: Fri, 8 Apr 2022 16:43:50 +0200 Subject: [PATCH 01/10] Creates a Dict from a given json List --- .idea/api-editor.iml | 8 ++- .idea/misc.xml | 2 +- package-parser/package-parser.iml | 6 ++ .../commands/generate_annotations/__init__.py | 0 .../_generate_unused_annotations.py | 39 +++++++++++++ .../commands/generate_annotations/__init__.py | 0 .../test_generate_unused_annotations.py | 58 +++++++++++++++++++ .../unused_functions_list.json | 6 ++ 8 files changed, 117 insertions(+), 2 deletions(-) create mode 100644 package-parser/package_parser/commands/generate_annotations/__init__.py create mode 100644 package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py create mode 100644 package-parser/tests/commands/generate_annotations/__init__.py create mode 100644 package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py create mode 100644 package-parser/tests/commands/generate_annotations/unused_functions_list.json diff --git a/.idea/api-editor.iml b/.idea/api-editor.iml index d6ebd4805..9b9c1652f 100644 --- a/.idea/api-editor.iml +++ b/.idea/api-editor.iml @@ -3,7 +3,13 @@ - + + + + + \ No newline at end of file diff --git a/.idea/misc.xml b/.idea/misc.xml index 09adcc7f2..66d44104f 100644 --- a/.idea/misc.xml +++ b/.idea/misc.xml @@ -1,5 +1,5 @@ - + \ No newline at end of file diff --git a/package-parser/package-parser.iml b/package-parser/package-parser.iml index 904bf2ca5..8a3e7597c 100644 --- a/package-parser/package-parser.iml +++ b/package-parser/package-parser.iml @@ -11,4 +11,10 @@ + + + + \ No newline at end of file diff --git a/package-parser/package_parser/commands/generate_annotations/__init__.py b/package-parser/package_parser/commands/generate_annotations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py b/package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py new file mode 100644 index 000000000..e5a997d22 --- /dev/null +++ b/package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py @@ -0,0 +1,39 @@ +import argparse +import json +import re +from typing import Dict, List, Tuple + + +def generate_unused_annotations(in_file_path: str): + with open(in_file_path, 'r', encoding="UTF-8") as in_file: + data = json.load(in_file) + + unuseds: Dict[Str, Dict[Str, Str]] = {} + for name in data: + formatted_name = format_name(name) + unuseds[formatted_name] = {"target": formatted_name} + + return unuseds + + +def format_name(name: str): + if name is None: + return None + + parts = re.split("\\.", name) + newname = "sklearn/" + parts[0] + + if len(parts) == 1: + return newname + + slash = False + for part in parts[1:-1]: + if not slash and re.match("^_{0,2}[A-Z]", part): + slash = True + if slash: + newname += "/" + part + else: + newname += "." + part + + newname += "/" + parts[-1] + return newname diff --git a/package-parser/tests/commands/generate_annotations/__init__.py b/package-parser/tests/commands/generate_annotations/__init__.py new file mode 100644 index 000000000..e69de29bb diff --git a/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py b/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py new file mode 100644 index 000000000..b31fc9ea4 --- /dev/null +++ b/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py @@ -0,0 +1,58 @@ +from package_parser.commands.generate_annotations._generate_unused_annotations import ( + generate_unused_annotations, + format_name +) + +EXPECTED_VALUE = { + 'sklearn/sklearn.base/_BaseEstimator/__setstate__': { + 'target': 'sklearn/sklearn.base/_BaseEstimator/__setstate__' + }, + 'sklearn/sklearn.base/is_regressor': { + 'target': 'sklearn/sklearn.base/is_regressor' + }, + 'sklearn/sklearn.cluster._agglomerative/linkage_tree': { + 'target': 'sklearn/sklearn.cluster._agglomerative/linkage_tree' + }, + 'sklearn/sklearn.cluster._kmeans/MiniBatchKMeans/init_size_': { + 'target': 'sklearn/sklearn.cluster._kmeans/MiniBatchKMeans/init_size_' + } +} + + +def test_format_underscores(): + assert format_name("sklearn.cluster._kmeans._MiniBatchKMeans.random_state_") \ + == "sklearn/sklearn.cluster._kmeans/_MiniBatchKMeans/random_state_" + + +def test_format_uppercase(): + assert format_name("sklearn.cluster._kmeans.MiniBatchKMeans.random_state_") \ + == "sklearn/sklearn.cluster._kmeans/MiniBatchKMeans/random_state_" + + +def test_format_normal(): + assert format_name("sklearn.cluster._mean_shift.get_bin_seeds") \ + == "sklearn/sklearn.cluster._mean_shift/get_bin_seeds" + + +def test_format_one_part(): + assert format_name("test") == "sklearn/test" + + +def test_format_none(): + assert format_name(None) is None + + +def test_format_empty(): + assert format_name("") == "sklearn/" + + +def test_generate(): + assert generate_unused_annotations("unused_functions_list.json") == EXPECTED_VALUE + + +def test_generate_bad_path(): + try: + generate_unused_annotations("aaaaaaaaaaaAAAAAAAAAAAA") + except FileNotFoundError: + return + assert False diff --git a/package-parser/tests/commands/generate_annotations/unused_functions_list.json b/package-parser/tests/commands/generate_annotations/unused_functions_list.json new file mode 100644 index 000000000..e4e850adc --- /dev/null +++ b/package-parser/tests/commands/generate_annotations/unused_functions_list.json @@ -0,0 +1,6 @@ +[ + "sklearn.base._BaseEstimator.__setstate__", + "sklearn.base.is_regressor", + "sklearn.cluster._agglomerative.linkage_tree", + "sklearn.cluster._kmeans.MiniBatchKMeans.init_size_" +] From 54bca7753e96930f84d47e3e4799ecc837cbe3fd Mon Sep 17 00:00:00 2001 From: IAmAlife Date: Fri, 8 Apr 2022 17:41:32 +0200 Subject: [PATCH 02/10] Changed wrong Type Str to str --- .../generate_annotations/_generate_unused_annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py b/package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py index e5a997d22..0a18383b2 100644 --- a/package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py +++ b/package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py @@ -8,7 +8,7 @@ def generate_unused_annotations(in_file_path: str): with open(in_file_path, 'r', encoding="UTF-8") as in_file: data = json.load(in_file) - unuseds: Dict[Str, Dict[Str, Str]] = {} + unuseds: Dict[str, Dict[str, str]] = {} for name in data: formatted_name = format_name(name) unuseds[formatted_name] = {"target": formatted_name} From fd131e2b6d86f12a990e50802cafb2efd9019698 Mon Sep 17 00:00:00 2001 From: jofaul Date: Fri, 8 Apr 2022 15:44:39 +0000 Subject: [PATCH 03/10] style: apply automatic fixes of linters --- .../_generate_unused_annotations.py | 2 +- .../test_generate_unused_annotations.py | 38 +++++++++++-------- 2 files changed, 23 insertions(+), 17 deletions(-) diff --git a/package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py b/package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py index 0a18383b2..ff38aca84 100644 --- a/package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py +++ b/package-parser/package_parser/commands/generate_annotations/_generate_unused_annotations.py @@ -5,7 +5,7 @@ def generate_unused_annotations(in_file_path: str): - with open(in_file_path, 'r', encoding="UTF-8") as in_file: + with open(in_file_path, "r", encoding="UTF-8") as in_file: data = json.load(in_file) unuseds: Dict[str, Dict[str, str]] = {} diff --git a/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py b/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py index b31fc9ea4..4519500e7 100644 --- a/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py +++ b/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py @@ -1,37 +1,43 @@ from package_parser.commands.generate_annotations._generate_unused_annotations import ( + format_name, generate_unused_annotations, - format_name ) EXPECTED_VALUE = { - 'sklearn/sklearn.base/_BaseEstimator/__setstate__': { - 'target': 'sklearn/sklearn.base/_BaseEstimator/__setstate__' + "sklearn/sklearn.base/_BaseEstimator/__setstate__": { + "target": "sklearn/sklearn.base/_BaseEstimator/__setstate__" }, - 'sklearn/sklearn.base/is_regressor': { - 'target': 'sklearn/sklearn.base/is_regressor' + "sklearn/sklearn.base/is_regressor": { + "target": "sklearn/sklearn.base/is_regressor" }, - 'sklearn/sklearn.cluster._agglomerative/linkage_tree': { - 'target': 'sklearn/sklearn.cluster._agglomerative/linkage_tree' + "sklearn/sklearn.cluster._agglomerative/linkage_tree": { + "target": "sklearn/sklearn.cluster._agglomerative/linkage_tree" + }, + "sklearn/sklearn.cluster._kmeans/MiniBatchKMeans/init_size_": { + "target": "sklearn/sklearn.cluster._kmeans/MiniBatchKMeans/init_size_" }, - 'sklearn/sklearn.cluster._kmeans/MiniBatchKMeans/init_size_': { - 'target': 'sklearn/sklearn.cluster._kmeans/MiniBatchKMeans/init_size_' - } } def test_format_underscores(): - assert format_name("sklearn.cluster._kmeans._MiniBatchKMeans.random_state_") \ - == "sklearn/sklearn.cluster._kmeans/_MiniBatchKMeans/random_state_" + assert ( + format_name("sklearn.cluster._kmeans._MiniBatchKMeans.random_state_") + == "sklearn/sklearn.cluster._kmeans/_MiniBatchKMeans/random_state_" + ) def test_format_uppercase(): - assert format_name("sklearn.cluster._kmeans.MiniBatchKMeans.random_state_") \ - == "sklearn/sklearn.cluster._kmeans/MiniBatchKMeans/random_state_" + assert ( + format_name("sklearn.cluster._kmeans.MiniBatchKMeans.random_state_") + == "sklearn/sklearn.cluster._kmeans/MiniBatchKMeans/random_state_" + ) def test_format_normal(): - assert format_name("sklearn.cluster._mean_shift.get_bin_seeds") \ - == "sklearn/sklearn.cluster._mean_shift/get_bin_seeds" + assert ( + format_name("sklearn.cluster._mean_shift.get_bin_seeds") + == "sklearn/sklearn.cluster._mean_shift/get_bin_seeds" + ) def test_format_one_part(): From 9c234cdc1f1c6c237204533405aea324272a0f76 Mon Sep 17 00:00:00 2001 From: Aclrian <32142988+Aclrian@users.noreply.github.com> Date: Fri, 8 Apr 2022 18:28:14 +0200 Subject: [PATCH 04/10] fix: pytest failed on expected exception --- .../test_generate_unused_annotations.py | 6 ++---- 1 file changed, 2 insertions(+), 4 deletions(-) diff --git a/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py b/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py index 4519500e7..8e3f258d7 100644 --- a/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py +++ b/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py @@ -1,3 +1,4 @@ +import pytest from package_parser.commands.generate_annotations._generate_unused_annotations import ( format_name, generate_unused_annotations, @@ -57,8 +58,5 @@ def test_generate(): def test_generate_bad_path(): - try: + with pytest.raises(FileNotFoundError): generate_unused_annotations("aaaaaaaaaaaAAAAAAAAAAAA") - except FileNotFoundError: - return - assert False From 8a57ab0b11af376645599a0df0530e884d4f3f19 Mon Sep 17 00:00:00 2001 From: Aclrian <32142988+Aclrian@users.noreply.github.com> Date: Fri, 8 Apr 2022 18:47:12 +0200 Subject: [PATCH 05/10] fix: file location is required --- .../generate_annotations/test_generate_unused_annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py b/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py index 8e3f258d7..96ec6741d 100644 --- a/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py +++ b/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py @@ -54,7 +54,7 @@ def test_format_empty(): def test_generate(): - assert generate_unused_annotations("unused_functions_list.json") == EXPECTED_VALUE + assert generate_unused_annotations("commands/generate_annotations/unused_functions_list.json") == EXPECTED_VALUE def test_generate_bad_path(): From 354a737e3382a3611b091ddcc6272c05bf657bc8 Mon Sep 17 00:00:00 2001 From: Aclrian Date: Fri, 8 Apr 2022 16:50:45 +0000 Subject: [PATCH 06/10] style: apply automatic fixes of linters --- .../test_generate_unused_annotations.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py b/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py index 96ec6741d..a64696481 100644 --- a/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py +++ b/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py @@ -54,7 +54,12 @@ def test_format_empty(): def test_generate(): - assert generate_unused_annotations("commands/generate_annotations/unused_functions_list.json") == EXPECTED_VALUE + assert ( + generate_unused_annotations( + "commands/generate_annotations/unused_functions_list.json" + ) + == EXPECTED_VALUE + ) def test_generate_bad_path(): From 81f5dec2b6d02d9f337e1ef46d673368c978a0a0 Mon Sep 17 00:00:00 2001 From: Aclrian <32142988+Aclrian@users.noreply.github.com> Date: Fri, 8 Apr 2022 19:11:02 +0200 Subject: [PATCH 07/10] fix: assumed wrong working dir --- .../generate_annotations/test_generate_unused_annotations.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py b/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py index a64696481..1c6a74ae6 100644 --- a/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py +++ b/package-parser/tests/commands/generate_annotations/test_generate_unused_annotations.py @@ -56,7 +56,7 @@ def test_format_empty(): def test_generate(): assert ( generate_unused_annotations( - "commands/generate_annotations/unused_functions_list.json" + "tests/commands/generate_annotations/unused_functions_list.json" ) == EXPECTED_VALUE ) From 961e74418abfff65ff9dddf1724e547731fd2f7a Mon Sep 17 00:00:00 2001 From: IAmAlife Date: Mon, 11 Apr 2022 15:26:58 +0200 Subject: [PATCH 08/10] removed changes in .idea and added it to gitignore of package-parser --- .idea/api-editor.iml | 10 ++-------- .idea/inspectionProfiles/Project_Default.xml | 5 +++++ .idea/misc.xml | 4 ++-- package-parser/.gitignore | 3 +++ 4 files changed, 12 insertions(+), 10 deletions(-) diff --git a/.idea/api-editor.iml b/.idea/api-editor.iml index 9b9c1652f..bdf896d56 100644 --- a/.idea/api-editor.iml +++ b/.idea/api-editor.iml @@ -3,13 +3,7 @@ - + - - - - - \ No newline at end of file + diff --git a/.idea/inspectionProfiles/Project_Default.xml b/.idea/inspectionProfiles/Project_Default.xml index cce1d8640..5dd3a168a 100644 --- a/.idea/inspectionProfiles/Project_Default.xml +++ b/.idea/inspectionProfiles/Project_Default.xml @@ -3,5 +3,10 @@