From a66b3d314f87d31002f916dc92b433a1cef08147 Mon Sep 17 00:00:00 2001 From: Ben Rowland Date: Mon, 15 May 2023 12:12:51 +0100 Subject: [PATCH 1/5] move clear_comtypes_cache to be a callable module This commit modifies the clear_comtypes_cache.py script so that it is inside the main comtypes module (renamed as just clear_cache) so that is can be called more easily as "py -m comtypes.clear_cache". The main function of the script is also exported using the "console_scripts" entry point so that the script also goes into the standard Python "Scripts" folder as before, but now as a .exe instead of a .py script, which makes it easier to run if systems are set to open .py files instead of running them. This version also includes a test case using the 3rd party package pyfakefs. Currently, this is not desired to avoid the requirement of 3rd party packages in comtypes, but is included here for potential use if the position changes. A subsequent commit will modify the tests to use unittest.patch instead, which is an inferior technical solution but avoids a 3rd party package. --- clear_comtypes_cache.py | 57 ------------------------------- comtypes/clear_cache.py | 43 +++++++++++++++++++++++ comtypes/test/test_clear_cache.py | 48 ++++++++++++++++++++++++++ setup.py | 12 +++---- 4 files changed, 96 insertions(+), 64 deletions(-) delete mode 100644 clear_comtypes_cache.py create mode 100644 comtypes/clear_cache.py create mode 100644 comtypes/test/test_clear_cache.py diff --git a/clear_comtypes_cache.py b/clear_comtypes_cache.py deleted file mode 100644 index bd743cd5..00000000 --- a/clear_comtypes_cache.py +++ /dev/null @@ -1,57 +0,0 @@ -import os -import sys -import shutil - -def get_next_cache_dir(): - work_dir = os.getcwd() - try: - # change working directory to avoid import from local folder - # during installation process - os.chdir(os.path.dirname(sys.executable)) - import comtypes.client - return comtypes.client._code_cache._find_gen_dir() - except ImportError: - return None - finally: - os.chdir(work_dir) - - -def _remove(directory): - shutil.rmtree(directory) - print('Removed directory "%s"' % directory) - - -def remove_directory(directory, silent): - if directory: - if silent: - _remove(directory) - else: - try: - confirm = raw_input('Remove comtypes cache directories? (y/n): ') - except NameError: - confirm = input('Remove comtypes cache directories? (y/n): ') - if confirm.lower() == 'y': - _remove(directory) - else: - print('Directory "%s" NOT removed' % directory) - return False - return True - - -if len(sys.argv) > 1 and "-y" in sys.argv[1:]: - silent = True -else: - silent = False - - -# First iteration may get folder with restricted rights. -# Second iteration always gets temp cache folder (writable for all). -directory = get_next_cache_dir() -removed = remove_directory(directory, silent) - -if removed: - directory = get_next_cache_dir() - - # do not request the second confirmation - # if the first folder was already removed - remove_directory(directory, silent=removed) diff --git a/comtypes/clear_cache.py b/comtypes/clear_cache.py new file mode 100644 index 00000000..d3595674 --- /dev/null +++ b/comtypes/clear_cache.py @@ -0,0 +1,43 @@ +import argparse +import os +from shutil import rmtree as _rmtree # TESTS RELY ON THIS IMPORT NOT CHANGING +import sys + + +def main(): + parser = argparse.ArgumentParser( + prog="py -m comtypes.clear_cache", description="Removes comtypes cache folders." + ) + parser.add_argument( + "-y", help="Pre-approve deleting all folders", action="store_true" + ) + args = parser.parse_args() + + if not args.y: + confirm = input("Remove comtypes cache directories? (y/n): ") + if confirm.lower() != "y": + print("Cache directories NOT removed") + return + + # change cwd to avoid import from local folder during installation process + work_dir = os.getcwd() + try: + os.chdir(os.path.dirname(sys.executable)) + import comtypes.client + except ImportError: + print("Could not import comtypes", file=sys.stderr) + sys.exit(1) + os.chdir(work_dir) + + # there are two possible locations for the cache folder (in the comtypes + # folder in site-packages if that is writable, otherwise in APPDATA) + # fortunately, by deleting the first location returned by _find_gen_dir() + # we make it un-writable, so calling it again gives us the APPDATA location + for _ in range(2): + dir_path = comtypes.client._find_gen_dir() + _rmtree(dir_path) + print(f"Removed directory \"{dir_path}\"") + + +if __name__ == "__main__": + main() diff --git a/comtypes/test/test_clear_cache.py b/comtypes/test/test_clear_cache.py new file mode 100644 index 00000000..0857e69b --- /dev/null +++ b/comtypes/test/test_clear_cache.py @@ -0,0 +1,48 @@ +""" +Tests for the ``comtypes.clear_cache`` module. + +This module provides tests for the ``clear_cache`` script. Because we don't +want a test invocation of the script to actually delete the real cache of the +calling Python (which would probably break many of the following tests), we use +pyfakefs which creates an in memory mock file system which mirrors the relevant +folders but does not propagate changes to the real version. + +Because there are various locations that the cache folder can be in, we include +all the folders from sys.path in the mock fs, as the comtypes.gen module has to +be in one of them. +""" +import io +import os +import runpy +import sys +from unittest.mock import patch + +from pyfakefs.fake_filesystem_unittest import TestCase + +# importing this will create a real gen cache dir which is necessary as +# comtypes relies on importing the module (and can't import from the fake fs) +import comtypes.client + + +class ClearCacheTestCase(TestCase): + def setUp(self) -> None: + self.setUpPyfakefs() + + # we patch sys.stdout so unittest doesn't show the print statements + @patch("sys.stdout", new=io.StringIO()) + def test_clear_cache(self): + + for site_path in sys.path: + try: + self.fs.add_real_directory(site_path, read_only=False) + except (FileExistsError, FileNotFoundError): + pass + + # ask comtypes where the cache dir is so we can check it is gone + cache_dir = comtypes.client._find_gen_dir() + assert os.path.exists(cache_dir) + + with patch("sys.argv", ["clear_cache.py", "-y"]): + runpy.run_module("comtypes.clear_cache", {}, "__main__") + + assert not os.path.exists(cache_dir) diff --git a/setup.py b/setup.py index 13045e88..c44cae15 100644 --- a/setup.py +++ b/setup.py @@ -111,14 +111,10 @@ def run(self): install.run(self) # Custom script we run at the end of installing if not self.dry_run and not self.root: - filename = os.path.join( - self.install_scripts, "clear_comtypes_cache.py") - if not os.path.isfile(filename): - raise RuntimeError("Can't find '%s'" % (filename,)) print("Executing post install script...") - print('"' + sys.executable + '" "' + filename + '" -y') + print('"' + sys.executable + '" -m comtypes.clear_cache -y') try: - subprocess.check_call([sys.executable, filename, '-y']) + subprocess.check_call([sys.executable, "-m", "comtypes.clear_cache", '-y']) except subprocess.CalledProcessError: print("Failed to run post install script!") @@ -146,7 +142,9 @@ def run(self): ]}, classifiers=classifiers, - scripts=["clear_comtypes_cache.py"], + entry_points={ + "console_scripts": ["clear_comtypes_cache=comtypes.clear_cache:main"] + }, cmdclass={ 'test': test, From e475bc11f85494eb7baae192f23bca38ff96feab Mon Sep 17 00:00:00 2001 From: Ben Rowland Date: Mon, 15 May 2023 13:16:04 +0100 Subject: [PATCH 2/5] modify clear_cache tests to not use pyfakefs This commit updates the test for comtypes.clear_cache to not use any 3rd party packages, instead relying on mocking the shutil.rmtree function which is used to do the actual cache deletion. --- comtypes/clear_cache.py | 4 +-- comtypes/test/test_clear_cache.py | 44 +++++++------------------------ 2 files changed, 11 insertions(+), 37 deletions(-) diff --git a/comtypes/clear_cache.py b/comtypes/clear_cache.py index d3595674..63ae9379 100644 --- a/comtypes/clear_cache.py +++ b/comtypes/clear_cache.py @@ -1,7 +1,7 @@ import argparse import os -from shutil import rmtree as _rmtree # TESTS RELY ON THIS IMPORT NOT CHANGING import sys +from shutil import rmtree # TESTS ASSUME USE OF RMTREE def main(): @@ -35,7 +35,7 @@ def main(): # we make it un-writable, so calling it again gives us the APPDATA location for _ in range(2): dir_path = comtypes.client._find_gen_dir() - _rmtree(dir_path) + rmtree(dir_path) print(f"Removed directory \"{dir_path}\"") diff --git a/comtypes/test/test_clear_cache.py b/comtypes/test/test_clear_cache.py index 0857e69b..514b6dc4 100644 --- a/comtypes/test/test_clear_cache.py +++ b/comtypes/test/test_clear_cache.py @@ -1,48 +1,22 @@ """ -Tests for the ``comtypes.clear_cache`` module. - -This module provides tests for the ``clear_cache`` script. Because we don't -want a test invocation of the script to actually delete the real cache of the -calling Python (which would probably break many of the following tests), we use -pyfakefs which creates an in memory mock file system which mirrors the relevant -folders but does not propagate changes to the real version. - -Because there are various locations that the cache folder can be in, we include -all the folders from sys.path in the mock fs, as the comtypes.gen module has to -be in one of them. +Test for the ``comtypes.clear_cache`` module. """ import io -import os import runpy -import sys -from unittest.mock import patch +from unittest.mock import patch, call +from unittest import TestCase -from pyfakefs.fake_filesystem_unittest import TestCase - -# importing this will create a real gen cache dir which is necessary as -# comtypes relies on importing the module (and can't import from the fake fs) -import comtypes.client +from comtypes.client import _find_gen_dir class ClearCacheTestCase(TestCase): - def setUp(self) -> None: - self.setUpPyfakefs() - # we patch sys.stdout so unittest doesn't show the print statements @patch("sys.stdout", new=io.StringIO()) - def test_clear_cache(self): - - for site_path in sys.path: - try: - self.fs.add_real_directory(site_path, read_only=False) - except (FileExistsError, FileNotFoundError): - pass - - # ask comtypes where the cache dir is so we can check it is gone - cache_dir = comtypes.client._find_gen_dir() - assert os.path.exists(cache_dir) - + @patch("shutil.rmtree") + def test_clear_cache(self, mock_rmtree): with patch("sys.argv", ["clear_cache.py", "-y"]): runpy.run_module("comtypes.clear_cache", {}, "__main__") - assert not os.path.exists(cache_dir) + # because we don't actually delete anything, _find_gen_dir() will + # give the same answer every time we call it + assert mock_rmtree.call_args_list == [call(_find_gen_dir()) for i in range(2)] From 836b78170f770881af30f8fa0801af17f779c578 Mon Sep 17 00:00:00 2001 From: Ben Rowland Date: Mon, 15 May 2023 13:33:12 +0100 Subject: [PATCH 3/5] change quotes in print string --- comtypes/clear_cache.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/comtypes/clear_cache.py b/comtypes/clear_cache.py index 63ae9379..82b8493d 100644 --- a/comtypes/clear_cache.py +++ b/comtypes/clear_cache.py @@ -36,7 +36,7 @@ def main(): for _ in range(2): dir_path = comtypes.client._find_gen_dir() rmtree(dir_path) - print(f"Removed directory \"{dir_path}\"") + print(f'Removed directory "{dir_path}"') if __name__ == "__main__": From ba61f4db319b825d9d08da9ef72daa521b0b1f1a Mon Sep 17 00:00:00 2001 From: Ben Rowland Date: Thu, 18 May 2023 11:53:32 +0100 Subject: [PATCH 4/5] style changes based on review by @junkmd --- CONTRIBUTING.md | 2 +- comtypes/test/test_clear_cache.py | 7 ++++--- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 555c8aca..472b88c1 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -91,7 +91,7 @@ Those `.py` files act like ”caches”. If there are some problems with the developing code base, partial or non-executable modules might be created in `.../comtypes/gen/...`. Importing them will cause some error. -If that happens, you should run `python -m clear_comtypes_cache` to clear those caches. +If that happens, you should run `python -m comtypes.clear_cache` to clear those caches. The command will delete the entire `.../comtypes/gen` directory. Importing `comtypes.gen.client` will restore the directory and `__init__.py` file. diff --git a/comtypes/test/test_clear_cache.py b/comtypes/test/test_clear_cache.py index 514b6dc4..c3b9be8f 100644 --- a/comtypes/test/test_clear_cache.py +++ b/comtypes/test/test_clear_cache.py @@ -1,7 +1,7 @@ """ Test for the ``comtypes.clear_cache`` module. """ -import io +import contextlib import runpy from unittest.mock import patch, call from unittest import TestCase @@ -11,10 +11,11 @@ class ClearCacheTestCase(TestCase): # we patch sys.stdout so unittest doesn't show the print statements - @patch("sys.stdout", new=io.StringIO()) + + @patch("sys.argv", ["clear_cache.py", "-y"]) @patch("shutil.rmtree") def test_clear_cache(self, mock_rmtree): - with patch("sys.argv", ["clear_cache.py", "-y"]): + with contextlib.redirect_stdout(None): runpy.run_module("comtypes.clear_cache", {}, "__main__") # because we don't actually delete anything, _find_gen_dir() will From 174f3b4791ef942ac8e332003ffbf4d7a3da233c Mon Sep 17 00:00:00 2001 From: Ben Rowland Date: Tue, 13 Jun 2023 09:55:33 +0100 Subject: [PATCH 5/5] Apply suggestions from code review Co-authored-by: Jun Komoda <45822440+junkmd@users.noreply.github.com> --- comtypes/clear_cache.py | 26 ++++++++++++++++++-------- comtypes/test/test_clear_cache.py | 4 +++- setup.py | 2 +- 3 files changed, 22 insertions(+), 10 deletions(-) diff --git a/comtypes/clear_cache.py b/comtypes/clear_cache.py index 82b8493d..f0e10ced 100644 --- a/comtypes/clear_cache.py +++ b/comtypes/clear_cache.py @@ -1,9 +1,21 @@ import argparse +import contextlib import os import sys from shutil import rmtree # TESTS ASSUME USE OF RMTREE +# if supporting Py>=3.11 only, this might be `contextlib.chdir`. +# https://docs.python.org/3/library/contextlib.html#contextlib.chdir +@contextlib.contextmanager +def chdir(path): + """Context manager to change the current working directory.""" + work_dir = os.getcwd() + os.chdir(path) + yield + os.chdir(work_dir) + + def main(): parser = argparse.ArgumentParser( prog="py -m comtypes.clear_cache", description="Removes comtypes cache folders." @@ -20,14 +32,12 @@ def main(): return # change cwd to avoid import from local folder during installation process - work_dir = os.getcwd() - try: - os.chdir(os.path.dirname(sys.executable)) - import comtypes.client - except ImportError: - print("Could not import comtypes", file=sys.stderr) - sys.exit(1) - os.chdir(work_dir) + with chdir(os.path.dirname(sys.executable)): + try: + import comtypes.client + except ImportError: + print("Could not import comtypes", file=sys.stderr) + sys.exit(1) # there are two possible locations for the cache folder (in the comtypes # folder in site-packages if that is writable, otherwise in APPDATA) diff --git a/comtypes/test/test_clear_cache.py b/comtypes/test/test_clear_cache.py index c3b9be8f..e3cf8147 100644 --- a/comtypes/test/test_clear_cache.py +++ b/comtypes/test/test_clear_cache.py @@ -20,4 +20,6 @@ def test_clear_cache(self, mock_rmtree): # because we don't actually delete anything, _find_gen_dir() will # give the same answer every time we call it - assert mock_rmtree.call_args_list == [call(_find_gen_dir()) for i in range(2)] + self.assertEqual( + mock_rmtree.call_args_list, [call(_find_gen_dir()) for _ in range(2)] + ) diff --git a/setup.py b/setup.py index c44cae15..54d574e4 100644 --- a/setup.py +++ b/setup.py @@ -112,7 +112,7 @@ def run(self): # Custom script we run at the end of installing if not self.dry_run and not self.root: print("Executing post install script...") - print('"' + sys.executable + '" -m comtypes.clear_cache -y') + print(f'"{sys.executable}" -m comtypes.clear_cache -y') try: subprocess.check_call([sys.executable, "-m", "comtypes.clear_cache", '-y']) except subprocess.CalledProcessError: