From 12d725a903c632de4e9bd7a03fd900b72c2b450c Mon Sep 17 00:00:00 2001 From: Matthew Flamm Date: Mon, 26 Dec 2022 07:56:13 -0500 Subject: [PATCH 1/5] allow saving images to dir --- pytest_pyvista/pytest_pyvista.py | 28 +++++++++++++++++++--------- tests/test_pyvista.py | 24 +++++++++++++++++++++++- 2 files changed, 42 insertions(+), 10 deletions(-) diff --git a/pytest_pyvista/pytest_pyvista.py b/pytest_pyvista/pytest_pyvista.py index 698c48b..a36ef79 100644 --- a/pytest_pyvista/pytest_pyvista.py +++ b/pytest_pyvista/pytest_pyvista.py @@ -1,6 +1,5 @@ # -*- coding: utf-8 -*- - import os import pathlib import platform @@ -28,6 +27,11 @@ def pytest_addoption(parser): action="store_true", help="Enables failure if image cache does not exist.", ) + group.addoption( + "--generated_image_dir", + action="store", + help="Path grnerated io thse cache folder.", + ) group.addoption( "--image_cache_dir", action="store", @@ -84,6 +88,7 @@ def __init__( warning_value=200, var_error_value=1000, var_warning_value=1000, + generated_image_dir=None, ): self.test_name = test_name @@ -98,6 +103,12 @@ def __init__( self.var_error_value = var_error_value self.var_warning_value = var_warning_value + self.generated_image_dir = generated_image_dir + + if self.generated_image_dir is not None and not os.path.isdir(self.generated_image_dir): + warnings.warn(f"pyvista test generated image dir: {self.generated_image_dir} does not yet exist. Creating dir.") + os.makedirs(self.generated_image_dir) + self.skip = False self.n_calls = 0 @@ -142,16 +153,13 @@ def __call__(self, plotter): # cached image name. We remove the first 5 characters of the function name # "test_" to get the name for the image. image_filename = os.path.join(self.cache_dir, test_name[5:] + ".png") - if not os.path.isfile(image_filename) and self.fail_extra_image_cache: raise RuntimeError(f"{image_filename} does not exist in image cache") - # simply save the last screenshot if it doesn't exist or the cache - # is being reset. - if self.reset_image_cache or not os.path.isfile(image_filename): - return plotter.screenshot(image_filename) - - # otherwise, compare with the existing cached image + if self.generated_image_dir is not None: + gen_image_filename =os.path.join(self.generated_image_dir, test_name[5:] + ".png") + plotter.screenshot(gen_image_filename) error = pyvista.compare_images(image_filename, plotter) + if error > allowed_error: raise RuntimeError( f"{test_name} Exceeded image regression error of " @@ -180,6 +188,8 @@ def verify_image_cache(request, pytestconfig): if cache_dir is None: cache_dir = pytestconfig.getini("image_cache_dir") - verify_image_cache = VerifyImageCache(request.node.name, cache_dir) + gen_dir = pytestconfig.getoption("generated_image_dir") + + verify_image_cache = VerifyImageCache(request.node.name, cache_dir, generated_image_dir=gen_dir) pyvista.global_theme.before_close_callback = verify_image_cache return verify_image_cache diff --git a/tests/test_pyvista.py b/tests/test_pyvista.py index c0071e4..75280bb 100644 --- a/tests/test_pyvista.py +++ b/tests/test_pyvista.py @@ -6,6 +6,7 @@ pv.OFF_SCREEN = True + def test_arguments(testdir): """Test pytest arguments""" testdir.makepyfile( @@ -24,7 +25,7 @@ def test_args(verify_image_cache): def make_cached_images(test_path, path="image_cache_dir"): - """Makes image cache in `test_path\path`.""" + """Makes image cache in `test_path/path`.""" d = os.path.join(test_path, path) os.mkdir(d) sphere = pv.Sphere() @@ -135,3 +136,24 @@ def test_imcache(verify_image_cache): ) result = testdir.runpytest("--fail_extra_image_cache") result.stdout.fnmatch_lines("*[Pp]assed*") + + +def test_generated_image_dir_commandline(testdir): + """Test setting image_cache_dir via CLI option.""" + make_cached_images(testdir.tmpdir) + testdir.makepyfile( + """ + import pyvista as pv + pv.OFF_SCREEN = True + def test_imcache(verify_image_cache): + sphere = pv.Sphere() + plotter = pv.Plotter() + plotter.add_mesh(sphere, color="red") + plotter.show() + """ + ) + + result = testdir.runpytest("--fail_extra_image_cache", "--generated_image_dir", "gen_dir") + assert os.path.isdir(os.path.join(testdir.tmpdir, "gen_dir")) + assert os.path.isfile(os.path.join(testdir.tmpdir, "gen_dir", "imcache.png")) + result.stdout.fnmatch_lines("*[Pp]assed*") From f7b27c416bc8d0e092cc32986fb6eb107f2f037a Mon Sep 17 00:00:00 2001 From: Matthew Flamm Date: Mon, 26 Dec 2022 08:55:30 -0500 Subject: [PATCH 2/5] add option for adding missing images to cache --- pytest_pyvista/pytest_pyvista.py | 15 ++++++++++++++- tests/test_pyvista.py | 21 ++++++++++++++++++++- 2 files changed, 34 insertions(+), 2 deletions(-) diff --git a/pytest_pyvista/pytest_pyvista.py b/pytest_pyvista/pytest_pyvista.py index a36ef79..cfbe3b0 100644 --- a/pytest_pyvista/pytest_pyvista.py +++ b/pytest_pyvista/pytest_pyvista.py @@ -30,7 +30,12 @@ def pytest_addoption(parser): group.addoption( "--generated_image_dir", action="store", - help="Path grnerated io thse cache folder.", + help="Path to generated images folder.", + ) + group.addoption( + "--add_missing_images", + action="store_true", + help="Adds images to cache if missing.", ) group.addoption( "--image_cache_dir", @@ -74,6 +79,7 @@ class VerifyImageCache: reset_image_cache = False ignore_image_cache = False fail_extra_image_cache = False + add_missing_images = False high_variance_tests = False windows_skip_image_cache = False @@ -155,6 +161,10 @@ def __call__(self, plotter): image_filename = os.path.join(self.cache_dir, test_name[5:] + ".png") if not os.path.isfile(image_filename) and self.fail_extra_image_cache: raise RuntimeError(f"{image_filename} does not exist in image cache") + + if self.add_missing_images and not os.path.isfile(image_filename): + plotter.screenshot(image_filename) + if self.generated_image_dir is not None: gen_image_filename =os.path.join(self.generated_image_dir, test_name[5:] + ".png") plotter.screenshot(gen_image_filename) @@ -183,6 +193,9 @@ def verify_image_cache(request, pytestconfig): VerifyImageCache.fail_extra_image_cache = pytestconfig.getoption( "fail_extra_image_cache" ) + VerifyImageCache.add_missing_images = pytestconfig.getoption( + "add_missing_images" + ) cache_dir = pytestconfig.getoption("image_cache_dir") if cache_dir is None: diff --git a/tests/test_pyvista.py b/tests/test_pyvista.py index 75280bb..a45435f 100644 --- a/tests/test_pyvista.py +++ b/tests/test_pyvista.py @@ -139,7 +139,7 @@ def test_imcache(verify_image_cache): def test_generated_image_dir_commandline(testdir): - """Test setting image_cache_dir via CLI option.""" + """Test setting generated_image_dir via CLI option.""" make_cached_images(testdir.tmpdir) testdir.makepyfile( """ @@ -157,3 +157,22 @@ def test_imcache(verify_image_cache): assert os.path.isdir(os.path.join(testdir.tmpdir, "gen_dir")) assert os.path.isfile(os.path.join(testdir.tmpdir, "gen_dir", "imcache.png")) result.stdout.fnmatch_lines("*[Pp]assed*") + + +def test_gladd_missing_images_commandline(testdir): + """Test setting add_missing_images via CLI option.""" + testdir.makepyfile( + """ + import pyvista as pv + pv.OFF_SCREEN = True + def test_imcache(verify_image_cache): + sphere = pv.Sphere() + plotter = pv.Plotter() + plotter.add_mesh(sphere, color="red") + plotter.show() + """ + ) + + result = testdir.runpytest("--add_missing_images") + assert os.path.isfile(os.path.join(testdir.tmpdir, "image_cache_dir", "imcache.png")) + result.stdout.fnmatch_lines("*[Pp]assed*") From 2c7c050046f0f40bad89b9de633e9ef821dc8f18 Mon Sep 17 00:00:00 2001 From: MatthewFlamm <39341281+MatthewFlamm@users.noreply.github.com> Date: Fri, 6 Jan 2023 11:37:39 -0500 Subject: [PATCH 3/5] Use better help message. Co-authored-by: Bane Sullivan --- pytest_pyvista/pytest_pyvista.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pytest_pyvista/pytest_pyvista.py b/pytest_pyvista/pytest_pyvista.py index cfbe3b0..d39a3fe 100644 --- a/pytest_pyvista/pytest_pyvista.py +++ b/pytest_pyvista/pytest_pyvista.py @@ -30,7 +30,7 @@ def pytest_addoption(parser): group.addoption( "--generated_image_dir", action="store", - help="Path to generated images folder.", + help="Path to dump test images from the current run.", ) group.addoption( "--add_missing_images", From 3a88ef24ba2cb4d9f22652cb53c5ecfbb0b85017 Mon Sep 17 00:00:00 2001 From: Matthew Flamm Date: Fri, 6 Jan 2023 13:39:21 -0500 Subject: [PATCH 4/5] fix merge --- pytest_pyvista/pytest_pyvista.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pytest_pyvista/pytest_pyvista.py b/pytest_pyvista/pytest_pyvista.py index b09959b..1ebd995 100644 --- a/pytest_pyvista/pytest_pyvista.py +++ b/pytest_pyvista/pytest_pyvista.py @@ -85,10 +85,6 @@ class VerifyImageCache: ignore_image_cache = False fail_extra_image_cache = False add_missing_images = False - - high_variance_tests = False - windows_skip_image_cache = False - macos_skip_image_cache = False skip_image_cache_vtk8 = False def __init__( From d9a6ea742185aef3a67b362bd1df0a7ce835b610 Mon Sep 17 00:00:00 2001 From: MatthewFlamm <39341281+MatthewFlamm@users.noreply.github.com> Date: Fri, 6 Jan 2023 13:43:23 -0500 Subject: [PATCH 5/5] typo --- tests/test_pyvista.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/tests/test_pyvista.py b/tests/test_pyvista.py index 9d14dc2..660e857 100644 --- a/tests/test_pyvista.py +++ b/tests/test_pyvista.py @@ -214,7 +214,7 @@ def test_imcache(verify_image_cache): @skip_vtk8 -def test_gladd_missing_images_commandline(testdir): +def test_add_missing_images_commandline(testdir): """Test setting add_missing_images via CLI option.""" testdir.makepyfile( """