Add downstream tests and guard against leakages across tests (#49)

* Add downstream tests

* fix requirements install

* changes to working directory and software report

* narrow testing scope

* use dev version

* Temporarily use upstream branch

* Bug: guard against leaking verify_image_cache (#51)

* teardown fixture

* Fix teardown

* Teardown on errors

* update verbosity of tests temporarily

* Update ci_cd.yml

Partially remove verbosity

* use addfinalizer

* format

* use tmp branch

* Back to main branch

* Guard against leakages

* revert verbosity

* whitespace

* Update ci_cd.yml

use other branch

* Try latest python

* upload generated_images as artifact

* run upload on failure

* use same ubuntu version

* use main branch again

* Use main branch again

* add test

.github/workflows/ci_cd.yml

@@ -142,3 +142,40 @@ jobs:
           files: |
             ./**/*.whl
             ./**/*.tar.gz
+
+  downstream:
+    name: Downstream tests
+    runs-on: ubuntu-20.04  # matching pyvista
+    steps:
+      - uses: actions/checkout@v3
+      - uses: actions/setup-python@v4
+        with:
+          python-version: '3.11'
+
+      - run: git clone --depth=1 https://github.com/pyvista/pyvista.git --branch main --single-branch
+
+      - name: Install pyvista
+        run: pip install -ve .
+        working-directory: pyvista
+      - name: Install pyvista test requirements
+        run: pip install -v -r requirements_test.txt
+        working-directory: pyvista
+      - name: Install pytest-pyvista
+        run: pip install -ve .
+      - name: Software Report
+        run: |
+          xvfb-run python -c "import pyvista; print(pyvista.Report()); from pyvista import examples; print('User data path:', examples.USER_DATA_PATH)"
+          which python
+          pip list
+        working-directory: pyvista
+
+      - name: Unit Testing
+        run: xvfb-run python -m pytest --fail_extra_image_cache -v --generated_image_dir gen_dir tests/plotting/test_plotting.py
+        working-directory: pyvista
+
+      - name: Upload generated image artifact
+        if: ${{ failure() }}
+        uses: actions/upload-artifact@v3
+        with:
+          name: generated_images
+          path: pyvista/gen_dir

pytest_pyvista/pytest_pyvista.py

@@ -189,6 +189,8 @@ def __call__(self, plotter):
             and self.fail_extra_image_cache
             and not self.reset_image_cache
         ):
+            # Make sure this doesn't get called again if this plotter doesn't close properly
+            plotter._before_close_callback = None
             raise RuntimeError(f"{image_filename} does not exist in image cache")
 
         if (
@@ -206,6 +208,8 @@ def __call__(self, plotter):
         error = pyvista.compare_images(image_filename, plotter)
 
         if error > allowed_error:
+            # Make sure this doesn't get called again if this plotter doesn't close properly
+            plotter._before_close_callback = None
             raise RuntimeError(
                 f"{test_name} Exceeded image regression error of "
                 f"{allowed_error} with an image error equal to: {error}"
@@ -241,4 +245,8 @@ def verify_image_cache(request, pytestconfig):
     )
     pyvista.global_theme.before_close_callback = verify_image_cache
 
+    def reset():
+        pyvista.global_theme.before_close_callback = None
+
+    request.addfinalizer(reset)
     return verify_image_cache

tests/test_pyvista.py

@@ -251,3 +251,33 @@ def test_imcache(verify_image_cache):
     assert not filecmp.cmp(filename, filename_original, shallow=False)
     # should pass even if image doesn't match
     result.stdout.fnmatch_lines("*[Pp]assed*")
+
+
+def test_cleanup(testdir):
+    """Test cleanup of the `verify_image_cache` fixture."""
+    make_cached_images(testdir.tmpdir)
+    testdir.makepyfile(
+        """
+       import pytest
+       import pyvista as pv
+       pv.OFF_SCREEN = True
+
+       @pytest.fixture()
+       def cleanup_tester():
+           yield
+           assert pv.global_theme.before_close_callback is None
+
+       def test_imcache(cleanup_tester, verify_image_cache):
+           sphere = pv.Sphere()
+           plotter = pv.Plotter()
+           plotter.add_mesh(sphere, color="blue")
+           try:
+               plotter.show()
+           except RuntimeError:
+               # continue so the cleanup is tested
+               pass
+       """
+    )
+
+    result = testdir.runpytest("--fail_extra_image_cache")
+    result.stdout.fnmatch_lines("*[Pp]assed*")