From 2ecb08665c636759e9a644e3999bd72531c9e957 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 23 Sep 2021 19:56:16 -0400 Subject: [PATCH 1/2] Switch to pytest-mpl for image testing --- .appveyor.yml | 5 +- .github/workflows/ci-testing.yml | 20 +- INSTALL | 6 +- environment.yml | 2 +- lib/cartopy/tests/mpl/__init__.py | 261 ------------------ lib/cartopy/tests/mpl/conftest.py | 17 ++ lib/cartopy/tests/mpl/test_axes.py | 9 +- lib/cartopy/tests/mpl/test_crs.py | 10 +- lib/cartopy/tests/mpl/test_examples.py | 14 +- lib/cartopy/tests/mpl/test_features.py | 14 +- lib/cartopy/tests/mpl/test_gridliner.py | 40 +-- lib/cartopy/tests/mpl/test_images.py | 28 +- lib/cartopy/tests/mpl/test_img_transform.py | 4 +- lib/cartopy/tests/mpl/test_mpl_integration.py | 176 ++++++++---- lib/cartopy/tests/mpl/test_nightshade.py | 4 +- lib/cartopy/tests/mpl/test_shapely_to_mpl.py | 10 +- lib/cartopy/tests/mpl/test_ticks.py | 16 +- lib/cartopy/tests/mpl/test_web_services.py | 7 +- requirements/tests.txt | 2 +- setup.cfg | 3 + 20 files changed, 252 insertions(+), 396 deletions(-) diff --git a/.appveyor.yml b/.appveyor.yml index cfb916a53..e2450eefe 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -15,7 +15,7 @@ install: - conda config --add channels conda-forge - conda config --add channels conda-forge/label/testing - set ENV_NAME=test-environment - - set PACKAGES=%PACKAGES% flufl.lock owslib pep8 pillow pyshp pytest + - set PACKAGES=%PACKAGES% owslib pep8 pillow pyshp pytest pytest-mpl - set PACKAGES=%PACKAGES% requests setuptools_scm setuptools_scm_git_archive - set PACKAGES=%PACKAGES% shapely - conda create -n %ENV_NAME% python=%PYTHON_VERSION% %PACKAGES% @@ -38,7 +38,8 @@ build_script: test_script: - set MPLBACKEND=Agg - set PYPROJ_GLOBAL_CONTEXT=ON - - pytest --pyargs cartopy + - pytest -ra --pyargs cartopy + --mpl --mpl-generate-summary=html --mpl-results-path=cartopy_test_output artifacts: - path: cartopy_test_output diff --git a/.github/workflows/ci-testing.yml b/.github/workflows/ci-testing.yml index 94d06ed63..53d7b04f4 100644 --- a/.github/workflows/ci-testing.yml +++ b/.github/workflows/ci-testing.yml @@ -48,7 +48,7 @@ jobs: - name: Install dependencies run: | - PACKAGES="$PACKAGES flufl.lock owslib pep8 pillow pyshp pytest" + PACKAGES="$PACKAGES owslib pep8 pillow pyshp pytest pytest-mpl" PACKAGES="$PACKAGES pytest-xdist requests setuptools_scm" PACKAGES="$PACKAGES setuptools_scm_git_archive shapely" conda install $PACKAGES @@ -70,7 +70,10 @@ jobs: # Check that the downloader tool at least knows where to get the data from (but don't actually download it) python tools/cartopy_feature_download.py gshhs physical --dry-run CARTOPY_GIT_DIR=$PWD - PYPROJ_GLOBAL_CONTEXT=ON pytest -ra -n 4 --doctest-modules --pyargs cartopy ${EXTRA_TEST_ARGS} + PYPROJ_GLOBAL_CONTEXT=ON pytest -ra -n 4 --doctest-modules \ + --mpl --mpl-generate-summary=html \ + --mpl-results-path="cartopy_test_output-${{ matrix.os }}-${{ matrix.python-version }}" \ + --pyargs cartopy ${EXTRA_TEST_ARGS} - name: Coveralls if: steps.coverage.conclusion == 'success' @@ -79,16 +82,9 @@ jobs: run: coveralls --service=github - - name: Create image output - if: failure() && steps.install.conclusion == 'success' - id: image-output - run: - python -c "import cartopy.tests.mpl; print(cartopy.tests.mpl.failed_images_html())" >> image-failures-${{ matrix.os }}-${{ matrix.python-version }}.html - - # Can't create image output and upload in the same step - name: Upload image results uses: actions/upload-artifact@v2 - if: failure() && steps.image-output.conclusion == 'success' + if: failure() with: - name: image-failures-${{ matrix.os }}-${{ matrix.python-version }}.html - path: image-failures-${{ matrix.os }}-${{ matrix.python-version }}.html + name: image-failures-${{ matrix.os }}-${{ matrix.python-version }} + path: cartopy_test_output-${{ matrix.os }}-${{ matrix.python-version }} diff --git a/INSTALL b/INSTALL index 834433f6b..50371dd7e 100644 --- a/INSTALL +++ b/INSTALL @@ -146,11 +146,11 @@ Testing Dependencies These packages are required for the full Cartopy test suite to run. -**flufl.lock** (https://flufllock.readthedocs.io/) - A platform independent file lock for Python. - **pytest** 5.1.2 or later (https://docs.pytest.org/en/latest/) Python package for software testing. +**pytest-mpl** 0.11 or later (https://github.com/matplotlib/pytest-mpl) + Pytest plugin to faciliate image comparison for Matplotlib figures + **pep8** 1.3.3 or later (https://pypi.python.org/pypi/pep8) Python package for software testing. diff --git a/environment.yml b/environment.yml index 35ffd6336..a8a39e47f 100644 --- a/environment.yml +++ b/environment.yml @@ -26,9 +26,9 @@ dependencies: - gdal>=2.3.2 - scipy>=1.3.1 # Testing - - flufl.lock - packaging>=20 - pytest + - pytest-mpl - pytest-xdist # Documentation - beautifulsoup4 diff --git a/lib/cartopy/tests/mpl/__init__.py b/lib/cartopy/tests/mpl/__init__.py index 1c989a921..5efa736c4 100644 --- a/lib/cartopy/tests/mpl/__init__.py +++ b/lib/cartopy/tests/mpl/__init__.py @@ -4,276 +4,15 @@ # See COPYING and COPYING.LESSER in the root of the repository for full # licensing details. -import base64 -import os -import glob -import shutil -import warnings - -import flufl.lock import matplotlib as mpl import matplotlib.pyplot as plt import matplotlib.patches as mpatches -from matplotlib.testing import setup as mpl_setup -import matplotlib.testing.compare as mcompare import packaging.version MPL_VERSION = packaging.version.parse(mpl.__version__) -class ImageTesting: - """ - Provides a convenient class for running visual Matplotlib tests. - - In general, this class should be used as a decorator to a test function - which generates one (or more) figures. - - :: - - @ImageTesting(['simple_test']) - def test_simple(): - - import matplotlib.pyplot as plt - plt.plot(range(10)) - - - To find out where the result and expected images reside one can create - a empty ImageTesting class instance and get the paths from the - :meth:`expected_path` and :meth:`result_path` methods:: - - >>> import os - >>> import cartopy.tests.mpl - >>> img_testing = cartopy.tests.mpl.ImageTesting([]) - >>> exp_fname = img_testing.expected_path('', '') - >>> result_fname = img_testing.result_path('', '') - >>> img_test_mod_dir = os.path.dirname(cartopy.__file__) - - >>> print('Result:', os.path.relpath(result_fname, img_test_mod_dir)) - ... # doctest: +ELLIPSIS - Result: ...output//result-.png - - >>> print('Expected:', os.path.relpath(exp_fname, img_test_mod_dir)) - Expected: tests/mpl/baseline_images/mpl//.png - - .. note:: - - Subclasses of the ImageTesting class may decide to change the - location of the expected and result images. However, the same - technique for finding the locations of the images should hold true. - - """ - - #: The path where the standard ``baseline_images`` exist. - root_image_results = os.path.dirname(__file__) - - #: The path where the images generated by the tests should go. - image_output_directory = os.path.join(root_image_results, 'output') - if not os.access(image_output_directory, os.W_OK): - if not os.access(os.getcwd(), os.W_OK): - raise OSError('Write access to a local disk is required to run ' - 'image tests. Run the tests from a current working ' - 'directory you have write access to to avoid this ' - 'issue.') - else: - image_output_directory = os.path.join(os.getcwd(), - 'cartopy_test_output') - - def __init__(self, img_names, tolerance=0.5, style='classic'): - # With matplotlib v1.3 the tolerance keyword is an RMS of the pixel - # differences, as computed by matplotlib.testing.compare.calculate_rms - self.img_names = img_names - self.style = style - self.tolerance = tolerance - - def expected_path(self, test_name, img_name, ext='.png'): - """ - Return the full path (minus extension) of where the expected image - should be found, given the name of the image being tested and the - name of the test being run. - - """ - expected_fname = os.path.join(self.root_image_results, - 'baseline_images', 'mpl', test_name, - img_name) - return expected_fname + ext - - def result_path(self, test_name, img_name, ext='.png'): - """ - Return the full path (minus extension) of where the result image - should be given the name of the image being tested and the - name of the test being run. - - """ - result_fname = os.path.join(self.image_output_directory, - test_name, 'result-' + img_name) - return result_fname + ext - - def run_figure_comparisons(self, figures, test_name): - """ - Run the figure comparisons against the ``image_names``. - - The number of figures passed must be equal to the number of - image names in ``self.image_names``. - - .. note:: - - The figures are not closed by this method. If using the decorator - version of ImageTesting, they will be closed for you. - - """ - n_figures_msg = ( - f'Expected {len(self.img_names)} figures (based on the number of ' - f'image result filenames), but there are {len(figures)} figures ' - f'available. The most likely reason for this is that this test is ' - f'producing too many figures, (alternatively if not using ' - f'ImageCompare as a decorator, it is possible that a test run ' - f'prior to this one has not closed its figures).') - assert len(figures) == len(self.img_names), n_figures_msg - - for img_name, figure in zip(self.img_names, figures): - expected_path = self.expected_path(test_name, img_name, '.png') - result_path = self.result_path(test_name, img_name, '.png') - - if not os.path.isdir(os.path.dirname(expected_path)): - os.makedirs(os.path.dirname(expected_path)) - - if not os.path.isdir(os.path.dirname(result_path)): - os.makedirs(os.path.dirname(result_path)) - - with flufl.lock.Lock(result_path + '.lock'): - self.save_figure(figure, result_path) - self.do_compare(result_path, expected_path, self.tolerance) - - def save_figure(self, figure, result_fname): - """ - The actual call which saves the figure. - - Returns nothing. - - May be overridden to do figure based pre-processing (such - as removing text objects etc.) - """ - figure.savefig(result_fname) - - def do_compare(self, result_fname, expected_fname, tol): - """ - Runs the comparison of the result file with the expected file. - - If an RMS difference greater than ``tol`` is found an assertion - error is raised with an appropriate message with the paths to - the files concerned. - - """ - if not os.path.exists(expected_fname): - warnings.warn('Created image in %s' % expected_fname) - shutil.copy2(result_fname, expected_fname) - - err = mcompare.compare_images(expected_fname, result_fname, - tol=tol, in_decorator=True) - - if err: - assert False, ( - f"Images were different (RMS: {err['rms']}).\n" - f"{err['actual']} {err['expected']} {err['diff']}\n" - f"Consider running idiff to inspect these differences.") - - def __call__(self, test_func): - """Called when the decorator is applied to a function.""" - test_name = test_func.__name__ - mod_name = test_func.__module__ - if mod_name == '__main__': - import sys - fname = sys.modules[mod_name].__file__ - mod_name = os.path.basename(os.path.splitext(fname)[0]) - mod_name = mod_name.rsplit('.', 1)[-1] - - def wrapped(*args, **kwargs): - orig_backend = plt.get_backend() - plt.switch_backend('agg') - mpl_setup() - - if plt.get_fignums(): - warnings.warn('Figures existed before running the %s %s test.' - ' All figures should be closed after they run. ' - 'They will be closed automatically now.' % - (mod_name, test_name)) - plt.close('all') - - with mpl.style.context(self.style): - if MPL_VERSION >= packaging.version.parse('3.2.0'): - mpl.rcParams['text.kerning_factor'] = 6 - - r = test_func(*args, **kwargs) - - figures = [plt.figure(num) for num in plt.get_fignums()] - - try: - self.run_figure_comparisons(figures, test_name=mod_name) - finally: - for figure in figures: - plt.close(figure) - plt.switch_backend(orig_backend) - return r - - # nose needs the function's name to be in the form "test_*" to - # pick it up - wrapped.__name__ = test_name - return wrapped - - -def failed_images_iter(): - """ - Return a generator of [expected, actual, diff] filenames for all failed - image tests since the test output directory was created. - """ - baseline_img_dir = os.path.join(ImageTesting.root_image_results, - 'baseline_images', 'mpl') - diff_dir = os.path.join(ImageTesting.image_output_directory) - - baselines = sorted(glob.glob(os.path.join(baseline_img_dir, - '*', '*.png'))) - for expected_fname in baselines: - # Get the relative path of the expected image 2 folders up. - expected_rel = os.path.relpath( - expected_fname, os.path.dirname(os.path.dirname(expected_fname))) - result_fname = os.path.join( - diff_dir, os.path.dirname(expected_rel), - 'result-' + os.path.basename(expected_rel)) - diff_fname = result_fname[:-4] + '-failed-diff.png' - if os.path.exists(diff_fname): - yield expected_fname, result_fname, diff_fname - - -def failed_images_html(): - """ - Generates HTML which shows the image failures side-by-side - when viewed in a web browser. - """ - data_uri_template = '{alt}' - - def image_as_base64(fname): - with open(fname, "rb") as fh: - return base64.b64encode(fh.read()).decode("ascii") - - html = ['', '', ''] - - for expected, actual, diff in failed_images_iter(): - expected_html = data_uri_template.format( - alt='expected', img=image_as_base64(expected)) - actual_html = data_uri_template.format( - alt='actual', img=image_as_base64(actual)) - diff_html = data_uri_template.format( - alt='diff', img=image_as_base64(diff)) - - html.extend([expected, '
', - expected_html, actual_html, diff_html, - '
']) - - html.extend(['', '']) - return '\n'.join(html) - - def show(projection, geometry): orig_backend = mpl.get_backend() plt.switch_backend('tkagg') diff --git a/lib/cartopy/tests/mpl/conftest.py b/lib/cartopy/tests/mpl/conftest.py index cdecfd6c9..15eb4ad03 100644 --- a/lib/cartopy/tests/mpl/conftest.py +++ b/lib/cartopy/tests/mpl/conftest.py @@ -19,3 +19,20 @@ def mpl_test_cleanup(request): finally: # Closes all open figures and switches backend back to original plt.switch_backend(orig_backend) + + +def pytest_itemcollected(item): + mpl_marker = item.get_closest_marker('mpl_image_compare') + if mpl_marker is None: + return + + # Matches old ImageTesting class default tolerance. + mpl_marker.kwargs.setdefault('tolerance', 0.5) + + for path in item.fspath.parts(reverse=True): + if path.basename == 'cartopy': + return + elif path.basename == 'tests': + subdir = item.fspath.relto(path)[:-len(item.fspath.ext)] + mpl_marker.kwargs['baseline_dir'] = f'baseline_images/{subdir}' + break diff --git a/lib/cartopy/tests/mpl/test_axes.py b/lib/cartopy/tests/mpl/test_axes.py index b4b0a11d8..a431ce989 100644 --- a/lib/cartopy/tests/mpl/test_axes.py +++ b/lib/cartopy/tests/mpl/test_axes.py @@ -14,7 +14,6 @@ import cartopy.crs as ccrs from cartopy.mpl.geoaxes import InterProjectionTransform, GeoAxes -from cartopy.tests.mpl import ImageTesting class TestNoSpherical: @@ -116,7 +115,7 @@ def test_geoaxes_subplot(): assert str(ax.__class__) == "" -@ImageTesting(['geoaxes_subslice']) +@pytest.mark.mpl_image_compare(filename='geoaxes_subslice.png') def test_geoaxes_no_subslice(): """Test that we do not trigger matplotlib's line subslice optimization.""" # This behavior caused lines with > 1000 points and @@ -128,8 +127,10 @@ def test_geoaxes_no_subslice(): lons = np.linspace(-117, -115, num_points) ax.plot(lons, lats, transform=ccrs.PlateCarree()) + return fig -@ImageTesting(['geoaxes_set_boundary_clipping']) + +@pytest.mark.mpl_image_compare(filename='geoaxes_set_boundary_clipping.png') def test_geoaxes_set_boundary_clipping(): """Test that setting the boundary works properly for clipping #1620.""" lon, lat = np.meshgrid(np.linspace(-180., 180., 361), @@ -145,3 +146,5 @@ def test_geoaxes_set_boundary_clipping(): ax1.set_boundary(mpath.Path.circle(center=(0.5, 0.5), radius=0.5), transform=ax1.transAxes) + + return fig diff --git a/lib/cartopy/tests/mpl/test_crs.py b/lib/cartopy/tests/mpl/test_crs.py index c61ac83be..5d0786ea3 100644 --- a/lib/cartopy/tests/mpl/test_crs.py +++ b/lib/cartopy/tests/mpl/test_crs.py @@ -9,20 +9,20 @@ import pytest import cartopy.crs as ccrs -from cartopy.tests.mpl import ImageTesting @pytest.mark.natural_earth -@ImageTesting(["igh_land"]) +@pytest.mark.mpl_image_compare(filename="igh_land.png") def test_igh_land(): crs = ccrs.InterruptedGoodeHomolosine(emphasis="land") ax = plt.axes(projection=crs) ax.coastlines() ax.gridlines() + return ax.figure @pytest.mark.natural_earth -@ImageTesting(["igh_ocean"]) +@pytest.mark.mpl_image_compare(filename="igh_ocean.png") def test_igh_ocean(): crs = ccrs.InterruptedGoodeHomolosine( central_longitude=-160, emphasis="ocean" @@ -30,10 +30,11 @@ def test_igh_ocean(): ax = plt.axes(projection=crs) ax.coastlines() ax.gridlines() + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['lambert_conformal_south']) +@pytest.mark.mpl_image_compare(filename='lambert_conformal_south.png') def test_lambert_south(): # Reference image: https://www.icsm.gov.au/mapping/map_projections.html crs = ccrs.LambertConformal(central_longitude=140, cutoff=65, @@ -41,6 +42,7 @@ def test_lambert_south(): ax = plt.axes(projection=crs) ax.coastlines() ax.gridlines() + return ax.figure @pytest.mark.natural_earth diff --git a/lib/cartopy/tests/mpl/test_examples.py b/lib/cartopy/tests/mpl/test_examples.py index 205ac0412..5ae17efb6 100644 --- a/lib/cartopy/tests/mpl/test_examples.py +++ b/lib/cartopy/tests/mpl/test_examples.py @@ -9,11 +9,11 @@ import pytest import cartopy.crs as ccrs -from cartopy.tests.mpl import MPL_VERSION, ImageTesting +from cartopy.tests.mpl import MPL_VERSION @pytest.mark.natural_earth -@ImageTesting(['global_map']) +@pytest.mark.mpl_image_compare(filename='global_map.png') def test_global_map(): fig = plt.figure(figsize=(10, 5)) ax = fig.add_subplot(1, 1, 1, projection=ccrs.Robinson()) @@ -29,10 +29,14 @@ def test_global_map(): ax.plot([-0.08, 132], [51.53, 43.17], transform=ccrs.PlateCarree()) ax.plot([-0.08, 132], [51.53, 43.17], transform=ccrs.Geodetic()) + return fig + @pytest.mark.natural_earth -@ImageTesting(['contour_label'], - tolerance=(9.9 if MPL_VERSION < parse_version('3.2') else 0.5)) +@pytest.mark.mpl_image_compare(filename='contour_label.png', + tolerance=(9.9 + if MPL_VERSION < parse_version('3.2') + else 0.5)) def test_contour_label(): from cartopy.tests.mpl.test_caching import sample_data fig = plt.figure() @@ -69,3 +73,5 @@ def test_contour_label(): inline=True, # Cut the line where the label will be placed. fmt=' {:.0f} '.format, # Labes as integers, with some extra space. ) + + return fig diff --git a/lib/cartopy/tests/mpl/test_features.py b/lib/cartopy/tests/mpl/test_features.py index 024775851..86ab21f2f 100644 --- a/lib/cartopy/tests/mpl/test_features.py +++ b/lib/cartopy/tests/mpl/test_features.py @@ -11,12 +11,10 @@ import cartopy.feature as cfeature from cartopy.io.ogc_clients import _OWSLIB_AVAILABLE -from cartopy.tests.mpl import ImageTesting - @pytest.mark.filterwarnings("ignore:Downloading") @pytest.mark.natural_earth -@ImageTesting(['natural_earth']) +@pytest.mark.mpl_image_compare(filename='natural_earth.png') def test_natural_earth(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.add_feature(cfeature.LAND) @@ -27,11 +25,12 @@ def test_natural_earth(): ax.add_feature(cfeature.RIVERS) ax.set_xlim((-20, 60)) ax.set_ylim((-40, 40)) + return ax.figure @pytest.mark.filterwarnings("ignore:Downloading") @pytest.mark.natural_earth -@ImageTesting(['natural_earth_custom']) +@pytest.mark.mpl_image_compare(filename='natural_earth_custom.png') def test_natural_earth_custom(): ax = plt.axes(projection=ccrs.PlateCarree()) feature = cfeature.NaturalEarthFeature('physical', 'coastline', '50m', @@ -40,9 +39,10 @@ def test_natural_earth_custom(): ax.add_feature(feature) ax.set_xlim((-26, -12)) ax.set_ylim((58, 72)) + return ax.figure -@ImageTesting(['gshhs_coastlines'], tolerance=0.95) +@pytest.mark.mpl_image_compare(filename='gshhs_coastlines.png', tolerance=0.95) def test_gshhs(): ax = plt.axes(projection=ccrs.Mollweide()) ax.set_extent([138, 142, 32, 42], ccrs.Geodetic()) @@ -53,11 +53,12 @@ def test_gshhs(): # Draw higher resolution lakes (and test overriding of kwargs) ax.add_feature(cfeature.GSHHSFeature('low', levels=[2], facecolor='green'), facecolor='blue') + return ax.figure @pytest.mark.network @pytest.mark.skipif(not _OWSLIB_AVAILABLE, reason='OWSLib is unavailable.') -@ImageTesting(['wfs']) +@pytest.mark.mpl_image_compare(filename='wfs.png') def test_wfs(): ax = plt.axes(projection=ccrs.OSGB(approx=True)) url = 'https://nsidc.org/cgi-bin/atlas_south?service=WFS' @@ -65,3 +66,4 @@ def test_wfs(): feature = cfeature.WFSFeature(url, typename, edgecolor='red') ax.add_feature(feature) + return ax.figure diff --git a/lib/cartopy/tests/mpl/test_gridliner.py b/lib/cartopy/tests/mpl/test_gridliner.py index 786df0622..6898165be 100644 --- a/lib/cartopy/tests/mpl/test_gridliner.py +++ b/lib/cartopy/tests/mpl/test_gridliner.py @@ -17,8 +17,6 @@ LATITUDE_FORMATTER, LONGITUDE_FORMATTER, classic_locator, classic_formatter) -from cartopy.tests.mpl import ImageTesting - TEST_PROJS = [ ccrs.PlateCarree, ccrs.AlbersEqualArea, @@ -52,9 +50,8 @@ @pytest.mark.natural_earth -@ImageTesting(['gridliner1'], - # Robinson projection is slightly better in Proj 6+. - tolerance=0.7) +# Robinson projection is slightly better in Proj 6+. +@pytest.mark.mpl_image_compare(filename='gridliner1.png', tolerance=0.7) def test_gridliner(): ny, nx = 2, 4 @@ -110,6 +107,7 @@ def test_gridliner(): delta = 1.5e-2 fig.subplots_adjust(left=0 + delta, right=1 - delta, top=1 - delta, bottom=0 + delta) + return fig def test_gridliner_specified_lines(): @@ -126,17 +124,14 @@ def test_gridliner_specified_lines(): # The tolerance on these tests are particularly high because of the high number # of text objects. A new testing strategy is needed for this kind of test. -grid_label_tol = grid_label_inline_tol = grid_label_inline_usa_tol = 3.1 -grid_label_tol += 0.8 -grid_label_inline_tol += 1.6 -grid_label_image = 'gridliner_labels' -grid_label_inline_image = 'gridliner_labels_inline' -grid_label_inline_usa_image = 'gridliner_labels_inline_usa' +grid_label_tol = 3.9 +grid_label_inline_tol = grid_label_inline_usa_tol = 10.5 @pytest.mark.skipif(geos_version == (3, 9, 0), reason="GEOS intersection bug") @pytest.mark.natural_earth -@ImageTesting([grid_label_image], tolerance=grid_label_tol) +@pytest.mark.mpl_image_compare(filename='gridliner_labels.png', + tolerance=grid_label_tol) def test_grid_labels(): fig = plt.figure(figsize=(10, 10)) @@ -207,10 +202,13 @@ def test_grid_labels(): # Increase margins between plots to stop them bumping into one another. fig.subplots_adjust(wspace=0.25, hspace=0.25) + return fig + @pytest.mark.skipif(geos_version == (3, 9, 0), reason="GEOS intersection bug") @pytest.mark.natural_earth -@ImageTesting(['gridliner_labels_tight'], tolerance=4) +@pytest.mark.mpl_image_compare(filename='gridliner_labels_tight.png', + tolerance=4) def test_grid_labels_tight(): # Ensure tight layout accounts for gridlines fig = plt.figure(figsize=(7, 5)) @@ -248,10 +246,13 @@ def test_grid_labels_tight(): for gl in ax._gridliners: assert hasattr(gl, '_drawn') and gl._drawn + return fig + @pytest.mark.skipif(geos_version == (3, 9, 0), reason="GEOS intersection bug") @pytest.mark.natural_earth -@ImageTesting([grid_label_inline_image], tolerance=grid_label_inline_tol) +@pytest.mark.mpl_image_compare(filename='gridliner_labels_inline.png', + tolerance=grid_label_inline_tol) def test_grid_labels_inline(): fig = plt.figure(figsize=(35, 35)) for i, proj in enumerate(TEST_PROJS, 1): @@ -264,12 +265,13 @@ def test_grid_labels_inline(): ax.coastlines(resolution="110m") ax.set_title(proj, y=1.075) fig.subplots_adjust(wspace=0.35, hspace=0.35) + return fig @pytest.mark.skipif(geos_version == (3, 9, 0), reason="GEOS intersection bug") @pytest.mark.natural_earth -@ImageTesting([grid_label_inline_usa_image], - tolerance=grid_label_inline_usa_tol) +@pytest.mark.mpl_image_compare(filename='gridliner_labels_inline_usa.png', + tolerance=grid_label_inline_usa_tol) def test_grid_labels_inline_usa(): top = 49.3457868 # north lat left = -124.7844079 # west long @@ -292,10 +294,12 @@ def test_grid_labels_inline_usa(): ax.coastlines(resolution="110m") fig.subplots_adjust(wspace=0.35, hspace=0.35) + return fig @pytest.mark.skipif(geos_version == (3, 9, 0), reason="GEOS intersection bug") -@ImageTesting(["gridliner_labels_bbox_style"], tolerance=grid_label_tol) +@pytest.mark.mpl_image_compare(filename='gridliner_labels_bbox_style.png', + tolerance=grid_label_tol) def test_gridliner_labels_bbox_style(): top = 49.3457868 # north lat left = -124.7844079 # west long @@ -317,6 +321,8 @@ def test_gridliner_labels_bbox_style(): "boxstyle": "round, pad=0.2", } + return fig + @pytest.mark.parametrize( "proj,gcrs,xloc,xfmt,xloc_expected,xfmt_expected", diff --git a/lib/cartopy/tests/mpl/test_images.py b/lib/cartopy/tests/mpl/test_images.py index 4f3e36eae..5f73a9e2a 100644 --- a/lib/cartopy/tests/mpl/test_images.py +++ b/lib/cartopy/tests/mpl/test_images.py @@ -19,7 +19,6 @@ import cartopy.crs as ccrs import cartopy.io.img_tiles as cimgt -from cartopy.tests.mpl import ImageTesting import cartopy.tests.test_img_tiles as ctest_tiles @@ -35,7 +34,7 @@ # care that it is putting images onto the map which are roughly correct. @pytest.mark.natural_earth @pytest.mark.network -@ImageTesting(['web_tiles'], tolerance=5.91) +@pytest.mark.mpl_image_compare(filename='web_tiles.png', tolerance=5.91) def test_web_tiles(): extent = [-15, 0.1, 50, 60] target_domain = sgeom.Polygon([[extent[0], extent[1]], @@ -69,10 +68,12 @@ def test_web_tiles(): interpolation='bilinear', origin=origin) ax.coastlines() + return fig + @pytest.mark.natural_earth @pytest.mark.network -@ImageTesting(['image_merge'], tolerance=0.01) +@pytest.mark.mpl_image_compare(filename='image_merge.png', tolerance=0.01) def test_image_merge(): # tests the basic image merging functionality tiles = [] @@ -97,8 +98,11 @@ def test_image_merge(): ax.coastlines() ax.imshow(img, origin=origin, extent=extent, alpha=0.5) + return ax.figure + -@ImageTesting(['imshow_natural_earth_ortho'], tolerance=0.7) +@pytest.mark.mpl_image_compare(filename='imshow_natural_earth_ortho.png', + tolerance=0.7) def test_imshow(): source_proj = ccrs.PlateCarree() img = plt.imread(NATURAL_EARTH_IMG) @@ -108,10 +112,12 @@ def test_imshow(): ax = plt.axes(projection=ccrs.Orthographic()) ax.imshow(img, transform=source_proj, extent=[-180, 180, -90, 90]) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['imshow_regional_projected'], tolerance=0.8) +@pytest.mark.mpl_image_compare(filename='imshow_regional_projected.png', + tolerance=0.8) def test_imshow_projected(): source_proj = ccrs.PlateCarree() img_extent = (-120.67660000000001, -106.32104523100001, @@ -121,6 +127,7 @@ def test_imshow_projected(): ax.set_extent(img_extent, crs=source_proj) ax.coastlines(resolution='50m') ax.imshow(img, extent=img_extent, transform=source_proj) + return ax.figure def test_imshow_wrapping(): @@ -160,25 +167,30 @@ def test_imshow_rgb(): assert sum(img.get_array().data[:, 0, 3]) == 0 -@ImageTesting(['imshow_natural_earth_ortho'], tolerance=0.7) +@pytest.mark.mpl_image_compare(filename='imshow_natural_earth_ortho.png', + tolerance=0.7) def test_stock_img(): ax = plt.axes(projection=ccrs.Orthographic()) ax.stock_img() + return ax.figure -@ImageTesting(['imshow_natural_earth_ortho'], tolerance=0.7) +@pytest.mark.mpl_image_compare(filename='imshow_natural_earth_ortho.png', + tolerance=0.7) def test_pil_Image(): img = Image.open(NATURAL_EARTH_IMG) source_proj = ccrs.PlateCarree() ax = plt.axes(projection=ccrs.Orthographic()) ax.imshow(img, transform=source_proj, extent=[-180, 180, -90, 90]) + return ax.figure -@ImageTesting(['imshow_natural_earth_ortho']) +@pytest.mark.mpl_image_compare(filename='imshow_natural_earth_ortho.png') def test_background_img(): ax = plt.axes(projection=ccrs.Orthographic()) ax.background_img(name='ne_shaded', resolution='low') + return ax.figure def test_alpha_2d_warp(): diff --git a/lib/cartopy/tests/mpl/test_img_transform.py b/lib/cartopy/tests/mpl/test_img_transform.py index b9bf3ea03..fb3f38810 100644 --- a/lib/cartopy/tests/mpl/test_img_transform.py +++ b/lib/cartopy/tests/mpl/test_img_transform.py @@ -13,7 +13,6 @@ import pytest from cartopy import config -from cartopy.tests.mpl import ImageTesting import cartopy.crs as ccrs import cartopy.img_transform as im_trans from functools import reduce @@ -80,7 +79,7 @@ def test_different_dims(self): # Bug in latest Matplotlib that we don't consider correct. @pytest.mark.natural_earth -@ImageTesting(['regrid_image'], tolerance=5.55) +@pytest.mark.mpl_image_compare(filename='regrid_image.png', tolerance=5.55) def test_regrid_image(): # Source data fname = os.path.join(config["repo_data_dir"], 'raster', 'natural_earth', @@ -123,3 +122,4 @@ def test_regrid_image(): # Tighten up layout gs.tight_layout(fig) + return fig diff --git a/lib/cartopy/tests/mpl/test_mpl_integration.py b/lib/cartopy/tests/mpl/test_mpl_integration.py index 0fe9feb73..65a3b4be5 100644 --- a/lib/cartopy/tests/mpl/test_mpl_integration.py +++ b/lib/cartopy/tests/mpl/test_mpl_integration.py @@ -14,76 +14,84 @@ import cartopy.crs as ccrs -from cartopy.tests.mpl import MPL_VERSION, ImageTesting +from cartopy.tests.mpl import MPL_VERSION # This is due to a change in MPL 3.5 contour line paths changing # ever so slightly. contour_tol = 2.24 + + @pytest.mark.natural_earth -@ImageTesting(['global_contour_wrap'], style='mpl20', - tolerance=contour_tol) +@pytest.mark.mpl_image_compare(filename='global_contour_wrap.png', + style='mpl20', tolerance=contour_tol) def test_global_contour_wrap_new_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines() x, y = np.meshgrid(np.linspace(0, 360), np.linspace(-90, 90)) data = np.sin(np.sqrt(x ** 2 + y ** 2)) ax.contour(x, y, data, transform=ccrs.PlateCarree()) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['global_contour_wrap'], style='mpl20', - tolerance=contour_tol) +@pytest.mark.mpl_image_compare(filename='global_contour_wrap.png', + style='mpl20', tolerance=contour_tol) def test_global_contour_wrap_no_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines() x, y = np.meshgrid(np.linspace(0, 360), np.linspace(-90, 90)) data = np.sin(np.sqrt(x ** 2 + y ** 2)) ax.contour(x, y, data) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['global_contourf_wrap']) +@pytest.mark.mpl_image_compare(filename='global_contourf_wrap.png') def test_global_contourf_wrap_new_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines() x, y = np.meshgrid(np.linspace(0, 360), np.linspace(-90, 90)) data = np.sin(np.sqrt(x ** 2 + y ** 2)) ax.contourf(x, y, data, transform=ccrs.PlateCarree()) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['global_contourf_wrap']) +@pytest.mark.mpl_image_compare(filename='global_contourf_wrap.png') def test_global_contourf_wrap_no_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines() x, y = np.meshgrid(np.linspace(0, 360), np.linspace(-90, 90)) data = np.sin(np.sqrt(x ** 2 + y ** 2)) ax.contourf(x, y, data) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['global_pcolor_wrap']) +@pytest.mark.mpl_image_compare(filename='global_pcolor_wrap.png') def test_global_pcolor_wrap_new_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines() x, y = np.meshgrid(np.linspace(0, 360), np.linspace(-90, 90)) data = np.sin(np.sqrt(x ** 2 + y ** 2))[:-1, :-1] ax.pcolor(x, y, data, transform=ccrs.PlateCarree()) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['global_pcolor_wrap']) +@pytest.mark.mpl_image_compare(filename='global_pcolor_wrap.png') def test_global_pcolor_wrap_no_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines() x, y = np.meshgrid(np.linspace(0, 360), np.linspace(-90, 90)) data = np.sin(np.sqrt(x ** 2 + y ** 2))[:-1, :-1] ax.pcolor(x, y, data) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['global_scatter_wrap']) +@pytest.mark.mpl_image_compare(filename='global_scatter_wrap.png') def test_global_scatter_wrap_new_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) # By default the coastline feature will be drawn after patches. @@ -93,21 +101,24 @@ def test_global_scatter_wrap_new_transform(): x, y = np.meshgrid(np.linspace(0, 360), np.linspace(-90, 90)) data = np.sin(np.sqrt(x ** 2 + y ** 2)) ax.scatter(x, y, c=data, transform=ccrs.PlateCarree()) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['global_scatter_wrap']) +@pytest.mark.mpl_image_compare(filename='global_scatter_wrap.png') def test_global_scatter_wrap_no_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines(zorder=0) x, y = np.meshgrid(np.linspace(0, 360), np.linspace(-90, 90)) data = np.sin(np.sqrt(x ** 2 + y ** 2)) ax.scatter(x, y, c=data) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['global_hexbin_wrap'], - tolerance=2 if MPL_VERSION < parse_version('3.2') else 0.5) +@pytest.mark.mpl_image_compare( + filename='global_hexbin_wrap.png', + tolerance=2 if MPL_VERSION < parse_version('3.2') else 0.5) def test_global_hexbin_wrap(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines(zorder=2) @@ -120,11 +131,13 @@ def test_global_hexbin_wrap(): gridsize=20, zorder=1, ) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['global_hexbin_wrap'], - tolerance=2 if MPL_VERSION < parse_version('3.2') else 0.5) +@pytest.mark.mpl_image_compare( + filename='global_hexbin_wrap.png', + tolerance=2 if MPL_VERSION < parse_version('3.2') else 0.5) def test_global_hexbin_wrap_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines(zorder=2) @@ -139,10 +152,10 @@ def test_global_hexbin_wrap_transform(): gridsize=20, zorder=1, ) + return ax.figure -@ImageTesting(['global_map'], - tolerance=0.55) +@pytest.mark.mpl_image_compare(filename='global_map.png', tolerance=0.55) def test_global_map(): ax = plt.axes(projection=ccrs.Robinson()) # ax.coastlines() @@ -155,21 +168,23 @@ def test_global_map(): ax.plot([-0.08, 132], [51.53, 43.17], color='blue', transform=ccrs.Geodetic()) + return ax.figure @pytest.mark.filterwarnings("ignore:Unable to determine extent") @pytest.mark.natural_earth -@ImageTesting(['simple_global']) +@pytest.mark.mpl_image_compare(filename='simple_global.png') def test_simple_global(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines() # produces a global map, despite not having needed to set the limits + return ax.figure @pytest.mark.filterwarnings("ignore:Unable to determine extent") @pytest.mark.natural_earth -@ImageTesting(['multiple_projections5'], - tolerance=2.05) +@pytest.mark.mpl_image_compare(filename='multiple_projections5.png', + tolerance=2.05) def test_multiple_projections(): projections = [ccrs.PlateCarree(), @@ -211,9 +226,12 @@ def test_multiple_projections(): ax.plot([-0.08, 132], [51.53, 43.17], color='blue', transform=prj.as_geodetic()) + return fig + @pytest.mark.natural_earth -@ImageTesting(['multiple_projections520'], tolerance=0.65) +@pytest.mark.mpl_image_compare(filename='multiple_projections520.png', + tolerance=0.65) def test_multiple_projections_520(): # Test projections added in Proj 5.2.0. @@ -232,6 +250,8 @@ def test_multiple_projections_520(): ax.plot([-0.08, 132], [51.53, 43.17], color='blue', transform=ccrs.Geodetic()) + return fig + def test_cursor_values(): ax = plt.axes(projection=ccrs.NorthPolarStereo()) @@ -255,7 +275,8 @@ def test_cursor_values(): @pytest.mark.natural_earth -@ImageTesting(['natural_earth_interface'], tolerance=0.21) +@pytest.mark.mpl_image_compare(filename='natural_earth_interface.png', + tolerance=0.21) def test_axes_natural_earth_interface(): rob = ccrs.Robinson() @@ -274,9 +295,12 @@ def test_axes_natural_earth_interface(): assert 'deprecated' in msg assert 'add_feature' in msg + return ax.figure + @pytest.mark.natural_earth -@ImageTesting(['pcolormesh_global_wrap1'], tolerance=1.27) +@pytest.mark.mpl_image_compare(filename='pcolormesh_global_wrap1.png', + tolerance=1.27) def test_pcolormesh_global_with_wrap1(): # make up some realistic data with bounds (such as data from the UM) nx, ny = 36, 18 @@ -298,6 +322,8 @@ def test_pcolormesh_global_with_wrap1(): ax.coastlines() ax.set_global() # make sure everything is visible + return fig + def test_pcolormesh_get_array_with_mask(): # make up some realistic data with bounds (such as data from the UM) @@ -343,7 +369,8 @@ def test_pcolormesh_get_array_with_mask(): @pytest.mark.natural_earth -@ImageTesting(['pcolormesh_global_wrap2'], tolerance=1.87) +@pytest.mark.mpl_image_compare(filename='pcolormesh_global_wrap2.png', + tolerance=1.87) def test_pcolormesh_global_with_wrap2(): # make up some realistic data with bounds (such as data from the UM) nx, ny = 36, 18 @@ -369,9 +396,12 @@ def test_pcolormesh_global_with_wrap2(): ax.coastlines() ax.set_global() # make sure everything is visible + return fig + @pytest.mark.natural_earth -@ImageTesting(['pcolormesh_global_wrap3'], tolerance=1.42) +@pytest.mark.mpl_image_compare(filename='pcolormesh_global_wrap3.png', + tolerance=1.42) def test_pcolormesh_global_with_wrap3(): nx, ny = 33, 17 xbnds = np.linspace(-1.875, 358.125, nx, endpoint=True) @@ -409,9 +439,12 @@ def test_pcolormesh_global_with_wrap3(): ax.coastlines() ax.set_global() # make sure everything is visible + return fig + @pytest.mark.natural_earth -@ImageTesting(['pcolormesh_global_wrap3'], tolerance=1.42) +@pytest.mark.mpl_image_compare(filename='pcolormesh_global_wrap3.png', + tolerance=1.42) def test_pcolormesh_set_array_with_mask(): """Testing that set_array works with masked arrays properly.""" nx, ny = 33, 17 @@ -457,9 +490,12 @@ def test_pcolormesh_set_array_with_mask(): ax.coastlines() ax.set_global() # make sure everything is visible + return fig + @pytest.mark.natural_earth -@ImageTesting(['pcolormesh_global_wrap3'], tolerance=1.42) +@pytest.mark.mpl_image_compare(filename='pcolormesh_global_wrap3.png', + tolerance=1.42) def test_pcolormesh_set_clim_with_mask(): """Testing that set_clim works with masked arrays properly.""" nx, ny = 33, 17 @@ -503,9 +539,12 @@ def test_pcolormesh_set_clim_with_mask(): # Fix clims on c so that test passes c.set_clim(data.min(), data.max()) + return fig + @pytest.mark.natural_earth -@ImageTesting(['pcolormesh_limited_area_wrap'], tolerance=1.82) +@pytest.mark.mpl_image_compare(filename='pcolormesh_limited_area_wrap.png', + tolerance=1.82) def test_pcolormesh_limited_area_wrap(): # make up some realistic data with bounds (such as data from the UM's North # Atlantic Europe model) @@ -544,9 +583,12 @@ def test_pcolormesh_limited_area_wrap(): snap=False) ax.coastlines() + return fig + @pytest.mark.natural_earth -@ImageTesting(['pcolormesh_single_column_wrap'], tolerance=0.7) +@pytest.mark.mpl_image_compare(filename='pcolormesh_single_column_wrap.png', + tolerance=0.7) def test_pcolormesh_single_column_wrap(): # Check a wrapped mesh like test_pcolormesh_limited_area_wrap, but only use # a single column, which could break depending on how wrapping is @@ -569,6 +611,8 @@ def test_pcolormesh_single_column_wrap(): ax.coastlines() ax.set_global() + return fig + def test_pcolormesh_diagonal_wrap(): # Check that a cell with the top edge on one side of the domain @@ -598,7 +642,7 @@ def test_pcolormesh_nan_wrap(): @pytest.mark.natural_earth -@ImageTesting(['pcolormesh_goode_wrap']) +@pytest.mark.mpl_image_compare(filename='pcolormesh_goode_wrap.png') def test_pcolormesh_goode_wrap(): # global data on an Interrupted Goode Homolosine projection # shouldn't spill outside projection boundary @@ -611,10 +655,12 @@ def test_pcolormesh_goode_wrap(): ax.coastlines() # TODO: Remove snap when updating this image ax.pcolormesh(x, y, Z, transform=ccrs.PlateCarree(), snap=False) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['pcolormesh_mercator_wrap'], tolerance=0.93) +@pytest.mark.mpl_image_compare(filename='pcolormesh_mercator_wrap.png', + tolerance=0.93) def test_pcolormesh_mercator_wrap(): x = np.linspace(0, 360, 73) y = np.linspace(-87.5, 87.5, 36) @@ -624,10 +670,12 @@ def test_pcolormesh_mercator_wrap(): ax = plt.axes(projection=ccrs.Mercator()) ax.coastlines() ax.pcolormesh(x, y, Z, transform=ccrs.PlateCarree(), snap=False) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['pcolormesh_mercator_wrap'], tolerance=0.93) +@pytest.mark.mpl_image_compare(filename='pcolormesh_mercator_wrap.png', + tolerance=0.93) def test_pcolormesh_wrap_set_array(): x = np.linspace(0, 360, 73) y = np.linspace(-87.5, 87.5, 36) @@ -642,10 +690,11 @@ def test_pcolormesh_wrap_set_array(): transform=ccrs.PlateCarree(), snap=False) # Now update the plot with the set_array method coll.set_array(Z.ravel()) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['quiver_plate_carree']) +@pytest.mark.mpl_image_compare(filename='quiver_plate_carree.png') def test_quiver_plate_carree(): x = np.arange(-60, 42.5, 2.5) y = np.arange(30, 72.5, 2.5) @@ -665,10 +714,11 @@ def test_quiver_plate_carree(): ax.set_extent(plot_extent, crs=ccrs.PlateCarree()) ax.coastlines() ax.quiver(x, y, u, v, mag, transform=ccrs.PlateCarree()) + return fig @pytest.mark.natural_earth -@ImageTesting(['quiver_rotated_pole']) +@pytest.mark.mpl_image_compare(filename='quiver_rotated_pole.png') def test_quiver_rotated_pole(): nx, ny = 22, 36 x = np.linspace(311.91998291, 391.11999512, nx, endpoint=True) @@ -690,10 +740,11 @@ def test_quiver_rotated_pole(): ax.set_extent(plot_extent, crs=rp) ax.coastlines() ax.quiver(x, y, u, v, mag, transform=rp) + return fig @pytest.mark.natural_earth -@ImageTesting(['quiver_regrid'], tolerance=1.3) +@pytest.mark.mpl_image_compare(filename='quiver_regrid.png', tolerance=1.3) def test_quiver_regrid(): x = np.arange(-60, 42.5, 2.5) y = np.arange(30, 72.5, 2.5) @@ -702,16 +753,18 @@ def test_quiver_regrid(): v = np.cos(2. * np.deg2rad(x2d)) mag = (u**2 + v**2)**.5 plot_extent = [-60, 40, 30, 70] - plt.figure(figsize=(6, 3)) - ax = plt.axes(projection=ccrs.NorthPolarStereo()) + fig = plt.figure(figsize=(6, 3)) + ax = fig.add_subplot(projection=ccrs.NorthPolarStereo()) ax.set_extent(plot_extent, crs=ccrs.PlateCarree()) ax.coastlines() ax.quiver(x, y, u, v, mag, transform=ccrs.PlateCarree(), regrid_shape=30) + return fig @pytest.mark.natural_earth -@ImageTesting(['quiver_regrid_with_extent'], tolerance=0.54) +@pytest.mark.mpl_image_compare(filename='quiver_regrid_with_extent.png', + tolerance=0.54) def test_quiver_regrid_with_extent(): x = np.arange(-60, 42.5, 2.5) y = np.arange(30, 72.5, 2.5) @@ -721,16 +774,17 @@ def test_quiver_regrid_with_extent(): mag = (u**2 + v**2)**.5 plot_extent = [-60, 40, 30, 70] target_extent = [-3e6, 2e6, -6e6, -2.5e6] - plt.figure(figsize=(6, 3)) - ax = plt.axes(projection=ccrs.NorthPolarStereo()) + fig = plt.figure(figsize=(6, 3)) + ax = fig.add_subplot(projection=ccrs.NorthPolarStereo()) ax.set_extent(plot_extent, crs=ccrs.PlateCarree()) ax.coastlines() ax.quiver(x, y, u, v, mag, transform=ccrs.PlateCarree(), regrid_shape=10, target_extent=target_extent) + return fig @pytest.mark.natural_earth -@ImageTesting(['barbs_plate_carree']) +@pytest.mark.mpl_image_compare(filename='barbs_plate_carree.png') def test_barbs(): x = np.arange(-60, 45, 5) y = np.arange(30, 75, 5) @@ -749,10 +803,11 @@ def test_barbs(): ax.set_extent(plot_extent, crs=ccrs.PlateCarree()) ax.coastlines(resolution="110m") ax.barbs(x, y, u, v, transform=ccrs.PlateCarree(), length=4, linewidth=.25) + return fig @pytest.mark.natural_earth -@ImageTesting(['barbs_regrid']) +@pytest.mark.mpl_image_compare(filename='barbs_regrid.png') def test_barbs_regrid(): x = np.arange(-60, 42.5, 2.5) y = np.arange(30, 72.5, 2.5) @@ -761,16 +816,18 @@ def test_barbs_regrid(): v = 40 * np.cos(2. * np.deg2rad(x2d)) mag = (u**2 + v**2)**.5 plot_extent = [-60, 40, 30, 70] - plt.figure(figsize=(6, 3)) - ax = plt.axes(projection=ccrs.NorthPolarStereo()) + fig = plt.figure(figsize=(6, 3)) + ax = fig.add_subplot(projection=ccrs.NorthPolarStereo()) ax.set_extent(plot_extent, crs=ccrs.PlateCarree()) ax.coastlines() ax.barbs(x, y, u, v, mag, transform=ccrs.PlateCarree(), length=4, linewidth=.4, regrid_shape=20) + return fig @pytest.mark.natural_earth -@ImageTesting(['barbs_regrid_with_extent'], tolerance=0.54) +@pytest.mark.mpl_image_compare(filename='barbs_regrid_with_extent.png', + tolerance=0.54) def test_barbs_regrid_with_extent(): x = np.arange(-60, 42.5, 2.5) y = np.arange(30, 72.5, 2.5) @@ -780,50 +837,54 @@ def test_barbs_regrid_with_extent(): mag = (u**2 + v**2)**.5 plot_extent = [-60, 40, 30, 70] target_extent = [-3e6, 2e6, -6e6, -2.5e6] - plt.figure(figsize=(6, 3)) - ax = plt.axes(projection=ccrs.NorthPolarStereo()) + fig = plt.figure(figsize=(6, 3)) + ax = fig.add_subplot(projection=ccrs.NorthPolarStereo()) ax.set_extent(plot_extent, crs=ccrs.PlateCarree()) ax.coastlines() ax.barbs(x, y, u, v, mag, transform=ccrs.PlateCarree(), length=4, linewidth=.25, regrid_shape=10, target_extent=target_extent) + return fig @pytest.mark.natural_earth -@ImageTesting(['barbs_1d']) +@pytest.mark.mpl_image_compare(filename='barbs_1d.png') def test_barbs_1d(): x = np.array([20., 30., -17., 15.]) y = np.array([-1., 35., 11., 40.]) u = np.array([23., -18., 2., -11.]) v = np.array([5., -4., 19., 11.]) plot_extent = [-21, 40, -5, 45] - plt.figure(figsize=(6, 5)) - ax = plt.axes(projection=ccrs.PlateCarree()) + fig = plt.figure(figsize=(6, 5)) + ax = fig.add_subplot(projection=ccrs.PlateCarree()) ax.set_extent(plot_extent, crs=ccrs.PlateCarree()) ax.coastlines(resolution="110m") ax.barbs(x, y, u, v, transform=ccrs.PlateCarree(), length=8, linewidth=1, color='#7f7f7f') + return fig @pytest.mark.natural_earth -@ImageTesting(['barbs_1d_transformed']) +@pytest.mark.mpl_image_compare(filename='barbs_1d_transformed.png') def test_barbs_1d_transformed(): x = np.array([20., 30., -17., 15.]) y = np.array([-1., 35., 11., 40.]) u = np.array([23., -18., 2., -11.]) v = np.array([5., -4., 19., 11.]) plot_extent = [-20, 31, -5, 45] - plt.figure(figsize=(6, 5)) - ax = plt.axes(projection=ccrs.NorthPolarStereo()) + fig = plt.figure(figsize=(6, 5)) + ax = fig.add_subplot(projection=ccrs.NorthPolarStereo()) ax.set_extent(plot_extent, crs=ccrs.PlateCarree()) ax.coastlines() ax.barbs(x, y, u, v, transform=ccrs.PlateCarree(), length=8, linewidth=1, color='#7f7f7f') + return fig @pytest.mark.natural_earth -@ImageTesting(['streamplot'], style='mpl20', - tolerance=42 if MPL_VERSION < parse_version('3.2') else 0.54) +@pytest.mark.mpl_image_compare( + filename='streamplot.png', style='mpl20', + tolerance=42 if MPL_VERSION < parse_version('3.2') else 0.54) def test_streamplot(): x = np.arange(-60, 42.5, 2.5) y = np.arange(30, 72.5, 2.5) @@ -832,9 +893,10 @@ def test_streamplot(): v = np.cos(2. * np.deg2rad(x2d)) mag = (u**2 + v**2)**.5 plot_extent = [-60, 40, 30, 70] - plt.figure(figsize=(6, 3)) - ax = plt.axes(projection=ccrs.NorthPolarStereo()) + fig = plt.figure(figsize=(6, 3)) + ax = fig.add_subplot(projection=ccrs.NorthPolarStereo()) ax.set_extent(plot_extent, crs=ccrs.PlateCarree()) ax.coastlines() ax.streamplot(x, y, u, v, transform=ccrs.PlateCarree(), density=(1.5, 2), color=mag, linewidth=2*mag) + return fig diff --git a/lib/cartopy/tests/mpl/test_nightshade.py b/lib/cartopy/tests/mpl/test_nightshade.py index 12effa0fd..b63e6d7c6 100644 --- a/lib/cartopy/tests/mpl/test_nightshade.py +++ b/lib/cartopy/tests/mpl/test_nightshade.py @@ -11,11 +11,10 @@ import cartopy.crs as ccrs from cartopy.feature.nightshade import Nightshade -from cartopy.tests.mpl import ImageTesting @pytest.mark.natural_earth -@ImageTesting(['nightshade_platecarree']) +@pytest.mark.mpl_image_compare(filename='nightshade_platecarree.png') def test_nightshade_image(): # Test the actual creation of the image ax = plt.axes(projection=ccrs.PlateCarree()) @@ -23,3 +22,4 @@ def test_nightshade_image(): dt = datetime(2018, 11, 10, 0, 0) ax.set_global() ax.add_feature(Nightshade(dt, alpha=0.75)) + return ax.figure diff --git a/lib/cartopy/tests/mpl/test_shapely_to_mpl.py b/lib/cartopy/tests/mpl/test_shapely_to_mpl.py index d0e315baf..2d47c1669 100644 --- a/lib/cartopy/tests/mpl/test_shapely_to_mpl.py +++ b/lib/cartopy/tests/mpl/test_shapely_to_mpl.py @@ -15,13 +15,11 @@ import cartopy.crs as ccrs import cartopy.mpl.patch as cpatch -from cartopy.tests.mpl import ImageTesting - # Note: Matplotlib is broken here # https://github.com/matplotlib/matplotlib/issues/15946 @pytest.mark.natural_earth -@ImageTesting(['poly_interiors'], tolerance=3.1) +@pytest.mark.mpl_image_compare(filename='poly_interiors.png', tolerance=3.1) def test_polygon_interiors(): fig = plt.figure() @@ -72,9 +70,11 @@ def test_polygon_interiors(): transform=ccrs.Geodetic(), zorder=10) ax.add_collection(collection) + return fig + @pytest.mark.natural_earth -@ImageTesting(['contour_with_interiors']) +@pytest.mark.mpl_image_compare(filename='contour_with_interiors.png') def test_contour_interiors(): # produces a polygon with multiple holes: nx, ny = 10, 10 @@ -112,3 +112,5 @@ def test_contour_interiors(): ax.set_global() ax.contourf(lons, lats, data, numlev, transform=ccrs.PlateCarree()) ax.coastlines() + + return fig diff --git a/lib/cartopy/tests/mpl/test_ticks.py b/lib/cartopy/tests/mpl/test_ticks.py index a3671537c..d629ccd7f 100644 --- a/lib/cartopy/tests/mpl/test_ticks.py +++ b/lib/cartopy/tests/mpl/test_ticks.py @@ -8,27 +8,28 @@ import cartopy.crs as ccrs from cartopy.mpl.ticker import LongitudeFormatter, LatitudeFormatter -from cartopy.tests.mpl import ImageTesting @pytest.mark.natural_earth -@ImageTesting(['xticks_no_transform']) +@pytest.mark.mpl_image_compare(filename='xticks_no_transform.png') def test_set_xticks_no_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines('110m') ax.xaxis.set_major_formatter(LongitudeFormatter(degree_symbol='')) ax.set_xticks([-180, -90, 0, 90, 180]) ax.set_xticks([-135, -45, 45, 135], minor=True) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['xticks_cylindrical']) +@pytest.mark.mpl_image_compare(filename='xticks_cylindrical.png') def test_set_xticks_cylindrical(): ax = plt.axes(projection=ccrs.Mercator(min_latitude=-85, max_latitude=85)) ax.coastlines('110m') ax.xaxis.set_major_formatter(LongitudeFormatter(degree_symbol='')) ax.set_xticks([-180, -90, 0, 90, 180], crs=ccrs.PlateCarree()) ax.set_xticks([-135, -45, 45, 135], minor=True, crs=ccrs.PlateCarree()) + return ax.figure def test_set_xticks_non_cylindrical(): @@ -40,17 +41,18 @@ def test_set_xticks_non_cylindrical(): @pytest.mark.natural_earth -@ImageTesting(['yticks_no_transform']) +@pytest.mark.mpl_image_compare(filename='yticks_no_transform.png') def test_set_yticks_no_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines('110m') ax.yaxis.set_major_formatter(LatitudeFormatter(degree_symbol='')) ax.set_yticks([-60, -30, 0, 30, 60]) ax.set_yticks([-75, -45, -15, 15, 45, 75], minor=True) + return ax.figure @pytest.mark.natural_earth -@ImageTesting(['yticks_cylindrical']) +@pytest.mark.mpl_image_compare(filename='yticks_cylindrical.png') def test_set_yticks_cylindrical(): ax = plt.axes(projection=ccrs.Mercator(min_latitude=-85, max_latitude=85)) ax.coastlines('110m') @@ -58,6 +60,7 @@ def test_set_yticks_cylindrical(): ax.set_yticks([-60, -30, 0, 30, 60], crs=ccrs.PlateCarree()) ax.set_yticks([-75, -45, -15, 15, 45, 75], minor=True, crs=ccrs.PlateCarree()) + return ax.figure def test_set_yticks_non_cylindrical(): @@ -70,7 +73,7 @@ def test_set_yticks_non_cylindrical(): @pytest.mark.natural_earth -@ImageTesting(['xyticks']) +@pytest.mark.mpl_image_compare(filename='xyticks.png') def test_set_xyticks(): fig = plt.figure(figsize=(10, 10)) projections = (ccrs.PlateCarree(), @@ -85,3 +88,4 @@ def test_set_xyticks(): p, q = prj.transform_point(x, y, ccrs.Geodetic()) ax.set_xticks([p]) ax.set_yticks([q]) + return fig diff --git a/lib/cartopy/tests/mpl/test_web_services.py b/lib/cartopy/tests/mpl/test_web_services.py index 1d4bfd3f4..52c8bb262 100644 --- a/lib/cartopy/tests/mpl/test_web_services.py +++ b/lib/cartopy/tests/mpl/test_web_services.py @@ -8,7 +8,6 @@ from matplotlib.testing.decorators import cleanup import pytest -from cartopy.tests.mpl import ImageTesting import cartopy.crs as ccrs from cartopy.io.ogc_clients import _OWSLIB_AVAILABLE @@ -17,12 +16,13 @@ @pytest.mark.network @pytest.mark.skipif(not _OWSLIB_AVAILABLE, reason='OWSLib is unavailable.') @pytest.mark.xfail(raises=KeyError, reason='OWSLib WMTS support is broken.') -@ImageTesting(['wmts'], tolerance=0) +@pytest.mark.mpl_image_compare(filename='wmts.png', tolerance=0) def test_wmts(): ax = plt.axes(projection=ccrs.PlateCarree()) url = 'https://map1c.vis.earthdata.nasa.gov/wmts-geo/wmts.cgi' # Use a layer which doesn't change over time. ax.add_wmts(url, 'MODIS_Water_Mask') + return ax.figure @pytest.mark.network @@ -38,9 +38,10 @@ def test_wms_tight_layout(): @pytest.mark.network @pytest.mark.skipif(not _OWSLIB_AVAILABLE, reason='OWSLib is unavailable.') -@ImageTesting(['wms'], tolerance=0.02) +@pytest.mark.mpl_image_compare(filename='wms.png', tolerance=0.02) def test_wms(): ax = plt.axes(projection=ccrs.Orthographic()) url = 'http://vmap0.tiles.osgeo.org/wms/vmap0' layer = 'basic' ax.add_wms(url, layer) + return ax.figure diff --git a/requirements/tests.txt b/requirements/tests.txt index faf0a427b..02688a323 100644 --- a/requirements/tests.txt +++ b/requirements/tests.txt @@ -1,3 +1,3 @@ -flufl.lock packaging>=20 pytest>=5.1.2 +pytest-mpl>=0.11 diff --git a/setup.cfg b/setup.cfg index 3e9cec0a2..1433c16c6 100644 --- a/setup.cfg +++ b/setup.cfg @@ -7,3 +7,6 @@ exclude = \ build, \ setup.py, \ docs/source/conf.py + +[tool:pytest] +addopts = --mpl From ff688e51ff25a365890b2cffe1b083b02eb76558 Mon Sep 17 00:00:00 2001 From: Elliott Sales de Andrade Date: Thu, 23 Sep 2021 21:16:51 -0400 Subject: [PATCH 2/2] Reduce test image tolerances where possible --- lib/cartopy/tests/mpl/test_gridliner.py | 3 +- lib/cartopy/tests/mpl/test_images.py | 9 ++---- lib/cartopy/tests/mpl/test_mpl_integration.py | 29 +++++++------------ 3 files changed, 15 insertions(+), 26 deletions(-) diff --git a/lib/cartopy/tests/mpl/test_gridliner.py b/lib/cartopy/tests/mpl/test_gridliner.py index 6898165be..a8ad2a760 100644 --- a/lib/cartopy/tests/mpl/test_gridliner.py +++ b/lib/cartopy/tests/mpl/test_gridliner.py @@ -208,7 +208,7 @@ def test_grid_labels(): @pytest.mark.skipif(geos_version == (3, 9, 0), reason="GEOS intersection bug") @pytest.mark.natural_earth @pytest.mark.mpl_image_compare(filename='gridliner_labels_tight.png', - tolerance=4) + tolerance=2.9) def test_grid_labels_tight(): # Ensure tight layout accounts for gridlines fig = plt.figure(figsize=(7, 5)) @@ -273,6 +273,7 @@ def test_grid_labels_inline(): @pytest.mark.mpl_image_compare(filename='gridliner_labels_inline_usa.png', tolerance=grid_label_inline_usa_tol) def test_grid_labels_inline_usa(): + print(plt.get_backend()) top = 49.3457868 # north lat left = -124.7844079 # west long right = -66.9513812 # east long diff --git a/lib/cartopy/tests/mpl/test_images.py b/lib/cartopy/tests/mpl/test_images.py index 5f73a9e2a..830af6e66 100644 --- a/lib/cartopy/tests/mpl/test_images.py +++ b/lib/cartopy/tests/mpl/test_images.py @@ -101,8 +101,7 @@ def test_image_merge(): return ax.figure -@pytest.mark.mpl_image_compare(filename='imshow_natural_earth_ortho.png', - tolerance=0.7) +@pytest.mark.mpl_image_compare(filename='imshow_natural_earth_ortho.png') def test_imshow(): source_proj = ccrs.PlateCarree() img = plt.imread(NATURAL_EARTH_IMG) @@ -167,16 +166,14 @@ def test_imshow_rgb(): assert sum(img.get_array().data[:, 0, 3]) == 0 -@pytest.mark.mpl_image_compare(filename='imshow_natural_earth_ortho.png', - tolerance=0.7) +@pytest.mark.mpl_image_compare(filename='imshow_natural_earth_ortho.png') def test_stock_img(): ax = plt.axes(projection=ccrs.Orthographic()) ax.stock_img() return ax.figure -@pytest.mark.mpl_image_compare(filename='imshow_natural_earth_ortho.png', - tolerance=0.7) +@pytest.mark.mpl_image_compare(filename='imshow_natural_earth_ortho.png') def test_pil_Image(): img = Image.open(NATURAL_EARTH_IMG) source_proj = ccrs.PlateCarree() diff --git a/lib/cartopy/tests/mpl/test_mpl_integration.py b/lib/cartopy/tests/mpl/test_mpl_integration.py index 65a3b4be5..dd04f0984 100644 --- a/lib/cartopy/tests/mpl/test_mpl_integration.py +++ b/lib/cartopy/tests/mpl/test_mpl_integration.py @@ -17,14 +17,9 @@ from cartopy.tests.mpl import MPL_VERSION -# This is due to a change in MPL 3.5 contour line paths changing -# ever so slightly. -contour_tol = 2.24 - - @pytest.mark.natural_earth @pytest.mark.mpl_image_compare(filename='global_contour_wrap.png', - style='mpl20', tolerance=contour_tol) + style='mpl20') def test_global_contour_wrap_new_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines() @@ -36,7 +31,7 @@ def test_global_contour_wrap_new_transform(): @pytest.mark.natural_earth @pytest.mark.mpl_image_compare(filename='global_contour_wrap.png', - style='mpl20', tolerance=contour_tol) + style='mpl20') def test_global_contour_wrap_no_transform(): ax = plt.axes(projection=ccrs.PlateCarree()) ax.coastlines() @@ -230,8 +225,7 @@ def test_multiple_projections(): @pytest.mark.natural_earth -@pytest.mark.mpl_image_compare(filename='multiple_projections520.png', - tolerance=0.65) +@pytest.mark.mpl_image_compare(filename='multiple_projections520.png') def test_multiple_projections_520(): # Test projections added in Proj 5.2.0. @@ -401,7 +395,7 @@ def test_pcolormesh_global_with_wrap2(): @pytest.mark.natural_earth @pytest.mark.mpl_image_compare(filename='pcolormesh_global_wrap3.png', - tolerance=1.42) + tolerance=1.39) def test_pcolormesh_global_with_wrap3(): nx, ny = 33, 17 xbnds = np.linspace(-1.875, 358.125, nx, endpoint=True) @@ -444,7 +438,7 @@ def test_pcolormesh_global_with_wrap3(): @pytest.mark.natural_earth @pytest.mark.mpl_image_compare(filename='pcolormesh_global_wrap3.png', - tolerance=1.42) + tolerance=1.39) def test_pcolormesh_set_array_with_mask(): """Testing that set_array works with masked arrays properly.""" nx, ny = 33, 17 @@ -495,7 +489,7 @@ def test_pcolormesh_set_array_with_mask(): @pytest.mark.natural_earth @pytest.mark.mpl_image_compare(filename='pcolormesh_global_wrap3.png', - tolerance=1.42) + tolerance=1.39) def test_pcolormesh_set_clim_with_mask(): """Testing that set_clim works with masked arrays properly.""" nx, ny = 33, 17 @@ -587,8 +581,7 @@ def test_pcolormesh_limited_area_wrap(): @pytest.mark.natural_earth -@pytest.mark.mpl_image_compare(filename='pcolormesh_single_column_wrap.png', - tolerance=0.7) +@pytest.mark.mpl_image_compare(filename='pcolormesh_single_column_wrap.png') def test_pcolormesh_single_column_wrap(): # Check a wrapped mesh like test_pcolormesh_limited_area_wrap, but only use # a single column, which could break depending on how wrapping is @@ -659,8 +652,7 @@ def test_pcolormesh_goode_wrap(): @pytest.mark.natural_earth -@pytest.mark.mpl_image_compare(filename='pcolormesh_mercator_wrap.png', - tolerance=0.93) +@pytest.mark.mpl_image_compare(filename='pcolormesh_mercator_wrap.png') def test_pcolormesh_mercator_wrap(): x = np.linspace(0, 360, 73) y = np.linspace(-87.5, 87.5, 36) @@ -674,8 +666,7 @@ def test_pcolormesh_mercator_wrap(): @pytest.mark.natural_earth -@pytest.mark.mpl_image_compare(filename='pcolormesh_mercator_wrap.png', - tolerance=0.93) +@pytest.mark.mpl_image_compare(filename='pcolormesh_mercator_wrap.png') def test_pcolormesh_wrap_set_array(): x = np.linspace(0, 360, 73) y = np.linspace(-87.5, 87.5, 36) @@ -744,7 +735,7 @@ def test_quiver_rotated_pole(): @pytest.mark.natural_earth -@pytest.mark.mpl_image_compare(filename='quiver_regrid.png', tolerance=1.3) +@pytest.mark.mpl_image_compare(filename='quiver_regrid.png') def test_quiver_regrid(): x = np.arange(-60, 42.5, 2.5) y = np.arange(30, 72.5, 2.5)