From 810c6034d7d0d993aa81f180d8056bee41e04471 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 27 Nov 2023 11:48:10 -0600 Subject: [PATCH 1/3] Cleanup HDF5 test warnings --- .github/workflows/ci.yaml | 2 +- build_environment.yml | 2 +- polar2grid/utils/legacy_compat.py | 21 +++++++++++++++++++++ polar2grid/writers/hdf5.py | 5 +++-- pyproject.toml | 3 +++ 5 files changed, 29 insertions(+), 4 deletions(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index f1077a43..891b67df 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -5,7 +5,7 @@ concurrency: group: ${{ github.workflow }}-${{ github.event.number }}-${{ github.event.type }} cancel-in-progress: true -on: [push, pull_request] +on: [push, pull_request, workflow_dispatch] env: CACHE_NUMBER: 1 diff --git a/build_environment.yml b/build_environment.yml index dc410973..a171f263 100644 --- a/build_environment.yml +++ b/build_environment.yml @@ -37,7 +37,7 @@ dependencies: - requests - setuptools - six - - trollimage>=1.21.0 + - trollimage>=1.22.2 - trollsift>=0.5.0 - scipy - zarr diff --git a/polar2grid/utils/legacy_compat.py b/polar2grid/utils/legacy_compat.py index 2cd920bb..80e581cc 100644 --- a/polar2grid/utils/legacy_compat.py +++ b/polar2grid/utils/legacy_compat.py @@ -24,7 +24,9 @@ from __future__ import annotations +import contextlib import logging +import warnings from typing import Generator, Iterable, Optional, Union from satpy import DataID, DataQuery, Scene @@ -257,3 +259,22 @@ def get_sensor_alias(satpy_sensor): if len(new_sensor) == 1: return new_sensor.pop() return new_sensor + + +@contextlib.contextmanager +def ignore_pyproj_proj_warnings(): + """Wrap operations that we know will produce a PROJ.4 precision warning. + + Only to be used internally to Pyresample when we have no other choice but + to use PROJ.4 strings/dicts. For example, serialization to YAML or other + human-readable formats or testing the methods that produce the PROJ.4 + versions of the CRS. + + """ + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + "You will likely lose important projection information", + UserWarning, + ) + yield diff --git a/polar2grid/writers/hdf5.py b/polar2grid/writers/hdf5.py index 9109915c..f93bd7c7 100644 --- a/polar2grid/writers/hdf5.py +++ b/polar2grid/writers/hdf5.py @@ -52,7 +52,7 @@ from pyresample.geometry import SwathDefinition from satpy.writers import Writer, compute_writer_results, split_results -from polar2grid.utils.legacy_compat import convert_p2g_pattern_to_satpy +from polar2grid.utils.legacy_compat import convert_p2g_pattern_to_satpy, ignore_pyproj_proj_warnings from polar2grid.writers.geotiff import NUMPY_DTYPE_STRS, NumpyDtypeList, str_to_dtype LOG = logging.getLogger(__name__) @@ -163,7 +163,8 @@ def create_proj_group(filename: str, parent: TextIO, area_def): group.attrs["height"], group.attrs["width"] = area_def.shape group.attrs["description"] = "No projection: native format" else: - group.attrs["proj4_definition"] = area_def.proj4_string + with ignore_pyproj_proj_warnings(): + group.attrs["proj4_definition"] = area_def.crs.to_string() for a in ["height", "width"]: ds_attr = getattr(area_def, a, None) if ds_attr is None: diff --git a/pyproject.toml b/pyproject.toml index 4dacc5af..8179a94b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -56,6 +56,9 @@ include-package-data = true [tool.setuptools.packages] find = {} +[tool.pytest.ini_options] +minversion = 7.0 + [tool.coverage.run] relative_files = true From 50e3251ed20e85ea1fb4fe9f790c54230ea189a6 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 27 Nov 2023 11:55:17 -0600 Subject: [PATCH 2/3] Ignore geotiff geolocation warnings in compare script tests --- polar2grid/tests/test_compare.py | 47 ++++++++++++++++++++++---------- 1 file changed, 32 insertions(+), 15 deletions(-) diff --git a/polar2grid/tests/test_compare.py b/polar2grid/tests/test_compare.py index 1f44d8b0..c65c2cce 100644 --- a/polar2grid/tests/test_compare.py +++ b/polar2grid/tests/test_compare.py @@ -21,8 +21,9 @@ # input into another program. # Documentation: http://www.ssec.wisc.edu/software/polar2grid/ """Tests for the compare.py script.""" - +import contextlib import os +import warnings from glob import glob import numpy as np @@ -43,6 +44,20 @@ IMAGE_LIST2 = [IMAGE1_L_UINT8_ZEROS, IMAGE2_L_UINT8_ZEROS, IMAGE4_RGB_UINT8_ZEROS] +@contextlib.contextmanager +def ignore_no_georef(): + """Wrap operations that we know will produce a rasterio geolocation warning.""" + from rasterio.errors import NotGeoreferencedWarning + + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + "Dataset has no geotransform", + NotGeoreferencedWarning, + ) + yield + + def _create_geotiffs(base_dir, img_data): import rasterio @@ -52,19 +67,20 @@ def _create_geotiffs(base_dir, img_data): for idx, img_arr in enumerate(img_data): band_count = 1 if img_arr.ndim == 2 else img_arr.shape[0] gtiff_fn = os.path.join(base_dir, f"test{idx}.tif") - with rasterio.open( - gtiff_fn, - "w", - driver="GTiff", - count=band_count, - height=img_arr.shape[-2], - width=img_arr.shape[-1], - dtype=img_arr.dtype, - ) as gtiff_file: - if img_arr.ndim == 2: - img_arr = img_arr[None, :, :] - for band_idx, band_arr in enumerate(img_arr): - gtiff_file.write(band_arr, band_idx + 1) + with ignore_no_georef(): + with rasterio.open( + gtiff_fn, + "w", + driver="GTiff", + count=band_count, + height=img_arr.shape[-2], + width=img_arr.shape[-1], + dtype=img_arr.dtype, + ) as gtiff_file: + if img_arr.ndim == 2: + img_arr = img_arr[None, :, :] + for band_idx, band_arr in enumerate(img_arr): + gtiff_file.write(band_arr, band_idx + 1) def _create_hdf5(base_dir, img_data): @@ -196,7 +212,8 @@ def test_basic_compare( if include_html: args.extend(["--html", str(html_file)]) - num_diff_files = main(args) + with ignore_no_georef(): + num_diff_files = main(args) exp_num_png_files = _get_exp_num_png_files(actual_data, expected_data, expected_file_func) _check_num_diff_files(num_diff_files, exp_num_diff, expected_file_func, actual_file_func) _check_html_output(include_html, html_file, exp_num_png_files, expected_file_func, actual_file_func) From 114266d1b740e52a9580b82f4e9493b4639cd6c9 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 27 Nov 2023 13:37:27 -0600 Subject: [PATCH 3/3] Add strict warning checks to tests --- polar2grid/add_coastlines.py | 10 +++---- polar2grid/filters/_utils.py | 7 +++-- polar2grid/tests/test_add_coastlines.py | 12 ++++---- polar2grid/tests/test_compare.py | 18 ++---------- polar2grid/tests/test_configs.py | 3 +- polar2grid/tests/test_glue.py | 4 ++- polar2grid/utils/legacy_compat.py | 21 -------------- polar2grid/utils/warnings.py | 38 +++++++++++++++++++++++++ polar2grid/writers/hdf5.py | 3 +- pyproject.toml | 11 ++++++- 10 files changed, 73 insertions(+), 54 deletions(-) create mode 100644 polar2grid/utils/warnings.py diff --git a/polar2grid/add_coastlines.py b/polar2grid/add_coastlines.py index 562893a0..60509863 100644 --- a/polar2grid/add_coastlines.py +++ b/polar2grid/add_coastlines.py @@ -34,12 +34,12 @@ import logging import os import sys +from pathlib import Path import numpy as np import rasterio from aggdraw import Font from PIL import Image, ImageFont -from pkg_resources import resource_filename as get_resource_filename from pycoast import ContourWriterAGG from pydecorate import DecoratorAGG from pyresample.utils import get_area_def_from_raster @@ -485,10 +485,10 @@ def find_font(font_name, size): font = ImageFont.truetype(font_name, size) return font.path except IOError as err: - font_path = get_resource_filename("polar2grid.fonts", font_name) - if not os.path.exists(font_path): - raise ValueError("Font path does not exist: {}".format(font_path)) from err - return font_path + font_path = Path(__file__).parent / "fonts" / font_name + if not font_path.is_file(): + raise ValueError(f"Font path does not exist: {font_path}") from err + return str(font_path) def _process_one_image( diff --git a/polar2grid/filters/_utils.py b/polar2grid/filters/_utils.py index b317fc29..682e6098 100644 --- a/polar2grid/filters/_utils.py +++ b/polar2grid/filters/_utils.py @@ -36,7 +36,7 @@ from typing import Union from pyresample.boundary import AreaBoundary, AreaDefBoundary, Boundary -from pyresample.geometry import AreaDefinition, SwathDefinition, get_geostationary_bounding_box +from pyresample.geometry import AreaDefinition, SwathDefinition, get_geostationary_bounding_box_in_lonlats from pyresample.spherical import SphPolygon logger = logging.getLogger(__name__) @@ -47,11 +47,12 @@ def boundary_for_area(area_def: PRGeometry) -> Boundary: """Create Boundary object representing the provided area.""" if getattr(area_def, "is_geostationary", False): - adp = Boundary(*get_geostationary_bounding_box(area_def, nb_points=100)) + adp = Boundary(*get_geostationary_bounding_box_in_lonlats(area_def, nb_points=100)) else: freq_fraction = 0.30 if isinstance(area_def, AreaDefinition) else 0.05 try: - adp = AreaDefBoundary(area_def, frequency=int(area_def.shape[0] * freq_fraction)) + adp = area_def.boundary() + adp.decimate(int(freq_fraction * area_def.shape[0])) except ValueError: if not isinstance(area_def, SwathDefinition): logger.error("Unable to generate bounding geolocation polygon") diff --git a/polar2grid/tests/test_add_coastlines.py b/polar2grid/tests/test_add_coastlines.py index 902b2b62..802156b0 100644 --- a/polar2grid/tests/test_add_coastlines.py +++ b/polar2grid/tests/test_add_coastlines.py @@ -171,8 +171,10 @@ def test_add_coastlines_basic( passed_cmap = add_scale_mock.call_args.kwargs["colormap"] _check_used_colormap(passed_cmap, has_colors, include_cmap_tag, include_scale_offset) - img = Image.open(output_fp) - arr = np.asarray(img) + with Image.open(output_fp) as img: + img.load() + arr = np.asarray(img) + out_tags = dict(img.tag_v2) if output_fp.endswith(".tif") else {} # bottom of the image is a colorbar image_arr = arr[:940] _check_exp_image_colors(image_arr, colormap, 0, has_colors) @@ -181,9 +183,9 @@ def test_add_coastlines_basic( assert (arr[940:] != 0).any() if output_fp.endswith(".tif"): - out_tags = dict(img.tag_v2) - in_img = Image.open(fp) - in_tags = dict(in_img.tag_v2) + with Image.open(fp) as in_img: + in_img.load() + in_tags = dict(in_img.tag_v2) assert len(out_tags) >= 14 for key, val in out_tags.items(): if key < 30000: diff --git a/polar2grid/tests/test_compare.py b/polar2grid/tests/test_compare.py index c65c2cce..0cb2aacd 100644 --- a/polar2grid/tests/test_compare.py +++ b/polar2grid/tests/test_compare.py @@ -21,14 +21,14 @@ # input into another program. # Documentation: http://www.ssec.wisc.edu/software/polar2grid/ """Tests for the compare.py script.""" -import contextlib import os -import warnings from glob import glob import numpy as np import pytest +from polar2grid.utils.warnings import ignore_no_georef + SHAPE1 = (200, 100) SHAPE2 = (200, 101) IMAGE1_L_UINT8_ZEROS = np.zeros(SHAPE1, dtype=np.uint8) @@ -44,20 +44,6 @@ IMAGE_LIST2 = [IMAGE1_L_UINT8_ZEROS, IMAGE2_L_UINT8_ZEROS, IMAGE4_RGB_UINT8_ZEROS] -@contextlib.contextmanager -def ignore_no_georef(): - """Wrap operations that we know will produce a rasterio geolocation warning.""" - from rasterio.errors import NotGeoreferencedWarning - - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - "Dataset has no geotransform", - NotGeoreferencedWarning, - ) - yield - - def _create_geotiffs(base_dir, img_data): import rasterio diff --git a/polar2grid/tests/test_configs.py b/polar2grid/tests/test_configs.py index ef57862d..02278387 100644 --- a/polar2grid/tests/test_configs.py +++ b/polar2grid/tests/test_configs.py @@ -35,9 +35,10 @@ def pytest_generate_tests(metafunc): """ if "yaml_config_file" in metafunc.fixturenames: - root_dir = os.path.join(os.path.dirname(__file__), "..", "..") + root_dir = os.path.join(os.path.dirname(__file__), "..") glob_pat = os.path.join(root_dir, "etc", "**", "*.yaml") p2g_yaml_files = sorted(glob(glob_pat, recursive=True)) + assert len(p2g_yaml_files) != 0 metafunc.parametrize("yaml_config_file", p2g_yaml_files) diff --git a/polar2grid/tests/test_glue.py b/polar2grid/tests/test_glue.py index 79550de3..a666ac7a 100644 --- a/polar2grid/tests/test_glue.py +++ b/polar2grid/tests/test_glue.py @@ -35,6 +35,7 @@ from satpy.tests.utils import CustomScheduler from polar2grid.utils.config import get_polar2grid_etc +from polar2grid.utils.warnings import ignore_no_georef @contextlib.contextmanager @@ -294,7 +295,8 @@ def test_viirs_sdr_scene(self, scene_fixture, product_names, num_outputs, extra_ args.extend(product_names) if extra_flags: args.extend(extra_flags) - ret = main(args) + with ignore_no_georef(): + ret = main(args) output_files = glob(str(chtmpdir / "*.tif")) assert len(output_files) == num_outputs assert ret == 0 diff --git a/polar2grid/utils/legacy_compat.py b/polar2grid/utils/legacy_compat.py index 80e581cc..2cd920bb 100644 --- a/polar2grid/utils/legacy_compat.py +++ b/polar2grid/utils/legacy_compat.py @@ -24,9 +24,7 @@ from __future__ import annotations -import contextlib import logging -import warnings from typing import Generator, Iterable, Optional, Union from satpy import DataID, DataQuery, Scene @@ -259,22 +257,3 @@ def get_sensor_alias(satpy_sensor): if len(new_sensor) == 1: return new_sensor.pop() return new_sensor - - -@contextlib.contextmanager -def ignore_pyproj_proj_warnings(): - """Wrap operations that we know will produce a PROJ.4 precision warning. - - Only to be used internally to Pyresample when we have no other choice but - to use PROJ.4 strings/dicts. For example, serialization to YAML or other - human-readable formats or testing the methods that produce the PROJ.4 - versions of the CRS. - - """ - with warnings.catch_warnings(): - warnings.filterwarnings( - "ignore", - "You will likely lose important projection information", - UserWarning, - ) - yield diff --git a/polar2grid/utils/warnings.py b/polar2grid/utils/warnings.py new file mode 100644 index 00000000..f6b5dae3 --- /dev/null +++ b/polar2grid/utils/warnings.py @@ -0,0 +1,38 @@ +"""Warnings or utilities for dealing with warnings.""" +from __future__ import annotations + +import contextlib +import warnings + + +@contextlib.contextmanager +def ignore_no_georef(): + """Wrap operations that we know will produce a rasterio geolocation warning.""" + from rasterio.errors import NotGeoreferencedWarning + + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + "Dataset has no geotransform", + NotGeoreferencedWarning, + ) + yield + + +@contextlib.contextmanager +def ignore_pyproj_proj_warnings(): + """Wrap operations that we know will produce a PROJ.4 precision warning. + + Only to be used internally to Pyresample when we have no other choice but + to use PROJ.4 strings/dicts. For example, serialization to YAML or other + human-readable formats or testing the methods that produce the PROJ.4 + versions of the CRS. + + """ + with warnings.catch_warnings(): + warnings.filterwarnings( + "ignore", + "You will likely lose important projection information", + UserWarning, + ) + yield diff --git a/polar2grid/writers/hdf5.py b/polar2grid/writers/hdf5.py index f93bd7c7..f892144b 100644 --- a/polar2grid/writers/hdf5.py +++ b/polar2grid/writers/hdf5.py @@ -52,7 +52,8 @@ from pyresample.geometry import SwathDefinition from satpy.writers import Writer, compute_writer_results, split_results -from polar2grid.utils.legacy_compat import convert_p2g_pattern_to_satpy, ignore_pyproj_proj_warnings +from polar2grid.utils.legacy_compat import convert_p2g_pattern_to_satpy +from polar2grid.utils.warnings import ignore_pyproj_proj_warnings from polar2grid.writers.geotiff import NUMPY_DTYPE_STRS, NumpyDtypeList, str_to_dtype LOG = logging.getLogger(__name__) diff --git a/pyproject.toml b/pyproject.toml index 8179a94b..cc6307f1 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -57,7 +57,16 @@ include-package-data = true find = {} [tool.pytest.ini_options] -minversion = 7.0 +minversion = 6.0 +addopts = ["-ra", "--showlocals", "--strict-markers", "--strict-config"] +xfail_strict = true +log_cli_level = "info" +testpaths = ["polar2grid/tests"] +filterwarnings = [ + "error", + "ignore:numpy.ndarray size changed, may indicate binary incompatibility:RuntimeWarning", + "ignore:The `frequency` argument is pending deprecation:PendingDeprecationWarning" +] [tool.coverage.run] relative_files = true