From ff6849d734c3abbc2d6a4c1d0c47ed50f4016048 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 8 Dec 2022 14:09:43 +0200 Subject: [PATCH 01/18] Add initial YAML for the VIIRS L2 Cloud Mask reader --- satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml | 53 +++++++++++++++++++ 1 file changed, 53 insertions(+) create mode 100644 satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml diff --git a/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml b/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml new file mode 100644 index 0000000000..382c59da02 --- /dev/null +++ b/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml @@ -0,0 +1,53 @@ +reader: + name: viirs_l2_cloud_mask_nc + short_name: VIIRS CSPP Cloud Mask + long_name: VIIRS CSPP Cloud Mask data in NetCDF4 format + description: VIIRS CSPP Cloud Mask reader + status: Nominal + supports_fsspec: false + reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader + sensors: [viirs] + # file pattern keys to sort files by with 'satpy.utils.group_files' + # by default, don't use start_time group files (only orbit and platform) + group_keys: ['orbit', 'platform_shortname'] + +file_types: + generic_file: + file_reader: !!python/name:satpy.readers.viirs_l2.VIIRSCloudMaskFileHandler + file_patterns: ['JRR-CloudMask_{delivery_package:4s}_{platform_shortname:3s}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}.nc'] + # Example filenames + # JRR-CloudMask_v3r0_npp_s202212070726217_e202212070727459_c202212071917430.nc + +datasets: + longitude: + name: longitude + resolution: 750 + file_type: generic_file + file_key: Longitude + file_units: "degrees_east" + standard_name: longitude + coordinates: [longitude, latitude] + latitude: + name: latitude + resolution: 750 + file_type: generic_file + file_key: Latiitude + file_units: "degrees_north" + standard_name: latitude + coordinates: [longitude, latitude] + cloudmask: + name: cloudmask + resolution: 750 + file_type: generic_file + file_key: CloudMask + file_units: "1" + standard_name: cloudmask + coordinates: [longitude, latitude] + cloudmask_binary: + name: cloudmask_binary + resolution: 750 + file_type: generic_file + file_key: CloudMaskBinary + file_units: "1" + standard_name: cloudmask_binary + coordinates: [longitude, latitude] From 2065920cdac7380f1d472a89597dae1477b27298 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 8 Dec 2022 16:11:45 +0200 Subject: [PATCH 02/18] Add file handler for VIIRS L2 cloud mask --- satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml | 31 ++-- satpy/readers/viirs_l2.py | 155 ++++++++++++++++++ 2 files changed, 169 insertions(+), 17 deletions(-) create mode 100644 satpy/readers/viirs_l2.py diff --git a/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml b/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml index 382c59da02..df3ba9c3b6 100644 --- a/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml +++ b/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml @@ -3,18 +3,15 @@ reader: short_name: VIIRS CSPP Cloud Mask long_name: VIIRS CSPP Cloud Mask data in NetCDF4 format description: VIIRS CSPP Cloud Mask reader - status: Nominal + status: alpha supports_fsspec: false reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader sensors: [viirs] - # file pattern keys to sort files by with 'satpy.utils.group_files' - # by default, don't use start_time group files (only orbit and platform) - group_keys: ['orbit', 'platform_shortname'] file_types: - generic_file: + cspp_cloud_mask_file: file_reader: !!python/name:satpy.readers.viirs_l2.VIIRSCloudMaskFileHandler - file_patterns: ['JRR-CloudMask_{delivery_package:4s}_{platform_shortname:3s}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_b{orbit:5d}_c{creation_time:%Y%m%d%H%M%S%f}.nc'] + file_patterns: ['JRR-CloudMask_{delivery_package:4s}_{platform_shortname:3s}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time:%Y%m%d%H%M%S%f}.nc'] # Example filenames # JRR-CloudMask_v3r0_npp_s202212070726217_e202212070727459_c202212071917430.nc @@ -22,7 +19,7 @@ datasets: longitude: name: longitude resolution: 750 - file_type: generic_file + file_type: cspp_cloud_mask_file file_key: Longitude file_units: "degrees_east" standard_name: longitude @@ -30,24 +27,24 @@ datasets: latitude: name: latitude resolution: 750 - file_type: generic_file - file_key: Latiitude + file_type: cspp_cloud_mask_file + file_key: Latitude file_units: "degrees_north" standard_name: latitude coordinates: [longitude, latitude] - cloudmask: - name: cloudmask + cloud_mask: + name: cloud_mask resolution: 750 - file_type: generic_file + file_type: cspp_cloud_mask_file file_key: CloudMask file_units: "1" - standard_name: cloudmask + standard_name: cloud_mask coordinates: [longitude, latitude] - cloudmask_binary: - name: cloudmask_binary + cloud_mask_binary: + name: cloud_mask_binary resolution: 750 - file_type: generic_file + file_type: cspp_cloud_mask_file file_key: CloudMaskBinary file_units: "1" - standard_name: cloudmask_binary + standard_name: cloud_mask_binary coordinates: [longitude, latitude] diff --git a/satpy/readers/viirs_l2.py b/satpy/readers/viirs_l2.py new file mode 100644 index 0000000000..db576a76d1 --- /dev/null +++ b/satpy/readers/viirs_l2.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2011-2019 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Interface to VIIRS L2 files.""" + +import logging +from datetime import datetime + +from satpy.readers.netcdf_utils import NetCDF4FileHandler + +LOG = logging.getLogger(__name__) + + +class VIIRSCloudMaskFileHandler(NetCDF4FileHandler): + """VIIRS L2 Cloud Mask reader.""" + + def __init__(self, filename, filename_info, filetype_info): + """Initialize the file handler.""" + super().__init__(filename, filename_info, filetype_info, cache_handle=True) + + def _parse_datetime(self, datestr): + """Parse datetime.""" + return datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ") + + @property + def start_orbit_number(self): + """Get start orbit number.""" + return int(self['/attr/start_orbit_number']) + + @property + def end_orbit_number(self): + """Get end orbit number.""" + return int(self['/attr/end_orbit_number']) + + @property + def platform_name(self): + """Get platform name.""" + try: + res = self.filename_info['platform_shortname'] + except KeyError: + res = 'Unknown' + + return { + 'npp': 'Suomi-NPP', + 'j01': 'NOAA-20', + 'j02': 'NOAA-21', + }.get(res, res) + + @property + def sensor_name(self): + """Get sensor name.""" + return self['/attr/instrument_name'].lower() + + def get_shape(self, ds_id, ds_info): + """Get shape.""" + return self.get(ds_id['name'] + '/shape', 1) + + @property + def start_time(self): + """Get start time.""" + return self._parse_datetime(self['/attr/time_coverage_start']) + + @property + def end_time(self): + """Get end time.""" + return self._parse_datetime(self['/attr/time_coverage_end']) + + def get_metadata(self, dataset_id, ds_info): + """Get metadata.""" + var_path = ds_info['file_key'] + shape = self.get_shape(dataset_id, ds_info) + file_units = self._get_dataset_file_units(ds_info, var_path) + + i = getattr(self[var_path], 'attrs', {}) + i.update(ds_info) + i.update(dataset_id.to_dict()) + i.update({ + "shape": shape, + "units": ds_info.get("units", file_units), + "file_units": file_units, + "platform_name": self.platform_name, + "sensor": self.sensor_name, + "start_orbit": self.start_orbit_number, + "end_orbit": self.end_orbit_number, + }) + i.update(dataset_id.to_dict()) + return i + + def _get_dataset_file_units(self, ds_info, var_path): + file_units = ds_info.get('file_units') + if file_units is None: + file_units = self.get(var_path + '/attr/units') + + return file_units + + def get_dataset(self, dataset_id, ds_info): + """Get dataset.""" + var_path = ds_info['file_key'] + metadata = self.get_metadata(dataset_id, ds_info) + + valid_min, valid_max = self._get_dataset_valid_range(var_path) + data = self[var_path] + data.attrs.update(metadata) + + if valid_min is not None and valid_max is not None: + data = data.where((data >= valid_min) & (data <= valid_max)) + + if isinstance(data.attrs.get('flag_meanings'), str): + data.attrs['flag_meanings'] = data.attrs['flag_meanings'].split(' ') + + # rename dimensions to correspond to satpy's 'y' and 'x' standard + if 'Rows' in data.dims: + data = data.rename({'Rows': 'y', 'Columns': 'x'}) + return data + + def _get_dataset_valid_range(self, var_path): + valid_range = self.get(var_path + '/attr/valid_range') + valid_min = valid_range[0] + valid_max = valid_range[1] + + return valid_min, valid_max + + def available_datasets(self, configured_datasets=None): + """Generate dataset info and their availablity. + + See + :meth:`satpy.readers.file_handlers.BaseFileHandler.available_datasets` + for details. + + """ + for is_avail, ds_info in (configured_datasets or []): + if is_avail is not None: + # some other file handler said it has this dataset + # we don't know any more information than the previous + # file handler so let's yield early + yield is_avail, ds_info + continue + ft_matches = self.file_type_matches(ds_info['file_type']) + var_path = ds_info['file_key'] + is_in_file = var_path in self + yield ft_matches and is_in_file, ds_info From f76be3189820e45a56162d47e2c45ad2d608cec6 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Wed, 14 Dec 2022 11:46:27 +0200 Subject: [PATCH 03/18] Add tests for VIIRS L2 cloud mask reader --- satpy/tests/reader_tests/test_viirs_l2.py | 155 ++++++++++++++++++++++ 1 file changed, 155 insertions(+) create mode 100644 satpy/tests/reader_tests/test_viirs_l2.py diff --git a/satpy/tests/reader_tests/test_viirs_l2.py b/satpy/tests/reader_tests/test_viirs_l2.py new file mode 100644 index 0000000000..12b79a4d7f --- /dev/null +++ b/satpy/tests/reader_tests/test_viirs_l2.py @@ -0,0 +1,155 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2022 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Tests for the VIIRS CSPP L2 readers.""" + +import os +import shutil +import tempfile + +import numpy as np +import xarray as xr + +from satpy import Scene + +CLOUD_MASK_FILE = "JRR-CloudMask_v3r0_npp_s202212070905565_e202212070907207_c202212071932513.nc" +NUM_COLUMNS = 3200 +NUM_ROWS = 768 +DATASETS = ['Latitude', 'Longitude', 'CloudMask', 'CloudMaskBinary'] + + +class JRRCloudMaskFile: + """Write a test JRR CloudMask file.""" + + def __init__(self): + """Initialize the class.""" + self._dir = tempfile.mkdtemp() + self.name = os.path.join(self._dir, CLOUD_MASK_FILE) + + def __enter__(self): + """Create the file when entering the context manager.""" + self._write_file() + return self.name + + def _write_file(self): + dset = xr.Dataset() + dset.attrs = _get_global_attrs() + dset['Latitude'] = _get_lat_arr() + dset['Longitude'] = _get_lon_arr() + dset['CloudMask'] = _get_cloud_mask_arr() + dset['CloudMaskBinary'] = _get_cloud_mask_binary_arr() + dset.to_netcdf(self.name, 'w') + + def __exit__(self, type, value, traceback): + """Delete the file when exiting the context manager.""" + shutil.rmtree(self._dir, ignore_errors=True) + + +def _get_global_attrs(): + return { + 'time_coverage_start': '2022-12-07T09:05:56Z', + 'time_coverage_end': '2022-12-07T09:07:20Z', + 'start_orbit_number': np.array(57573), + 'end_orbit_number': np.array(57573), + 'instrument_name': 'VIIRS', + } + + +def _get_lat_arr(): + arr = np.zeros((NUM_ROWS, NUM_COLUMNS), dtype=np.float32) + attrs = { + 'long_name': 'Latitude', + 'units': 'degrees_north', + 'valid_range': np.array([-90, 90], dtype=np.float32), + '_FillValue': -999. + } + return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns')) + + +def _get_lon_arr(): + arr = np.zeros((NUM_ROWS, NUM_COLUMNS), dtype=np.float32) + attrs = { + 'long_name': 'Longitude', + 'units': 'degrees_east', + 'valid_range': np.array([-180, 180], dtype=np.float32), + '_FillValue': -999. + } + return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns')) + + +def _get_cloud_mask_arr(): + arr = np.random.randint(0, 4, (NUM_ROWS, NUM_COLUMNS), dtype=np.byte) + attrs = { + 'long_name': 'Cloud Mask', + '_FillValue': np.byte(-128), + 'valid_range': np.array([0, 3], dtype=np.byte), + 'units': '1', + 'flag_values': np.array([0, 1, 2, 3], dtype=np.byte), + 'flag_meanings': 'clear probably_clear probably_cloudy cloudy', + } + return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns')) + + +def _get_cloud_mask_binary_arr(): + arr = np.random.randint(0, 2, (NUM_ROWS, NUM_COLUMNS), dtype=np.byte) + attrs = { + 'long_name': 'Cloud Mask Binary', + '_FillValue': np.byte(-128), + 'valid_range': np.array([0, 1], dtype=np.byte), + 'units': '1', + } + return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns')) + + +def test_cloud_mask_read_latitude(): + """Test reading latitude dataset.""" + with JRRCloudMaskFile() as fname: + data = _read_viirs_l2_cloud_mask_nc_data(fname, 'latitude') + _assert_common(data) + + +def test_cloud_mask_read_longitude(): + """Test reading longitude dataset.""" + with JRRCloudMaskFile() as fname: + data = _read_viirs_l2_cloud_mask_nc_data(fname, 'longitude') + _assert_common(data) + + +def test_cloud_mask_read_cloud_mask(): + """Test reading cloud mask dataset.""" + with JRRCloudMaskFile() as fname: + data = _read_viirs_l2_cloud_mask_nc_data(fname, 'cloud_mask') + _assert_common(data) + np.testing.assert_equal(data.attrs['flag_values'], [0, 1, 2, 3]) + assert data.attrs['flag_meanings'] == ['clear', 'probably_clear', 'probably_cloudy', 'cloudy'] + + +def test_cloud_mas_read_binary_cloud_mask(): + """Test reading binary cloud mask dataset.""" + with JRRCloudMaskFile() as fname: + data = _read_viirs_l2_cloud_mask_nc_data(fname, 'cloud_mask_binary') + _assert_common(data) + + +def _read_viirs_l2_cloud_mask_nc_data(fname, dset_name): + scn = Scene(reader="viirs_l2_cloud_mask_nc", filenames=[fname]) + scn.load([dset_name]) + return scn[dset_name] + + +def _assert_common(data): + assert data.dims == ('y', 'x') From c6c97a440f365dd63711138113dd0a53bc68c493 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 10 Jan 2023 12:51:12 +0200 Subject: [PATCH 04/18] Fix header text --- satpy/readers/viirs_l2.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/satpy/readers/viirs_l2.py b/satpy/readers/viirs_l2.py index db576a76d1..0fd315b20a 100644 --- a/satpy/readers/viirs_l2.py +++ b/satpy/readers/viirs_l2.py @@ -1,6 +1,4 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- -# Copyright (c) 2011-2019 Satpy developers +# Copyright (c) 2022-2023 Satpy developers # # This file is part of satpy. # From 0720a1f050ab5871575e3b61b5d395b9df717a8c Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 10 Jan 2023 12:52:41 +0200 Subject: [PATCH 05/18] Remove unused logging --- satpy/readers/viirs_l2.py | 3 --- 1 file changed, 3 deletions(-) diff --git a/satpy/readers/viirs_l2.py b/satpy/readers/viirs_l2.py index 0fd315b20a..536f62899d 100644 --- a/satpy/readers/viirs_l2.py +++ b/satpy/readers/viirs_l2.py @@ -15,13 +15,10 @@ # satpy. If not, see . """Interface to VIIRS L2 files.""" -import logging from datetime import datetime from satpy.readers.netcdf_utils import NetCDF4FileHandler -LOG = logging.getLogger(__name__) - class VIIRSCloudMaskFileHandler(NetCDF4FileHandler): """VIIRS L2 Cloud Mask reader.""" From 732ad514967f27ded2defa93db6625009ebcaac6 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 10 Jan 2023 12:55:24 +0200 Subject: [PATCH 06/18] Clarify variable naming --- satpy/readers/viirs_l2.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/satpy/readers/viirs_l2.py b/satpy/readers/viirs_l2.py index 536f62899d..2276fc612a 100644 --- a/satpy/readers/viirs_l2.py +++ b/satpy/readers/viirs_l2.py @@ -80,10 +80,10 @@ def get_metadata(self, dataset_id, ds_info): shape = self.get_shape(dataset_id, ds_info) file_units = self._get_dataset_file_units(ds_info, var_path) - i = getattr(self[var_path], 'attrs', {}) - i.update(ds_info) - i.update(dataset_id.to_dict()) - i.update({ + attr = getattr(self[var_path], 'attrs', {}) + attr.update(ds_info) + attr.update(dataset_id.to_dict()) + attr.update({ "shape": shape, "units": ds_info.get("units", file_units), "file_units": file_units, @@ -92,8 +92,8 @@ def get_metadata(self, dataset_id, ds_info): "start_orbit": self.start_orbit_number, "end_orbit": self.end_orbit_number, }) - i.update(dataset_id.to_dict()) - return i + attr.update(dataset_id.to_dict()) + return attr def _get_dataset_file_units(self, ds_info, var_path): file_units = ds_info.get('file_units') From 526b4549f6d20158f9deb92fc21668712c7cccd5 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 10 Jan 2023 12:58:03 +0200 Subject: [PATCH 07/18] Remove unnecessary header text --- satpy/tests/reader_tests/test_viirs_l2.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/satpy/tests/reader_tests/test_viirs_l2.py b/satpy/tests/reader_tests/test_viirs_l2.py index 12b79a4d7f..e8a4dc458c 100644 --- a/satpy/tests/reader_tests/test_viirs_l2.py +++ b/satpy/tests/reader_tests/test_viirs_l2.py @@ -1,5 +1,3 @@ -#!/usr/bin/env python -# -*- coding: utf-8 -*- # Copyright (c) 2022 Satpy developers # # This file is part of satpy. From c32ec3d22ea6c059b285052d6d1481e673fb9e44 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 10 Jan 2023 13:15:12 +0200 Subject: [PATCH 08/18] Remove unnecessary unit handling code, rely on getting the units from reader YAML --- satpy/readers/viirs_l2.py | 9 +-------- 1 file changed, 1 insertion(+), 8 deletions(-) diff --git a/satpy/readers/viirs_l2.py b/satpy/readers/viirs_l2.py index 2276fc612a..445b92bd55 100644 --- a/satpy/readers/viirs_l2.py +++ b/satpy/readers/viirs_l2.py @@ -78,7 +78,7 @@ def get_metadata(self, dataset_id, ds_info): """Get metadata.""" var_path = ds_info['file_key'] shape = self.get_shape(dataset_id, ds_info) - file_units = self._get_dataset_file_units(ds_info, var_path) + file_units = ds_info.get('file_units') attr = getattr(self[var_path], 'attrs', {}) attr.update(ds_info) @@ -95,13 +95,6 @@ def get_metadata(self, dataset_id, ds_info): attr.update(dataset_id.to_dict()) return attr - def _get_dataset_file_units(self, ds_info, var_path): - file_units = ds_info.get('file_units') - if file_units is None: - file_units = self.get(var_path + '/attr/units') - - return file_units - def get_dataset(self, dataset_id, ds_info): """Get dataset.""" var_path = ds_info['file_key'] From 6e41d2361793fe755dab79c456b8c93fc016b913 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 10 Jan 2023 13:18:45 +0200 Subject: [PATCH 09/18] Check that units are in the dataset attributes --- satpy/tests/reader_tests/test_viirs_l2.py | 1 + 1 file changed, 1 insertion(+) diff --git a/satpy/tests/reader_tests/test_viirs_l2.py b/satpy/tests/reader_tests/test_viirs_l2.py index e8a4dc458c..4ce16fd6e4 100644 --- a/satpy/tests/reader_tests/test_viirs_l2.py +++ b/satpy/tests/reader_tests/test_viirs_l2.py @@ -151,3 +151,4 @@ def _read_viirs_l2_cloud_mask_nc_data(fname, dset_name): def _assert_common(data): assert data.dims == ('y', 'x') + assert "units" in data.attrs From 23b0a419a8fe58bec1e439081005caae15083105 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Tue, 10 Jan 2023 16:48:50 +0200 Subject: [PATCH 10/18] Change reader status from alpha to beta --- satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml b/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml index df3ba9c3b6..0f2650bdc1 100644 --- a/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml +++ b/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml @@ -3,7 +3,7 @@ reader: short_name: VIIRS CSPP Cloud Mask long_name: VIIRS CSPP Cloud Mask data in NetCDF4 format description: VIIRS CSPP Cloud Mask reader - status: alpha + status: beta supports_fsspec: false reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader sensors: [viirs] From 07457d81e72ba00bf61128ae2bd49da1a05e265a Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 12 Jan 2023 13:22:58 +0200 Subject: [PATCH 11/18] Use fixtures for writing the temporary files --- satpy/tests/reader_tests/test_viirs_l2.py | 74 +++++++++-------------- 1 file changed, 29 insertions(+), 45 deletions(-) diff --git a/satpy/tests/reader_tests/test_viirs_l2.py b/satpy/tests/reader_tests/test_viirs_l2.py index 4ce16fd6e4..d01618fe5f 100644 --- a/satpy/tests/reader_tests/test_viirs_l2.py +++ b/satpy/tests/reader_tests/test_viirs_l2.py @@ -15,11 +15,8 @@ # satpy. If not, see . """Tests for the VIIRS CSPP L2 readers.""" -import os -import shutil -import tempfile - import numpy as np +import pytest import xarray as xr from satpy import Scene @@ -30,31 +27,22 @@ DATASETS = ['Latitude', 'Longitude', 'CloudMask', 'CloudMaskBinary'] -class JRRCloudMaskFile: - """Write a test JRR CloudMask file.""" - - def __init__(self): - """Initialize the class.""" - self._dir = tempfile.mkdtemp() - self.name = os.path.join(self._dir, CLOUD_MASK_FILE) - - def __enter__(self): - """Create the file when entering the context manager.""" - self._write_file() - return self.name +@pytest.fixture +def cloud_mask_file(tmp_path): + """Create a temporary JRR CloudMask file as a fixture.""" + file_path = tmp_path / CLOUD_MASK_FILE + _write_cloud_mask_file(file_path) + yield file_path - def _write_file(self): - dset = xr.Dataset() - dset.attrs = _get_global_attrs() - dset['Latitude'] = _get_lat_arr() - dset['Longitude'] = _get_lon_arr() - dset['CloudMask'] = _get_cloud_mask_arr() - dset['CloudMaskBinary'] = _get_cloud_mask_binary_arr() - dset.to_netcdf(self.name, 'w') - def __exit__(self, type, value, traceback): - """Delete the file when exiting the context manager.""" - shutil.rmtree(self._dir, ignore_errors=True) +def _write_cloud_mask_file(file_path): + dset = xr.Dataset() + dset.attrs = _get_global_attrs() + dset['Latitude'] = _get_lat_arr() + dset['Longitude'] = _get_lon_arr() + dset['CloudMask'] = _get_cloud_mask_arr() + dset['CloudMaskBinary'] = _get_cloud_mask_binary_arr() + dset.to_netcdf(file_path, 'w') def _get_global_attrs(): @@ -113,34 +101,30 @@ def _get_cloud_mask_binary_arr(): return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns')) -def test_cloud_mask_read_latitude(): +def test_cloud_mask_read_latitude(cloud_mask_file): """Test reading latitude dataset.""" - with JRRCloudMaskFile() as fname: - data = _read_viirs_l2_cloud_mask_nc_data(fname, 'latitude') - _assert_common(data) + data = _read_viirs_l2_cloud_mask_nc_data(cloud_mask_file, 'latitude') + _assert_common(data) -def test_cloud_mask_read_longitude(): +def test_cloud_mask_read_longitude(cloud_mask_file): """Test reading longitude dataset.""" - with JRRCloudMaskFile() as fname: - data = _read_viirs_l2_cloud_mask_nc_data(fname, 'longitude') - _assert_common(data) + data = _read_viirs_l2_cloud_mask_nc_data(cloud_mask_file, 'longitude') + _assert_common(data) -def test_cloud_mask_read_cloud_mask(): +def test_cloud_mask_read_cloud_mask(cloud_mask_file): """Test reading cloud mask dataset.""" - with JRRCloudMaskFile() as fname: - data = _read_viirs_l2_cloud_mask_nc_data(fname, 'cloud_mask') - _assert_common(data) - np.testing.assert_equal(data.attrs['flag_values'], [0, 1, 2, 3]) - assert data.attrs['flag_meanings'] == ['clear', 'probably_clear', 'probably_cloudy', 'cloudy'] + data = _read_viirs_l2_cloud_mask_nc_data(cloud_mask_file, 'cloud_mask') + _assert_common(data) + np.testing.assert_equal(data.attrs['flag_values'], [0, 1, 2, 3]) + assert data.attrs['flag_meanings'] == ['clear', 'probably_clear', 'probably_cloudy', 'cloudy'] -def test_cloud_mas_read_binary_cloud_mask(): +def test_cloud_mas_read_binary_cloud_mask(cloud_mask_file): """Test reading binary cloud mask dataset.""" - with JRRCloudMaskFile() as fname: - data = _read_viirs_l2_cloud_mask_nc_data(fname, 'cloud_mask_binary') - _assert_common(data) + data = _read_viirs_l2_cloud_mask_nc_data(cloud_mask_file, 'cloud_mask_binary') + _assert_common(data) def _read_viirs_l2_cloud_mask_nc_data(fname, dset_name): From 97847c6df067f179130370c2b805ef868d1a9a15 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 12 Jan 2023 13:23:18 +0200 Subject: [PATCH 12/18] Fix passing pathlib.Path as filename --- satpy/utils.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/satpy/utils.py b/satpy/utils.py index 267f796e8e..1794fead4c 100644 --- a/satpy/utils.py +++ b/satpy/utils.py @@ -24,6 +24,7 @@ import datetime import logging import os +import pathlib import warnings from contextlib import contextmanager from typing import Mapping, Optional @@ -636,6 +637,8 @@ def _sort_files_to_local_remote_and_fsfiles(filenames): for f in filenames: if isinstance(f, FSFile): fs_files.append(f) + elif isinstance(f, pathlib.Path): + local_files.append(f) elif urlparse(f).scheme in ('', 'file') or "\\" in f: local_files.append(f) else: From 5830aa2edaa43a77f18c9d2043a96be3dd45b1a3 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 12 Jan 2023 13:24:27 +0200 Subject: [PATCH 13/18] Add a note listing which external fixtures are used --- satpy/tests/reader_tests/test_viirs_l2.py | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/satpy/tests/reader_tests/test_viirs_l2.py b/satpy/tests/reader_tests/test_viirs_l2.py index d01618fe5f..87463507de 100644 --- a/satpy/tests/reader_tests/test_viirs_l2.py +++ b/satpy/tests/reader_tests/test_viirs_l2.py @@ -15,6 +15,10 @@ # satpy. If not, see . """Tests for the VIIRS CSPP L2 readers.""" +# NOTE: +# The following Pytest fixtures are not defined in this file, but are used and injected by Pytest: +# - tmp_path + import numpy as np import pytest import xarray as xr From 962580b4e4b3c87399e85f6229ea21daa34faf0d Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 12 Jan 2023 13:34:31 +0200 Subject: [PATCH 14/18] Remove unused import --- satpy/utils.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/utils.py b/satpy/utils.py index b74e7bc54f..920f4c3158 100644 --- a/satpy/utils.py +++ b/satpy/utils.py @@ -21,7 +21,6 @@ import contextlib import datetime import logging -import os import pathlib import warnings from contextlib import contextmanager From e8fcaaf9a1ad3e36bd7634ba871eac3a98066cfe Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 12 Jan 2023 15:06:24 +0200 Subject: [PATCH 15/18] Remove unnecessary try/except in getting platform short_name --- satpy/readers/viirs_l2.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/satpy/readers/viirs_l2.py b/satpy/readers/viirs_l2.py index 445b92bd55..12a7960a1b 100644 --- a/satpy/readers/viirs_l2.py +++ b/satpy/readers/viirs_l2.py @@ -44,10 +44,7 @@ def end_orbit_number(self): @property def platform_name(self): """Get platform name.""" - try: - res = self.filename_info['platform_shortname'] - except KeyError: - res = 'Unknown' + res = self.filename_info['platform_shortname'] return { 'npp': 'Suomi-NPP', From de481452c1daaae1899b92c2f9a21e551dfde180 Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 12 Jan 2023 15:28:17 +0200 Subject: [PATCH 16/18] Remove the unnecessary available_datasets() method --- satpy/readers/viirs_l2.py | 20 -------------------- 1 file changed, 20 deletions(-) diff --git a/satpy/readers/viirs_l2.py b/satpy/readers/viirs_l2.py index 12a7960a1b..90d272504a 100644 --- a/satpy/readers/viirs_l2.py +++ b/satpy/readers/viirs_l2.py @@ -118,23 +118,3 @@ def _get_dataset_valid_range(self, var_path): valid_max = valid_range[1] return valid_min, valid_max - - def available_datasets(self, configured_datasets=None): - """Generate dataset info and their availablity. - - See - :meth:`satpy.readers.file_handlers.BaseFileHandler.available_datasets` - for details. - - """ - for is_avail, ds_info in (configured_datasets or []): - if is_avail is not None: - # some other file handler said it has this dataset - # we don't know any more information than the previous - # file handler so let's yield early - yield is_avail, ds_info - continue - ft_matches = self.file_type_matches(ds_info['file_type']) - var_path = ds_info['file_key'] - is_in_file = var_path in self - yield ft_matches and is_in_file, ds_info From 596aa59660491a74eae65521b4fcd0d0ab6f890c Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 12 Jan 2023 15:52:54 +0200 Subject: [PATCH 17/18] Add a test for pathlib filenames --- satpy/tests/test_utils.py | 14 ++++++++++++++ 1 file changed, 14 insertions(+) diff --git a/satpy/tests/test_utils.py b/satpy/tests/test_utils.py index 741a5d9196..d0785e7a13 100644 --- a/satpy/tests/test_utils.py +++ b/satpy/tests/test_utils.py @@ -498,6 +498,20 @@ def test_convert_remote_files_to_fsspec_local_files(): assert res == filenames +def test_convert_remote_files_to_fsspec_local_pathlib_files(): + """Test convertion of remote files to fsspec objects. + + Case using pathlib objects as filenames. + """ + import pathlib + + from satpy.utils import convert_remote_files_to_fsspec + + filenames = [pathlib.Path("/tmp/file1.nc"), pathlib.Path("c:\tmp\file2.nc")] + res = convert_remote_files_to_fsspec(filenames) + assert res == filenames + + def test_convert_remote_files_to_fsspec_mixed_sources(): """Test convertion of remote files to fsspec objects. From 899e426b37c8fdb9e0516115bf07b1438433b4fc Mon Sep 17 00:00:00 2001 From: Panu Lahtinen Date: Thu, 12 Jan 2023 17:06:40 +0200 Subject: [PATCH 18/18] Fix ColormapCompositor docstring --- satpy/composites/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/composites/__init__.py b/satpy/composites/__init__.py index 1f3c2af8f8..c53b658744 100644 --- a/satpy/composites/__init__.py +++ b/satpy/composites/__init__.py @@ -536,7 +536,7 @@ class ColormapCompositor(GenericCompositor): - ctth_alti tandard_name: cloud_top_height - and the enhancement: + and the enhancement:: cloud_top_height: standard_name: cloud_top_height