VIIRS L2 Cloud Mask reader

satpy/composites/__init__.py

@@ -536,7 +536,7 @@ class ColormapCompositor(GenericCompositor):
         - ctth_alti
         tandard_name: cloud_top_height
 
-    and the enhancement:
+    and the enhancement::
 
       cloud_top_height:
         standard_name: cloud_top_height

satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml

@@ -0,0 +1,50 @@
+reader:
+  name: viirs_l2_cloud_mask_nc
+  short_name: VIIRS CSPP Cloud Mask
+  long_name: VIIRS CSPP Cloud Mask data in NetCDF4 format
+  description: VIIRS CSPP Cloud Mask reader
+  status: beta
+  supports_fsspec: false
+  reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader
+  sensors: [viirs]
+
+file_types:
+  cspp_cloud_mask_file:
+    file_reader: !!python/name:satpy.readers.viirs_l2.VIIRSCloudMaskFileHandler
+    file_patterns: ['JRR-CloudMask_{delivery_package:4s}_{platform_shortname:3s}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time:%Y%m%d%H%M%S%f}.nc']
+    # Example filenames
+    # JRR-CloudMask_v3r0_npp_s202212070726217_e202212070727459_c202212071917430.nc
+
+datasets:
+  longitude:
+    name: longitude
+    resolution: 750
+    file_type: cspp_cloud_mask_file
+    file_key: Longitude
+    file_units: "degrees_east"
+    standard_name: longitude
+    coordinates: [longitude, latitude]
+  latitude:
+    name: latitude
+    resolution: 750
+    file_type: cspp_cloud_mask_file
+    file_key: Latitude
+    file_units: "degrees_north"
+    standard_name: latitude
+    coordinates: [longitude, latitude]
+  cloud_mask:
+    name: cloud_mask
+    resolution: 750
+    file_type: cspp_cloud_mask_file
+    file_key: CloudMask
+    file_units: "1"
+    standard_name: cloud_mask
+    coordinates: [longitude, latitude]
+  cloud_mask_binary:
+    name: cloud_mask_binary
+    resolution: 750
+    file_type: cspp_cloud_mask_file
+    file_key: CloudMaskBinary
+    file_units: "1"
+    standard_name: cloud_mask_binary
+    coordinates: [longitude, latitude]

satpy/readers/viirs_l2.py

@@ -0,0 +1,120 @@
+# Copyright (c) 2022-2023 Satpy developers
+#
+# This file is part of satpy.
+#
+# satpy is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# satpy.  If not, see <http://www.gnu.org/licenses/>.
+"""Interface to VIIRS L2 files."""
+
+from datetime import datetime
+
+from satpy.readers.netcdf_utils import NetCDF4FileHandler
+
+
+class VIIRSCloudMaskFileHandler(NetCDF4FileHandler):
+    """VIIRS L2 Cloud Mask reader."""
+
+    def __init__(self, filename, filename_info, filetype_info):
+        """Initialize the file handler."""
+        super().__init__(filename, filename_info, filetype_info, cache_handle=True)
+
+    def _parse_datetime(self, datestr):
+        """Parse datetime."""
+        return datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ")
+
+    @property
+    def start_orbit_number(self):
+        """Get start orbit number."""
+        return int(self['/attr/start_orbit_number'])
+
+    @property
+    def end_orbit_number(self):
+        """Get end orbit number."""
+        return int(self['/attr/end_orbit_number'])
+
+    @property
+    def platform_name(self):
+        """Get platform name."""
+        res = self.filename_info['platform_shortname']
+
+        return {
+            'npp': 'Suomi-NPP',
+            'j01': 'NOAA-20',
+            'j02': 'NOAA-21',
+        }.get(res, res)
+
+    @property
+    def sensor_name(self):
+        """Get sensor name."""
+        return self['/attr/instrument_name'].lower()
+
+    def get_shape(self, ds_id, ds_info):
+        """Get shape."""
+        return self.get(ds_id['name'] + '/shape', 1)
+
+    @property
+    def start_time(self):
+        """Get start time."""
+        return self._parse_datetime(self['/attr/time_coverage_start'])
+
+    @property
+    def end_time(self):
+        """Get end time."""
+        return self._parse_datetime(self['/attr/time_coverage_end'])
+
+    def get_metadata(self, dataset_id, ds_info):
+        """Get metadata."""
+        var_path = ds_info['file_key']
+        shape = self.get_shape(dataset_id, ds_info)
+        file_units = ds_info.get('file_units')
+
+        attr = getattr(self[var_path], 'attrs', {})
+        attr.update(ds_info)
+        attr.update(dataset_id.to_dict())
+        attr.update({
+            "shape": shape,
+            "units": ds_info.get("units", file_units),
+            "file_units": file_units,
+            "platform_name": self.platform_name,
+            "sensor": self.sensor_name,
+            "start_orbit": self.start_orbit_number,
+            "end_orbit": self.end_orbit_number,
+        })
+        attr.update(dataset_id.to_dict())
+        return attr
+
+    def get_dataset(self, dataset_id, ds_info):
+        """Get dataset."""
+        var_path = ds_info['file_key']
+        metadata = self.get_metadata(dataset_id, ds_info)
+
+        valid_min, valid_max = self._get_dataset_valid_range(var_path)
+        data = self[var_path]
+        data.attrs.update(metadata)
+
+        if valid_min is not None and valid_max is not None:
+            data = data.where((data >= valid_min) & (data <= valid_max))
+
+        if isinstance(data.attrs.get('flag_meanings'), str):
+            data.attrs['flag_meanings'] = data.attrs['flag_meanings'].split(' ')
+
+        # rename dimensions to correspond to satpy's 'y' and 'x' standard
+        if 'Rows' in data.dims:
+            data = data.rename({'Rows': 'y', 'Columns': 'x'})
+        return data
+
+    def _get_dataset_valid_range(self, var_path):
+        valid_range = self.get(var_path + '/attr/valid_range')
+        valid_min = valid_range[0]
+        valid_max = valid_range[1]
+
+        return valid_min, valid_max

satpy/tests/reader_tests/test_viirs_l2.py

@@ -0,0 +1,142 @@
+# Copyright (c) 2022 Satpy developers
+#
+# This file is part of satpy.
+#
+# satpy is free software: you can redistribute it and/or modify it under the
+# terms of the GNU General Public License as published by the Free Software
+# Foundation, either version 3 of the License, or (at your option) any later
+# version.
+#
+# satpy is distributed in the hope that it will be useful, but WITHOUT ANY
+# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR
+# A PARTICULAR PURPOSE.  See the GNU General Public License for more details.
+#
+# You should have received a copy of the GNU General Public License along with
+# satpy.  If not, see <http://www.gnu.org/licenses/>.
+"""Tests for the VIIRS CSPP L2 readers."""
+
+# NOTE:
+# The following Pytest fixtures are not defined in this file, but are used and injected by Pytest:
+# - tmp_path
+
+import numpy as np
+import pytest
+import xarray as xr
+
+from satpy import Scene
+
+CLOUD_MASK_FILE = "JRR-CloudMask_v3r0_npp_s202212070905565_e202212070907207_c202212071932513.nc"
+NUM_COLUMNS = 3200
+NUM_ROWS = 768
+DATASETS = ['Latitude', 'Longitude', 'CloudMask', 'CloudMaskBinary']
+
+
+@pytest.fixture
+def cloud_mask_file(tmp_path):
+    """Create a temporary JRR CloudMask file as a fixture."""
+    file_path = tmp_path / CLOUD_MASK_FILE
+    _write_cloud_mask_file(file_path)
+    yield file_path
+
+
+def _write_cloud_mask_file(file_path):
+    dset = xr.Dataset()
+    dset.attrs = _get_global_attrs()
+    dset['Latitude'] = _get_lat_arr()
+    dset['Longitude'] = _get_lon_arr()
+    dset['CloudMask'] = _get_cloud_mask_arr()
+    dset['CloudMaskBinary'] = _get_cloud_mask_binary_arr()
+    dset.to_netcdf(file_path, 'w')
+
+
+def _get_global_attrs():
+    return {
+        'time_coverage_start': '2022-12-07T09:05:56Z',
+        'time_coverage_end': '2022-12-07T09:07:20Z',
+        'start_orbit_number': np.array(57573),
+        'end_orbit_number': np.array(57573),
+        'instrument_name': 'VIIRS',
+    }
+
+
+def _get_lat_arr():
+    arr = np.zeros((NUM_ROWS, NUM_COLUMNS), dtype=np.float32)
+    attrs = {
+        'long_name': 'Latitude',
+        'units': 'degrees_north',
+        'valid_range': np.array([-90, 90], dtype=np.float32),
+        '_FillValue': -999.
+    }
+    return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns'))
+
+
+def _get_lon_arr():
+    arr = np.zeros((NUM_ROWS, NUM_COLUMNS), dtype=np.float32)
+    attrs = {
+        'long_name': 'Longitude',
+        'units': 'degrees_east',
+        'valid_range': np.array([-180, 180], dtype=np.float32),
+        '_FillValue': -999.
+    }
+    return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns'))
+
+
+def _get_cloud_mask_arr():
+    arr = np.random.randint(0, 4, (NUM_ROWS, NUM_COLUMNS), dtype=np.byte)
+    attrs = {
+        'long_name': 'Cloud Mask',
+        '_FillValue': np.byte(-128),
+        'valid_range': np.array([0, 3], dtype=np.byte),
+        'units': '1',
+        'flag_values': np.array([0, 1, 2, 3], dtype=np.byte),
+        'flag_meanings': 'clear probably_clear probably_cloudy cloudy',
+    }
+    return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns'))
+
+
+def _get_cloud_mask_binary_arr():
+    arr = np.random.randint(0, 2, (NUM_ROWS, NUM_COLUMNS), dtype=np.byte)
+    attrs = {
+        'long_name': 'Cloud Mask Binary',
+        '_FillValue': np.byte(-128),
+        'valid_range': np.array([0, 1], dtype=np.byte),
+        'units': '1',
+    }
+    return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns'))
+
+
+def test_cloud_mask_read_latitude(cloud_mask_file):
+    """Test reading latitude dataset."""
+    data = _read_viirs_l2_cloud_mask_nc_data(cloud_mask_file, 'latitude')
+    _assert_common(data)
+
+
+def test_cloud_mask_read_longitude(cloud_mask_file):
+    """Test reading longitude dataset."""
+    data = _read_viirs_l2_cloud_mask_nc_data(cloud_mask_file, 'longitude')
+    _assert_common(data)
+
+
+def test_cloud_mask_read_cloud_mask(cloud_mask_file):
+    """Test reading cloud mask dataset."""
+    data = _read_viirs_l2_cloud_mask_nc_data(cloud_mask_file, 'cloud_mask')
+    _assert_common(data)
+    np.testing.assert_equal(data.attrs['flag_values'], [0, 1, 2, 3])
+    assert data.attrs['flag_meanings'] == ['clear', 'probably_clear', 'probably_cloudy', 'cloudy']
+
+
+def test_cloud_mas_read_binary_cloud_mask(cloud_mask_file):
+    """Test reading binary cloud mask dataset."""
+    data = _read_viirs_l2_cloud_mask_nc_data(cloud_mask_file, 'cloud_mask_binary')
+    _assert_common(data)
+
+
+def _read_viirs_l2_cloud_mask_nc_data(fname, dset_name):
+    scn = Scene(reader="viirs_l2_cloud_mask_nc", filenames=[fname])
+    scn.load([dset_name])
+    return scn[dset_name]
+
+
+def _assert_common(data):
+    assert data.dims == ('y', 'x')
+    assert "units" in data.attrs

satpy/tests/test_utils.py

@@ -498,6 +498,20 @@ def test_convert_remote_files_to_fsspec_local_files():
     assert res == filenames
 
 
+def test_convert_remote_files_to_fsspec_local_pathlib_files():
+    """Test convertion of remote files to fsspec objects.
+
+    Case using pathlib objects as filenames.
+    """
+    import pathlib
+
+    from satpy.utils import convert_remote_files_to_fsspec
+
+    filenames = [pathlib.Path("/tmp/file1.nc"), pathlib.Path("c:\tmp\file2.nc")]
+    res = convert_remote_files_to_fsspec(filenames)
+    assert res == filenames
+
+
 def test_convert_remote_files_to_fsspec_mixed_sources():
     """Test convertion of remote files to fsspec objects.
 

satpy/utils.py

@@ -21,6 +21,7 @@
 import contextlib
 import datetime
 import logging
+import pathlib
 import warnings
 from contextlib import contextmanager
 from copy import deepcopy
@@ -627,6 +628,8 @@ def _sort_files_to_local_remote_and_fsfiles(filenames):
     for f in filenames:
         if isinstance(f, FSFile):
             fs_files.append(f)
+        elif isinstance(f, pathlib.Path):
+            local_files.append(f)
         elif urlparse(f).scheme in ('', 'file') or "\\" in f:
             local_files.append(f)
         else: