From 4f080e4344a023d12ef7c9a985c1362a027f54d2 Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 11:19:17 +0000 Subject: [PATCH 01/59] Add reader for VIIRS level 2 products produced by the NOAA enterprise suite. --- satpy/etc/readers/viirs_jrr.yaml | 37 ++++++++++++++ satpy/readers/viirs_jrr.py | 85 ++++++++++++++++++++++++++++++++ 2 files changed, 122 insertions(+) create mode 100644 satpy/etc/readers/viirs_jrr.yaml create mode 100644 satpy/readers/viirs_jrr.py diff --git a/satpy/etc/readers/viirs_jrr.yaml b/satpy/etc/readers/viirs_jrr.yaml new file mode 100644 index 0000000000..fe896c1cf8 --- /dev/null +++ b/satpy/etc/readers/viirs_jrr.yaml @@ -0,0 +1,37 @@ +reader: + description: VIIRS NOAA Enterprise L2 product reader + name: viirs_jrr + reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader + sensors: [viirs] + + +file_types: + jrr_cloudmask: + file_reader: !!python/name:satpy.readers.viirs_jrr.VIIRSJRRFileHandler + variable_prefix: "" + file_patterns: + - 'JRR-CloudMask_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' + + +datasets: + longitude: + name: longitude + standard_name: longitude + file_type: [jrr_cloudmask] + file_key: "Longitude" + units: 'degrees_east' + latitude: + name: latitude + standard_name: latitude + file_type: [jrr_cloudmask] + file_key: "Latitude" + units: 'degrees_north' + cloud_mask: + name: cloud_mask + file_type: [jrr_cloudmask] + file_key: "CloudMask" + coordinates: [longitude, latitude] + units: '1' + flag_meanings: ['Clear', 'Probably Clear', 'Probably Cloudy', 'Cloudy'] + flag_values: [0, 1, 2, 3] + _FillValue: -128 \ No newline at end of file diff --git a/satpy/readers/viirs_jrr.py b/satpy/readers/viirs_jrr.py new file mode 100644 index 0000000000..2ae344bd95 --- /dev/null +++ b/satpy/readers/viirs_jrr.py @@ -0,0 +1,85 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2022 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""VIIRS NOAA enterprise L2 product reader. + +This module implements readers for the NOAA enterprise level 2 products for the +VIIRS instrument. These replace the 'old' EDR products. +""" + +import logging + +from satpy import CHUNK_SIZE +from satpy.readers.file_handlers import BaseFileHandler + +import dask.array as da +import numpy as np +import xarray as xr + +# map platform attributes to Oscar standard name +PLATFORM_MAP = { + "NPP": "Suomi-NPP", + "J01": "NOAA-20", + "J02": "NOAA-21" +} + +LOG = logging.getLogger(__name__) + + +class VIIRSJRRFileHandler(BaseFileHandler): + """NetCDF4 reader for VIIRS Active Fires.""" + + def __init__(self, filename, filename_info, filetype_info): + """Initialize the geo filehandler.""" + super(VIIRSJRRFileHandler, self).__init__(filename, filename_info, + filetype_info) + self.nc = xr.open_dataset(self.filename, + decode_cf=True, + mask_and_scale=True, + chunks={'Columns': CHUNK_SIZE, + 'Rows': CHUNK_SIZE}) + self.nc = self.nc.rename({'Columns': 'x', 'Rows': 'y'}) + if 'Latitude' in self.nc: + self.nc['Latitude'].attrs.update({'standard_name': 'latitude'}) + if 'Longitude' in self.nc: + self.nc['Longitude'].attrs.update({'standard_name': 'longitude'}) + + def get_dataset(self, dataset_id, info): + """Get the dataset.""" + ds = self.nc[info['file_key']] + + return ds + + @property + def start_time(self): + """Get first date/time when observations were recorded.""" + return self.filename_info['start_time'] + + @property + def end_time(self): + """Get last date/time when observations were recorded.""" + return self.filename_info.get('end_time', self.start_time) + + @property + def sensor_name(self): + """Name of sensor for this file.""" + return self["sensor"] + + @property + def platform_name(self): + """Name of platform/satellite for this file.""" + return self["platform_name"] From 439cbdfbc6f60a20f4c1cf8650406d7a23238aeb Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 11:28:38 +0000 Subject: [PATCH 02/59] Add reader for VIIRS level 2 products produced by the NOAA enterprise suite. --- satpy/readers/viirs_jrr.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) diff --git a/satpy/readers/viirs_jrr.py b/satpy/readers/viirs_jrr.py index 2ae344bd95..0704834da4 100644 --- a/satpy/readers/viirs_jrr.py +++ b/satpy/readers/viirs_jrr.py @@ -34,7 +34,7 @@ PLATFORM_MAP = { "NPP": "Suomi-NPP", "J01": "NOAA-20", - "J02": "NOAA-21" + "J02": "NOAA-21", } LOG = logging.getLogger(__name__) @@ -53,11 +53,16 @@ def __init__(self, filename, filename_info, filetype_info): chunks={'Columns': CHUNK_SIZE, 'Rows': CHUNK_SIZE}) self.nc = self.nc.rename({'Columns': 'x', 'Rows': 'y'}) + + # For some reason, no 'standard_name' is defined in the netCDF files, so + # here we manually make the definitions. if 'Latitude' in self.nc: self.nc['Latitude'].attrs.update({'standard_name': 'latitude'}) if 'Longitude' in self.nc: self.nc['Longitude'].attrs.update({'standard_name': 'longitude'}) + self.algorithm_version = filename_info['platform_shortname'] + def get_dataset(self, dataset_id, info): """Get the dataset.""" ds = self.nc[info['file_key']] @@ -74,12 +79,13 @@ def end_time(self): """Get last date/time when observations were recorded.""" return self.filename_info.get('end_time', self.start_time) - @property - def sensor_name(self): - """Name of sensor for this file.""" - return self["sensor"] - @property def platform_name(self): - """Name of platform/satellite for this file.""" - return self["platform_name"] + """Get platform name.""" + platform_path = self.filetype_info['platform_name'] + platform_dict = {'NPP': 'Suomi-NPP', + 'JPSS-1': 'NOAA-20', + 'J01': 'NOAA-20', + 'JPSS-2': 'NOAA-21', + 'J02': 'NOAA-21'} + return platform_dict[platform_path] From d53e6b146e2da2d7a22960b452f4ee7015a9632d Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 11:42:24 +0000 Subject: [PATCH 03/59] Complete JRR cloudmask product list --- satpy/etc/readers/viirs_jrr.yaml | 43 ++++++++++++++++++++++++++++++++ satpy/readers/viirs_jrr.py | 7 ++---- 2 files changed, 45 insertions(+), 5 deletions(-) diff --git a/satpy/etc/readers/viirs_jrr.yaml b/satpy/etc/readers/viirs_jrr.yaml index fe896c1cf8..d9326460e7 100644 --- a/satpy/etc/readers/viirs_jrr.yaml +++ b/satpy/etc/readers/viirs_jrr.yaml @@ -34,4 +34,47 @@ datasets: units: '1' flag_meanings: ['Clear', 'Probably Clear', 'Probably Cloudy', 'Cloudy'] flag_values: [0, 1, 2, 3] + _FillValue: -128 + cloud_mask_binary: + name: cloud_mask_binary + file_type: [jrr_cloudmask] + file_key: "CloudMaskBinary" + coordinates: [longitude, latitude] + units: '1' + flag_meanings: ['Clear', 'Cloudy'] + flag_values: [0, 1] + _FillValue: -128 + cloud_probability: + name: cloud_probability + file_type: [jrr_cloudmask] + file_key: "CloudProbability" + coordinates: [longitude, latitude] + units: '1' + _FillValue: -999. + dust_mask: + name: dust_mask + file_type: [jrr_cloudmask] + file_key: "Dust_Mask" + coordinates: [longitude, latitude] + units: '1' + flag_meanings: ['Clear', 'Dusty'] + flag_values: [0, 1] + _FillValue: -128 + fire_mask: + name: fire_mask + file_type: [jrr_cloudmask] + file_key: "Fire_Mask" + coordinates: [longitude, latitude] + units: '1' + flag_meanings: ['No fire', 'Fire'] + flag_values: [0, 1] + _FillValue: -128 + smoke_mask: + name: smoke_mask + file_type: [jrr_cloudmask] + file_key: "Smoke_Mask" + coordinates: [longitude, latitude] + units: '1' + flag_meanings: ['Clear', 'Smoky'] + flag_values: [0, 1] _FillValue: -128 \ No newline at end of file diff --git a/satpy/readers/viirs_jrr.py b/satpy/readers/viirs_jrr.py index 0704834da4..42f13547f0 100644 --- a/satpy/readers/viirs_jrr.py +++ b/satpy/readers/viirs_jrr.py @@ -21,14 +21,11 @@ VIIRS instrument. These replace the 'old' EDR products. """ -import logging -from satpy import CHUNK_SIZE from satpy.readers.file_handlers import BaseFileHandler - -import dask.array as da -import numpy as np +from satpy import CHUNK_SIZE import xarray as xr +import logging # map platform attributes to Oscar standard name PLATFORM_MAP = { From c1523d1b108122c23ba28b34686fc083bbd45e46 Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 12:21:36 +0000 Subject: [PATCH 04/59] Add JRR aerosol product list --- satpy/etc/readers/viirs_jrr.yaml | 117 +++++++++++++++++++++++++++++-- 1 file changed, 113 insertions(+), 4 deletions(-) diff --git a/satpy/etc/readers/viirs_jrr.yaml b/satpy/etc/readers/viirs_jrr.yaml index d9326460e7..70a49ed041 100644 --- a/satpy/etc/readers/viirs_jrr.yaml +++ b/satpy/etc/readers/viirs_jrr.yaml @@ -11,24 +11,32 @@ file_types: variable_prefix: "" file_patterns: - 'JRR-CloudMask_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' + jrr_aerosol_product: + file_reader: !!python/name:satpy.readers.viirs_jrr.VIIRSJRRFileHandler + variable_prefix: "" + file_patterns: + - 'JRR-ADP_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' datasets: + # Common datasets longitude: name: longitude standard_name: longitude - file_type: [jrr_cloudmask] + file_type: [jrr_cloudmask, jrr_aerosol_product] file_key: "Longitude" units: 'degrees_east' latitude: name: latitude standard_name: latitude - file_type: [jrr_cloudmask] + file_type: [jrr_cloudmask, jrr_aerosol_product] file_key: "Latitude" units: 'degrees_north' + + # Cloudmask product datasets cloud_mask: name: cloud_mask - file_type: [jrr_cloudmask] + file_type: jrr_cloudmask file_key: "CloudMask" coordinates: [longitude, latitude] units: '1' @@ -77,4 +85,105 @@ datasets: units: '1' flag_meanings: ['Clear', 'Smoky'] flag_values: [0, 1] - _FillValue: -128 \ No newline at end of file + _FillValue: -128 + + # Aerosol optical depth product datasets + ash_mask: + name: ash_mask + file_type: [jrr_aerosol_product] + file_key: "Ash" + coordinates: [longitude, latitude] + units: '1' + flag_meanings: ['Clear', 'Ash'] + flag_values: [0, 1] + _FillValue: -128 + cloud_mask_adp: + name: cloud_mask_adp + file_type: [jrr_aerosol_product] + file_key: "Cloud" + coordinates: [longitude, latitude] + units: '1' + flag_meanings: ['Clear', 'Probably Clear', 'Probably Cloudy', 'Cloudy'] + flag_values: [0, 1, 2, 3] + _FillValue: -128 + dust_smoke_discrimination_index: + name: dust_smoke_discrimination_index + file_type: [jrr_aerosol_product] + file_key: "DSDI" + coordinates: [longitude, latitude] + units: '1' + _FillValue: -999 + nuc: + name: nuc + file_type: [jrr_aerosol_product] + file_key: "NUC" + coordinates: [longitude, latitude] + units: '1' + flag_meanings: ['No', 'Yes'] + flag_values: [0, 1] + _FillValue: -128 + pqi1: + name: pqi1 + file_type: [jrr_aerosol_product] + file_key: "PQI1" + coordinates: [longitude, latitude] + units: '1' + _FillValue: -128 + pqi2: + name: pqi2 + file_type: [jrr_aerosol_product] + file_key: "PQI2" + coordinates: [longitude, latitude] + units: '1' + _FillValue: -128 + pqi3: + name: pqi3 + file_type: [jrr_aerosol_product] + file_key: "PQI3" + coordinates: [longitude, latitude] + units: '1' + _FillValue: -128 + pqi4: + name: pqi4 + file_type: [jrr_aerosol_product] + file_key: "PQI4" + coordinates: [longitude, latitude] + units: '1' + _FillValue: -128 + qcflag: + name: qcflag + file_type: [jrr_aerosol_product] + file_key: "QC_Flag" + coordinates: [longitude, latitude] + units: '1' + _FillValue: -128 + saai: + name: saai + file_type: [jrr_aerosol_product] + file_key: "SAAI" + coordinates: [longitude, latitude] + units: '1' + _FillValue: -999 + smoke: + name: smoke + file_type: [jrr_aerosol_product] + file_key: "Smoke" + coordinates: [longitude, latitude] + units: '1' + _FillValue: -999 + smoke_concentration: + name: smoke_concentration + file_type: [jrr_aerosol_product] + file_key: "SmokeCon" + coordinates: [longitude, latitude] + units: 'ug/m^3' + _FillValue: -999 + snow_ice: + name: snow_ice + file_type: [jrr_aerosol_product] + file_key: "SnowIce" + coordinates: [longitude, latitude] + units: '1' + flag_meanings: ['No', 'Yes'] + flag_values: [0, 1] + _FillValue: -128 From d5f3ba573231ddaf4bab397b539a4680f7b30fec Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 12:54:09 +0000 Subject: [PATCH 05/59] Add surface reflectance JRR product --- satpy/etc/readers/viirs_jrr.yaml | 166 ++++++++++++++++++++++++++++++- satpy/readers/viirs_jrr.py | 8 +- 2 files changed, 171 insertions(+), 3 deletions(-) diff --git a/satpy/etc/readers/viirs_jrr.yaml b/satpy/etc/readers/viirs_jrr.yaml index 70a49ed041..81468aa313 100644 --- a/satpy/etc/readers/viirs_jrr.yaml +++ b/satpy/etc/readers/viirs_jrr.yaml @@ -16,10 +16,15 @@ file_types: variable_prefix: "" file_patterns: - 'JRR-ADP_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' + jrr_surfref_product: + file_reader: !!python/name:satpy.readers.viirs_jrr.VIIRSJRRFileHandler + variable_prefix: "" + file_patterns: + - 'SurfRefl_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' datasets: - # Common datasets + # Geolocation datasets longitude: name: longitude standard_name: longitude @@ -32,6 +37,30 @@ datasets: file_type: [jrr_cloudmask, jrr_aerosol_product] file_key: "Latitude" units: 'degrees_north' + longitude_375: + name: longitude_375 + standard_name: longitude + file_type: jrr_surfref_product + file_key: "Longitude_at_375m_resolution" + units: 'degrees_east' + latitude_375: + name: latitude_375 + standard_name: latitude + file_type: jrr_surfref_product + file_key: "Latitude_at_375m_resolution" + units: 'degrees_north' + longitude_750: + name: longitude_750 + standard_name: longitude + file_type: jrr_surfref_product + file_key: "Longitude_at_750m_resolution" + units: 'degrees_east' + latitude_750: + name: latitude_750 + standard_name: latitude + file_type: jrr_surfref_product + file_key: "Latitude_at_750m_resolution" + units: 'degrees_north' # Cloudmask product datasets cloud_mask: @@ -187,3 +216,138 @@ datasets: flag_meanings: ['No', 'Yes'] flag_values: [0, 1] _FillValue: -128 + + # Surface reflectance products + surf_refl_I01: + name: surf_refl_I01 + file_type: [jrr_surfref_product] + file_key: "375m Surface Reflectance Band I1" + coordinates: [longitude_375, latitude_375] + units: '1' + _FillValue: -9999 + surf_refl_I02: + name: surf_refl_I02 + file_type: [jrr_surfref_product] + file_key: "375m Surface Reflectance Band I2" + coordinates: [longitude_375, latitude_375] + units: '1' + _FillValue: -9999 + surf_refl_I03: + name: surf_refl_I03 + file_type: [jrr_surfref_product] + file_key: "375m Surface Reflectance Band I3" + coordinates: [longitude_375, latitude_375] + units: '1' + _FillValue: -9999 + surf_refl_M01: + name: surf_refl_M01 + file_type: [jrr_surfref_product] + file_key: "375m Surface Reflectance Band M1" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_M02: + name: surf_refl_M02 + file_type: [jrr_surfref_product] + file_key: "375m Surface Reflectance Band M2" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_M03: + name: surf_refl_M03 + file_type: [jrr_surfref_product] + file_key: "375m Surface Reflectance Band M3" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_M04: + name: surf_refl_M04 + file_type: [jrr_surfref_product] + file_key: "375m Surface Reflectance Band M4" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_M05: + name: surf_refl_M05 + file_type: [jrr_surfref_product] + file_key: "375m Surface Reflectance Band M5" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_M06: + name: surf_refl_M06 + file_type: [jrr_surfref_product] + file_key: "375m Surface Reflectance Band M6" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_M07: + name: surf_refl_M07 + file_type: [jrr_surfref_product] + file_key: "375m Surface Reflectance Band M7" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_M08: + name: surf_refl_M08 + file_type: [jrr_surfref_product] + file_key: "375m Surface Reflectance Band M8" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_M10: + name: surf_refl_M10 + file_type: [jrr_surfref_product] + file_key: "375m Surface Reflectance Band M10" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_qf1: + name: surf_refl_qf1 + file_type: [jrr_surfref_product] + file_key: "QF1 Surface Reflectance" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_qf2: + name: surf_refl_qf2 + file_type: [jrr_surfref_product] + file_key: "QF2 Surface Reflectance" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_qf3: + name: surf_refl_qf3 + file_type: [jrr_surfref_product] + file_key: "QF3 Surface Reflectance" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_qf4: + name: surf_refl_qf4 + file_type: [jrr_surfref_product] + file_key: "QF4 Surface Reflectance" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_qf5: + name: surf_refl_qf5 + file_type: [jrr_surfref_product] + file_key: "QF5 Surface Reflectance" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_qf6: + name: surf_refl_qf6 + file_type: [jrr_surfref_product] + file_key: "QF6 Surface Reflectance" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 + surf_refl_qf7: + name: surf_refl_qf7 + file_type: [jrr_surfref_product] + file_key: "QF7 Surface Reflectance" + coordinates: [longitude_750, latitude_750] + units: '1' + _FillValue: -9999 diff --git a/satpy/readers/viirs_jrr.py b/satpy/readers/viirs_jrr.py index 42f13547f0..2d4944ae3d 100644 --- a/satpy/readers/viirs_jrr.py +++ b/satpy/readers/viirs_jrr.py @@ -49,9 +49,13 @@ def __init__(self, filename, filename_info, filetype_info): mask_and_scale=True, chunks={'Columns': CHUNK_SIZE, 'Rows': CHUNK_SIZE}) - self.nc = self.nc.rename({'Columns': 'x', 'Rows': 'y'}) + if 'columns' in self.nc.dims: + self.nc = self.nc.rename({'Columns': 'x', 'Rows': 'y'}) + elif 'Along_Track_375m' in self.nc.dims: + self.nc = self.nc.rename({'Along_Scan_375m': 'x', 'Along_Track_375m': 'y'}) + self.nc = self.nc.rename({'Along_Scan_750m': 'x', 'Along_Track_750m': 'y'}) - # For some reason, no 'standard_name' is defined in the netCDF files, so + # For some reason, no 'standard_name' is defined in some netCDF files, so # here we manually make the definitions. if 'Latitude' in self.nc: self.nc['Latitude'].attrs.update({'standard_name': 'latitude'}) From 9f4954da02b325b3b6d340d7dd9575b23e0c2d1f Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 13:35:11 +0000 Subject: [PATCH 06/59] Add VIIRS JRR composites, update JRR dataset keys for resolution. --- satpy/etc/composites/viirs.yaml | 24 ++++++++++ satpy/etc/readers/viirs_jrr.yaml | 76 ++++++++++++++++++++++++++++---- satpy/readers/viirs_jrr.py | 1 + 3 files changed, 92 insertions(+), 9 deletions(-) diff --git a/satpy/etc/composites/viirs.yaml b/satpy/etc/composites/viirs.yaml index 4fbc3a6b3c..da324d58b0 100644 --- a/satpy/etc/composites/viirs.yaml +++ b/satpy/etc/composites/viirs.yaml @@ -306,6 +306,30 @@ composites: modifiers: [sunz_corrected_iband] standard_name: natural_color + natural_color_iband_surf_nocorr: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - name: surf_refl_I03 + - name: surf_refl_I02 + - name: surf_refl_I01 + standard_name: natural_color + + natural_color_mband_surf_nocorr: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - name: surf_refl_M10 + - name: surf_refl_M07 + - name: surf_refl_M05 + standard_name: natural_color + + true_color_mband_nocorr: + compositor: !!python/name:satpy.composites.RGBCompositor + prerequisites: + - name: surf_refl_M05 + - name: surf_refl_M04 + - name: surf_refl_M03 + standard_name: true_color + natural_color_sun_lowres: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: diff --git a/satpy/etc/readers/viirs_jrr.yaml b/satpy/etc/readers/viirs_jrr.yaml index 81468aa313..7255756418 100644 --- a/satpy/etc/readers/viirs_jrr.yaml +++ b/satpy/etc/readers/viirs_jrr.yaml @@ -3,6 +3,8 @@ reader: name: viirs_jrr reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader sensors: [viirs] + group_keys: ['platform_shortname'] + default_datasets: file_types: @@ -31,40 +33,47 @@ datasets: file_type: [jrr_cloudmask, jrr_aerosol_product] file_key: "Longitude" units: 'degrees_east' + resolution: 750 latitude: name: latitude standard_name: latitude file_type: [jrr_cloudmask, jrr_aerosol_product] file_key: "Latitude" units: 'degrees_north' + resolution: 750 longitude_375: name: longitude_375 standard_name: longitude file_type: jrr_surfref_product file_key: "Longitude_at_375m_resolution" units: 'degrees_east' + resolution: 375 latitude_375: name: latitude_375 standard_name: latitude file_type: jrr_surfref_product file_key: "Latitude_at_375m_resolution" units: 'degrees_north' + resolution: 375 longitude_750: name: longitude_750 standard_name: longitude file_type: jrr_surfref_product file_key: "Longitude_at_750m_resolution" units: 'degrees_east' + resolution: 750 latitude_750: name: latitude_750 standard_name: latitude file_type: jrr_surfref_product file_key: "Latitude_at_750m_resolution" units: 'degrees_north' + resolution: 750 # Cloudmask product datasets cloud_mask: name: cloud_mask + resolution: 750 file_type: jrr_cloudmask file_key: "CloudMask" coordinates: [longitude, latitude] @@ -74,6 +83,7 @@ datasets: _FillValue: -128 cloud_mask_binary: name: cloud_mask_binary + resolution: 750 file_type: [jrr_cloudmask] file_key: "CloudMaskBinary" coordinates: [longitude, latitude] @@ -83,6 +93,7 @@ datasets: _FillValue: -128 cloud_probability: name: cloud_probability + resolution: 750 file_type: [jrr_cloudmask] file_key: "CloudProbability" coordinates: [longitude, latitude] @@ -90,6 +101,7 @@ datasets: _FillValue: -999. dust_mask: name: dust_mask + resolution: 750 file_type: [jrr_cloudmask] file_key: "Dust_Mask" coordinates: [longitude, latitude] @@ -99,6 +111,7 @@ datasets: _FillValue: -128 fire_mask: name: fire_mask + resolution: 750 file_type: [jrr_cloudmask] file_key: "Fire_Mask" coordinates: [longitude, latitude] @@ -108,6 +121,7 @@ datasets: _FillValue: -128 smoke_mask: name: smoke_mask + resolution: 750 file_type: [jrr_cloudmask] file_key: "Smoke_Mask" coordinates: [longitude, latitude] @@ -119,6 +133,7 @@ datasets: # Aerosol optical depth product datasets ash_mask: name: ash_mask + resolution: 750 file_type: [jrr_aerosol_product] file_key: "Ash" coordinates: [longitude, latitude] @@ -128,6 +143,7 @@ datasets: _FillValue: -128 cloud_mask_adp: name: cloud_mask_adp + resolution: 750 file_type: [jrr_aerosol_product] file_key: "Cloud" coordinates: [longitude, latitude] @@ -137,6 +153,7 @@ datasets: _FillValue: -128 dust_smoke_discrimination_index: name: dust_smoke_discrimination_index + resolution: 750 file_type: [jrr_aerosol_product] file_key: "DSDI" coordinates: [longitude, latitude] @@ -144,6 +161,7 @@ datasets: _FillValue: -999 nuc: name: nuc + resolution: 750 file_type: [jrr_aerosol_product] file_key: "NUC" coordinates: [longitude, latitude] @@ -153,6 +171,7 @@ datasets: _FillValue: -128 pqi1: name: pqi1 + resolution: 750 file_type: [jrr_aerosol_product] file_key: "PQI1" coordinates: [longitude, latitude] @@ -160,6 +179,7 @@ datasets: _FillValue: -128 pqi2: name: pqi2 + resolution: 750 file_type: [jrr_aerosol_product] file_key: "PQI2" coordinates: [longitude, latitude] @@ -167,6 +187,7 @@ datasets: _FillValue: -128 pqi3: name: pqi3 + resolution: 750 file_type: [jrr_aerosol_product] file_key: "PQI3" coordinates: [longitude, latitude] @@ -174,6 +195,7 @@ datasets: _FillValue: -128 pqi4: name: pqi4 + resolution: 750 file_type: [jrr_aerosol_product] file_key: "PQI4" coordinates: [longitude, latitude] @@ -181,6 +203,7 @@ datasets: _FillValue: -128 qcflag: name: qcflag + resolution: 750 file_type: [jrr_aerosol_product] file_key: "QC_Flag" coordinates: [longitude, latitude] @@ -188,6 +211,7 @@ datasets: _FillValue: -128 saai: name: saai + resolution: 750 file_type: [jrr_aerosol_product] file_key: "SAAI" coordinates: [longitude, latitude] @@ -195,6 +219,7 @@ datasets: _FillValue: -999 smoke: name: smoke + resolution: 750 file_type: [jrr_aerosol_product] file_key: "Smoke" coordinates: [longitude, latitude] @@ -202,6 +227,7 @@ datasets: _FillValue: -999 smoke_concentration: name: smoke_concentration + resolution: 750 file_type: [jrr_aerosol_product] file_key: "SmokeCon" coordinates: [longitude, latitude] @@ -209,6 +235,7 @@ datasets: _FillValue: -999 snow_ice: name: snow_ice + resolution: 750 file_type: [jrr_aerosol_product] file_key: "SnowIce" coordinates: [longitude, latitude] @@ -220,6 +247,8 @@ datasets: # Surface reflectance products surf_refl_I01: name: surf_refl_I01 + resolution: 375 + wavelength: [0.600, 0.640, 0.680] file_type: [jrr_surfref_product] file_key: "375m Surface Reflectance Band I1" coordinates: [longitude_375, latitude_375] @@ -227,6 +256,8 @@ datasets: _FillValue: -9999 surf_refl_I02: name: surf_refl_I02 + resolution: 375 + wavelength: [0.845, 0.865, 0.884] file_type: [jrr_surfref_product] file_key: "375m Surface Reflectance Band I2" coordinates: [longitude_375, latitude_375] @@ -234,6 +265,8 @@ datasets: _FillValue: -9999 surf_refl_I03: name: surf_refl_I03 + resolution: 375 + wavelength: [1.580, 1.610, 1.640] file_type: [jrr_surfref_product] file_key: "375m Surface Reflectance Band I3" coordinates: [longitude_375, latitude_375] @@ -241,69 +274,88 @@ datasets: _FillValue: -9999 surf_refl_M01: name: surf_refl_M01 + resolution: 750 + wavelength: [0.402, 0.412, 0.422] file_type: [jrr_surfref_product] - file_key: "375m Surface Reflectance Band M1" + file_key: "750m Surface Reflectance Band M1" coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 surf_refl_M02: name: surf_refl_M02 + resolution: 750 + wavelength: [0.436, 0.445, 0.454] file_type: [jrr_surfref_product] - file_key: "375m Surface Reflectance Band M2" + file_key: "750m Surface Reflectance Band M2" coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 surf_refl_M03: name: surf_refl_M03 + resolution: 750 + wavelength: [0.478, 0.488, 0.498] file_type: [jrr_surfref_product] - file_key: "375m Surface Reflectance Band M3" + file_key: "750m Surface Reflectance Band M3" coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 surf_refl_M04: name: surf_refl_M04 + resolution: 750 + wavelength: [0.545, 0.555, 0.565] file_type: [jrr_surfref_product] - file_key: "375m Surface Reflectance Band M4" + file_key: "750m Surface Reflectance Band M4" coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 surf_refl_M05: name: surf_refl_M05 + resolution: 750 + wavelength: [0.662, 0.672, 0.682] file_type: [jrr_surfref_product] - file_key: "375m Surface Reflectance Band M5" + file_key: "750m Surface Reflectance Band M5" coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 surf_refl_M06: name: surf_refl_M06 + resolution: 750 + wavelength: [0.739, 0.746, 0.754] file_type: [jrr_surfref_product] - file_key: "375m Surface Reflectance Band M6" + file_key: "750m Surface Reflectance Band M6" coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 surf_refl_M07: name: surf_refl_M07 + resolution: 750 + wavelength: [0.846, 0.865, 0.885] file_type: [jrr_surfref_product] - file_key: "375m Surface Reflectance Band M7" + file_key: "750m Surface Reflectance Band M7" coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 surf_refl_M08: name: surf_refl_M08 + resolution: 750 + wavelength: [1.230, 1.240, 1.250] file_type: [jrr_surfref_product] - file_key: "375m Surface Reflectance Band M8" + file_key: "750m Surface Reflectance Band M8" coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 surf_refl_M10: name: surf_refl_M10 + resolution: 750 + wavelength: [1.580, 1.610, 1.640] file_type: [jrr_surfref_product] - file_key: "375m Surface Reflectance Band M10" + file_key: "750m Surface Reflectance Band M10" coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 surf_refl_qf1: name: surf_refl_qf1 + resolution: 750 file_type: [jrr_surfref_product] file_key: "QF1 Surface Reflectance" coordinates: [longitude_750, latitude_750] @@ -311,6 +363,7 @@ datasets: _FillValue: -9999 surf_refl_qf2: name: surf_refl_qf2 + resolution: 750 file_type: [jrr_surfref_product] file_key: "QF2 Surface Reflectance" coordinates: [longitude_750, latitude_750] @@ -318,6 +371,7 @@ datasets: _FillValue: -9999 surf_refl_qf3: name: surf_refl_qf3 + resolution: 750 file_type: [jrr_surfref_product] file_key: "QF3 Surface Reflectance" coordinates: [longitude_750, latitude_750] @@ -325,6 +379,7 @@ datasets: _FillValue: -9999 surf_refl_qf4: name: surf_refl_qf4 + resolution: 750 file_type: [jrr_surfref_product] file_key: "QF4 Surface Reflectance" coordinates: [longitude_750, latitude_750] @@ -332,6 +387,7 @@ datasets: _FillValue: -9999 surf_refl_qf5: name: surf_refl_qf5 + resolution: 750 file_type: [jrr_surfref_product] file_key: "QF5 Surface Reflectance" coordinates: [longitude_750, latitude_750] @@ -339,6 +395,7 @@ datasets: _FillValue: -9999 surf_refl_qf6: name: surf_refl_qf6 + resolution: 750 file_type: [jrr_surfref_product] file_key: "QF6 Surface Reflectance" coordinates: [longitude_750, latitude_750] @@ -346,6 +403,7 @@ datasets: _FillValue: -9999 surf_refl_qf7: name: surf_refl_qf7 + resolution: 750 file_type: [jrr_surfref_product] file_key: "QF7 Surface Reflectance" coordinates: [longitude_750, latitude_750] diff --git a/satpy/readers/viirs_jrr.py b/satpy/readers/viirs_jrr.py index 2d4944ae3d..a04886d852 100644 --- a/satpy/readers/viirs_jrr.py +++ b/satpy/readers/viirs_jrr.py @@ -63,6 +63,7 @@ def __init__(self, filename, filename_info, filetype_info): self.nc['Longitude'].attrs.update({'standard_name': 'longitude'}) self.algorithm_version = filename_info['platform_shortname'] + self.sensor_name = 'viirs' def get_dataset(self, dataset_id, info): """Get the dataset.""" From dea90d659e1c652e2a71875ab270d2e297dc3557 Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 13:41:39 +0000 Subject: [PATCH 07/59] Update VIIRS JRR module docstring. --- satpy/readers/viirs_jrr.py | 24 ++++++++++++++++++++++-- 1 file changed, 22 insertions(+), 2 deletions(-) diff --git a/satpy/readers/viirs_jrr.py b/satpy/readers/viirs_jrr.py index a04886d852..f8a502e583 100644 --- a/satpy/readers/viirs_jrr.py +++ b/satpy/readers/viirs_jrr.py @@ -17,8 +17,28 @@ # satpy. If not, see . """VIIRS NOAA enterprise L2 product reader. -This module implements readers for the NOAA enterprise level 2 products for the -VIIRS instrument. These replace the 'old' EDR products. +This module defines the :class:`VIIRSJRRFileHandler` file handler, to +be used for reading VIIRS Level 2 products generated by the NOAA enterprise +suite, which are downloadable via NOAA CLASS. +A wide variety of such products exist and, at present, only three are +supported here, showing example filenames: + - Cloud mask: JRR-CloudMask_v2r3_j01_s202112250807275_e202112250808520_c202112250837300.nc + - Aerosol properties: JRR-ADP_v2r3_j01_s202112250807275_e202112250808520_c202112250839550.nc + - Surface reflectance: SurfRefl_v1r1_j01_s202112250807275_e202112250808520_c202112250845080.nc +All products use the same base reader `viirs_jrr` and can be read through satpy with:: + + import satpy + import glob + + filenames = glob.glob('JRR-ADP*.nc') + scene = satpy.Scene(filenames, + reader='viirs_jrr') + scene.load(['smoke_concentration']) + +NOTE: + Multiple products contain datasets with the same name! For example, both the cloud mask + and aerosol files contain a cloud mask, but these are not identical. + For clarity, the aerosol file cloudmask is named `cloud_mask_adp` in this reader. """ From 2a9cf101065374123a0c620c917ac59ec9103ff8 Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 14:52:17 +0000 Subject: [PATCH 08/59] Update VIIRS JRR reader name and add tests. --- .../{viirs_jrr.yaml => viirs_l2_jrr.yaml} | 6 +- .../readers/{viirs_jrr.py => viirs_l2_jrr.py} | 14 +-- satpy/tests/reader_tests/test_viirs_l2_jrr.py | 93 +++++++++++++++++++ 3 files changed, 100 insertions(+), 13 deletions(-) rename satpy/etc/readers/{viirs_jrr.yaml => viirs_l2_jrr.yaml} (98%) rename satpy/readers/{viirs_jrr.py => viirs_l2_jrr.py} (93%) create mode 100644 satpy/tests/reader_tests/test_viirs_l2_jrr.py diff --git a/satpy/etc/readers/viirs_jrr.yaml b/satpy/etc/readers/viirs_l2_jrr.yaml similarity index 98% rename from satpy/etc/readers/viirs_jrr.yaml rename to satpy/etc/readers/viirs_l2_jrr.yaml index 7255756418..98bcd9253c 100644 --- a/satpy/etc/readers/viirs_jrr.yaml +++ b/satpy/etc/readers/viirs_l2_jrr.yaml @@ -9,17 +9,17 @@ reader: file_types: jrr_cloudmask: - file_reader: !!python/name:satpy.readers.viirs_jrr.VIIRSJRRFileHandler + file_reader: !!python/name:satpy.readers.viirs_l2_jrr.VIIRSJRRFileHandler variable_prefix: "" file_patterns: - 'JRR-CloudMask_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' jrr_aerosol_product: - file_reader: !!python/name:satpy.readers.viirs_jrr.VIIRSJRRFileHandler + file_reader: !!python/name:satpy.readers.viirs_l2_jrr.VIIRSJRRFileHandler variable_prefix: "" file_patterns: - 'JRR-ADP_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' jrr_surfref_product: - file_reader: !!python/name:satpy.readers.viirs_jrr.VIIRSJRRFileHandler + file_reader: !!python/name:satpy.readers.viirs_l2_jrr.VIIRSJRRFileHandler variable_prefix: "" file_patterns: - 'SurfRefl_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' diff --git a/satpy/readers/viirs_jrr.py b/satpy/readers/viirs_l2_jrr.py similarity index 93% rename from satpy/readers/viirs_jrr.py rename to satpy/readers/viirs_l2_jrr.py index f8a502e583..4c0694fcd5 100644 --- a/satpy/readers/viirs_jrr.py +++ b/satpy/readers/viirs_l2_jrr.py @@ -43,17 +43,11 @@ from satpy.readers.file_handlers import BaseFileHandler +from datetime import datetime from satpy import CHUNK_SIZE import xarray as xr import logging -# map platform attributes to Oscar standard name -PLATFORM_MAP = { - "NPP": "Suomi-NPP", - "J01": "NOAA-20", - "J02": "NOAA-21", -} - LOG = logging.getLogger(__name__) @@ -99,15 +93,15 @@ def start_time(self): @property def end_time(self): """Get last date/time when observations were recorded.""" - return self.filename_info.get('end_time', self.start_time) + return self.filename_info['end_time'] @property def platform_name(self): """Get platform name.""" - platform_path = self.filetype_info['platform_name'] + platform_path = self.filename_info['platform_shortname'] platform_dict = {'NPP': 'Suomi-NPP', 'JPSS-1': 'NOAA-20', 'J01': 'NOAA-20', 'JPSS-2': 'NOAA-21', 'J02': 'NOAA-21'} - return platform_dict[platform_path] + return platform_dict[platform_path.upper()] diff --git a/satpy/tests/reader_tests/test_viirs_l2_jrr.py b/satpy/tests/reader_tests/test_viirs_l2_jrr.py new file mode 100644 index 0000000000..c482319721 --- /dev/null +++ b/satpy/tests/reader_tests/test_viirs_l2_jrr.py @@ -0,0 +1,93 @@ +#!/usr/bin/env python +# -*- coding: utf-8 -*- +# Copyright (c) 2022 Satpy developers +# +# This file is part of satpy. +# +# satpy is free software: you can redistribute it and/or modify it under the +# terms of the GNU General Public License as published by the Free Software +# Foundation, either version 3 of the License, or (at your option) any later +# version. +# +# satpy is distributed in the hope that it will be useful, but WITHOUT ANY +# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR +# A PARTICULAR PURPOSE. See the GNU General Public License for more details. +# +# You should have received a copy of the GNU General Public License along with +# satpy. If not, see . +"""Module for testing the satpy.readers.viirs_jrr module. +Note: This is adapted from the test_slstr_l2.py code. +""" + +from unittest.mock import MagicMock +from datetime import datetime +from unittest import mock +import xarray as xr +import unittest + + +from satpy.readers.viirs_l2_jrr import VIIRSJRRFileHandler + + +class TestVIIRSJRRReader(unittest.TestCase): + """Test the VIIRS JRR L2 reader.""" + @mock.patch('xarray.open_dataset') + def test_instantiate(self, mocked_dataset): + """Test initialization of file handlers.""" + filename_info = {'platform_shortname': 'npp'} + tmp = MagicMock(start_time='20191120T125002Z', stop_time='20191120T125002Z') + tmp.rename.return_value = tmp + xr.open_dataset.return_value = tmp + VIIRSJRRFileHandler('somedir/somefile.nc', filename_info, None) + mocked_dataset.assert_called() + mocked_dataset.reset_mock() + + @mock.patch('xarray.open_dataset') + def test_get_dataset(self, mocked_dataset): + """Test retrieval of datasets.""" + filename_info = {'platform_shortname': 'npp'} + tmp = MagicMock(start_time='20191120T125002Z', stop_time='20191120T125002Z') + xr.open_dataset.return_value = tmp + test = VIIRSJRRFileHandler('somedir/somefile.nc', filename_info, None) + test.nc = {'Longitude': xr.Dataset(), + 'Latitude': xr.Dataset(), + 'smoke_concentration': xr.Dataset(), + 'fire_mask': xr.Dataset(), + 'surf_refl_I01': xr.Dataset(), + 'surf_refl_M05': xr.Dataset(), + } + test.get_dataset('longitude', {'file_key': 'Longitude'}) + test.get_dataset('latitude', {'file_key': 'Latitude'}) + test.get_dataset('smoke_concentration', {'file_key': 'smoke_concentration'}) + test.get_dataset('fire_mask', {'file_key': 'fire_mask'}) + with self.assertRaises(KeyError): + test.get_dataset('erroneous dataset', {'file_key': 'erroneous dataset'}) + mocked_dataset.assert_called() + mocked_dataset.reset_mock() + test.get_dataset('surf_refl_I01', {'file_key': 'surf_refl_I01'}) + + @mock.patch('xarray.open_dataset') + def test_get_startend_times(self, mocked_dataset): + """Test finding start and end times of granules.""" + filename_info = {'platform_shortname': 'npp', + 'start_time': datetime(2021, 4, 3, 12, 0, 10), + 'end_time': datetime(2021, 4, 3, 12, 4, 28)} + tmp = MagicMock() + tmp.rename.return_value = tmp + xr.open_dataset.return_value = tmp + hdl = VIIRSJRRFileHandler('somedir/somefile.nc', filename_info, None) + self.assertEqual(hdl.start_time, datetime(2021, 4, 3, 12, 0, 10)) + self.assertEqual(hdl.end_time, datetime(2021, 4, 3, 12, 4, 28)) + + @mock.patch('xarray.open_dataset') + def test_get_platformname(self, mocked_dataset): + """Test finding start and end times of granules.""" + tmp = MagicMock() + tmp.rename.return_value = tmp + xr.open_dataset.return_value = tmp + hdl = VIIRSJRRFileHandler('somedir/somefile.nc', {'platform_shortname': 'npp'}, None) + self.assertEqual(hdl.platform_name, 'Suomi-NPP') + hdl = VIIRSJRRFileHandler('somedir/somefile.nc', {'platform_shortname': 'JPSS-1'}, None) + self.assertEqual(hdl.platform_name, 'NOAA-20') + hdl = VIIRSJRRFileHandler('somedir/somefile.nc', {'platform_shortname': 'J01'}, None) + self.assertEqual(hdl.platform_name, 'NOAA-20') From 1fa0aecd0203f77a7680d1b33a2553edc0c9fd70 Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 14:57:01 +0000 Subject: [PATCH 09/59] Remove unused import. --- satpy/readers/viirs_l2_jrr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/readers/viirs_l2_jrr.py b/satpy/readers/viirs_l2_jrr.py index 4c0694fcd5..73de4d499d 100644 --- a/satpy/readers/viirs_l2_jrr.py +++ b/satpy/readers/viirs_l2_jrr.py @@ -43,7 +43,6 @@ from satpy.readers.file_handlers import BaseFileHandler -from datetime import datetime from satpy import CHUNK_SIZE import xarray as xr import logging From 213d7358cdde0419b537ea9cf28bb7c635f56fb1 Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 15:04:21 +0000 Subject: [PATCH 10/59] Add blank lines. --- satpy/tests/reader_tests/test_viirs_l2_jrr.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/satpy/tests/reader_tests/test_viirs_l2_jrr.py b/satpy/tests/reader_tests/test_viirs_l2_jrr.py index c482319721..4364b5c43c 100644 --- a/satpy/tests/reader_tests/test_viirs_l2_jrr.py +++ b/satpy/tests/reader_tests/test_viirs_l2_jrr.py @@ -16,6 +16,7 @@ # You should have received a copy of the GNU General Public License along with # satpy. If not, see . """Module for testing the satpy.readers.viirs_jrr module. + Note: This is adapted from the test_slstr_l2.py code. """ @@ -31,6 +32,7 @@ class TestVIIRSJRRReader(unittest.TestCase): """Test the VIIRS JRR L2 reader.""" + @mock.patch('xarray.open_dataset') def test_instantiate(self, mocked_dataset): """Test initialization of file handlers.""" From e194830020e6e7dc31ee5bdfcdce7d4a5e4a911e Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 15:12:39 +0000 Subject: [PATCH 11/59] Fix indentation. --- satpy/readers/viirs_l2_jrr.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/satpy/readers/viirs_l2_jrr.py b/satpy/readers/viirs_l2_jrr.py index 73de4d499d..09eac0cb17 100644 --- a/satpy/readers/viirs_l2_jrr.py +++ b/satpy/readers/viirs_l2_jrr.py @@ -36,9 +36,9 @@ scene.load(['smoke_concentration']) NOTE: - Multiple products contain datasets with the same name! For example, both the cloud mask - and aerosol files contain a cloud mask, but these are not identical. - For clarity, the aerosol file cloudmask is named `cloud_mask_adp` in this reader. +Multiple products contain datasets with the same name! For example, both the cloud mask +and aerosol files contain a cloud mask, but these are not identical. +For clarity, the aerosol file cloudmask is named `cloud_mask_adp` in this reader. """ From caadd86139d5b43b509598eaab4b5380e9faa350 Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 15:37:53 +0000 Subject: [PATCH 12/59] Update some reader name changes that were missed previously. --- satpy/etc/readers/viirs_l2_jrr.yaml | 2 +- satpy/readers/viirs_l2_jrr.py | 4 ++-- satpy/tests/reader_tests/test_viirs_l2_jrr.py | 2 +- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/satpy/etc/readers/viirs_l2_jrr.yaml b/satpy/etc/readers/viirs_l2_jrr.yaml index 98bcd9253c..f337909134 100644 --- a/satpy/etc/readers/viirs_l2_jrr.yaml +++ b/satpy/etc/readers/viirs_l2_jrr.yaml @@ -1,6 +1,6 @@ reader: description: VIIRS NOAA Enterprise L2 product reader - name: viirs_jrr + name: viirs_l2_jrr reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader sensors: [viirs] group_keys: ['platform_shortname'] diff --git a/satpy/readers/viirs_l2_jrr.py b/satpy/readers/viirs_l2_jrr.py index 09eac0cb17..e1e01c9de7 100644 --- a/satpy/readers/viirs_l2_jrr.py +++ b/satpy/readers/viirs_l2_jrr.py @@ -25,14 +25,14 @@ - Cloud mask: JRR-CloudMask_v2r3_j01_s202112250807275_e202112250808520_c202112250837300.nc - Aerosol properties: JRR-ADP_v2r3_j01_s202112250807275_e202112250808520_c202112250839550.nc - Surface reflectance: SurfRefl_v1r1_j01_s202112250807275_e202112250808520_c202112250845080.nc -All products use the same base reader `viirs_jrr` and can be read through satpy with:: +All products use the same base reader `viirs_l2_jrr` and can be read through satpy with:: import satpy import glob filenames = glob.glob('JRR-ADP*.nc') scene = satpy.Scene(filenames, - reader='viirs_jrr') + reader='viirs_l2_jrr') scene.load(['smoke_concentration']) NOTE: diff --git a/satpy/tests/reader_tests/test_viirs_l2_jrr.py b/satpy/tests/reader_tests/test_viirs_l2_jrr.py index 4364b5c43c..c572481adb 100644 --- a/satpy/tests/reader_tests/test_viirs_l2_jrr.py +++ b/satpy/tests/reader_tests/test_viirs_l2_jrr.py @@ -15,7 +15,7 @@ # # You should have received a copy of the GNU General Public License along with # satpy. If not, see . -"""Module for testing the satpy.readers.viirs_jrr module. +"""Module for testing the satpy.readers.viirs_l2_jrr module. Note: This is adapted from the test_slstr_l2.py code. """ From a65f58dfff0b71a6a92d5dff4b1a3c7d918f4c4f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Tue, 11 Jan 2022 15:40:40 +0000 Subject: [PATCH 13/59] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- satpy/readers/viirs_l2_jrr.py | 8 +++++--- satpy/tests/reader_tests/test_viirs_l2_jrr.py | 6 +++--- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/satpy/readers/viirs_l2_jrr.py b/satpy/readers/viirs_l2_jrr.py index e1e01c9de7..a27e9c30dc 100644 --- a/satpy/readers/viirs_l2_jrr.py +++ b/satpy/readers/viirs_l2_jrr.py @@ -42,11 +42,13 @@ """ -from satpy.readers.file_handlers import BaseFileHandler -from satpy import CHUNK_SIZE -import xarray as xr import logging +import xarray as xr + +from satpy import CHUNK_SIZE +from satpy.readers.file_handlers import BaseFileHandler + LOG = logging.getLogger(__name__) diff --git a/satpy/tests/reader_tests/test_viirs_l2_jrr.py b/satpy/tests/reader_tests/test_viirs_l2_jrr.py index c572481adb..a462ec1416 100644 --- a/satpy/tests/reader_tests/test_viirs_l2_jrr.py +++ b/satpy/tests/reader_tests/test_viirs_l2_jrr.py @@ -20,12 +20,12 @@ Note: This is adapted from the test_slstr_l2.py code. """ -from unittest.mock import MagicMock +import unittest from datetime import datetime from unittest import mock -import xarray as xr -import unittest +from unittest.mock import MagicMock +import xarray as xr from satpy.readers.viirs_l2_jrr import VIIRSJRRFileHandler From 6e5d46d8611195984796cccafae5f5844b77163b Mon Sep 17 00:00:00 2001 From: simonrp84 Date: Tue, 11 Jan 2022 16:26:35 +0000 Subject: [PATCH 14/59] Remove unnecessary indentation. --- satpy/readers/viirs_l2_jrr.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/satpy/readers/viirs_l2_jrr.py b/satpy/readers/viirs_l2_jrr.py index a27e9c30dc..8d07b3a7c5 100644 --- a/satpy/readers/viirs_l2_jrr.py +++ b/satpy/readers/viirs_l2_jrr.py @@ -27,13 +27,13 @@ - Surface reflectance: SurfRefl_v1r1_j01_s202112250807275_e202112250808520_c202112250845080.nc All products use the same base reader `viirs_l2_jrr` and can be read through satpy with:: - import satpy - import glob + import satpy + import glob - filenames = glob.glob('JRR-ADP*.nc') - scene = satpy.Scene(filenames, - reader='viirs_l2_jrr') - scene.load(['smoke_concentration']) + filenames = glob.glob('JRR-ADP*.nc') + scene = satpy.Scene(filenames, + reader='viirs_l2_jrr') + scene.load(['smoke_concentration']) NOTE: Multiple products contain datasets with the same name! For example, both the cloud mask From bc7001f75573a2a5815f8e6195085791113e5d7c Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 18 Jul 2023 13:01:13 -0500 Subject: [PATCH 15/59] Rename viirs_l2_jrr reader to viirs_edr --- satpy/etc/readers/{viirs_l2_jrr.yaml => viirs_edr.yaml} | 4 ++-- satpy/readers/{viirs_l2_jrr.py => viirs_edr.py} | 7 ++++--- .../{test_viirs_l2_jrr.py => test_viirs_edr.py} | 2 +- 3 files changed, 7 insertions(+), 6 deletions(-) rename satpy/etc/readers/{viirs_l2_jrr.yaml => viirs_edr.yaml} (99%) rename satpy/readers/{viirs_l2_jrr.py => viirs_edr.py} (95%) rename satpy/tests/reader_tests/{test_viirs_l2_jrr.py => test_viirs_edr.py} (98%) diff --git a/satpy/etc/readers/viirs_l2_jrr.yaml b/satpy/etc/readers/viirs_edr.yaml similarity index 99% rename from satpy/etc/readers/viirs_l2_jrr.yaml rename to satpy/etc/readers/viirs_edr.yaml index f337909134..3f268b86b5 100644 --- a/satpy/etc/readers/viirs_l2_jrr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -1,6 +1,6 @@ reader: - description: VIIRS NOAA Enterprise L2 product reader - name: viirs_l2_jrr + description: VIIRS NOAA Enterprise EDR product reader + name: viirs_edr reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader sensors: [viirs] group_keys: ['platform_shortname'] diff --git a/satpy/readers/viirs_l2_jrr.py b/satpy/readers/viirs_edr.py similarity index 95% rename from satpy/readers/viirs_l2_jrr.py rename to satpy/readers/viirs_edr.py index 8d07b3a7c5..80a4c347c3 100644 --- a/satpy/readers/viirs_l2_jrr.py +++ b/satpy/readers/viirs_edr.py @@ -46,8 +46,8 @@ import xarray as xr -from satpy import CHUNK_SIZE from satpy.readers.file_handlers import BaseFileHandler +from satpy.utils import get_legacy_chunk_size LOG = logging.getLogger(__name__) @@ -59,11 +59,12 @@ def __init__(self, filename, filename_info, filetype_info): """Initialize the geo filehandler.""" super(VIIRSJRRFileHandler, self).__init__(filename, filename_info, filetype_info) + chunk_size = get_legacy_chunk_size() self.nc = xr.open_dataset(self.filename, decode_cf=True, mask_and_scale=True, - chunks={'Columns': CHUNK_SIZE, - 'Rows': CHUNK_SIZE}) + chunks={'Columns': chunk_size, + 'Rows': chunk_size}) if 'columns' in self.nc.dims: self.nc = self.nc.rename({'Columns': 'x', 'Rows': 'y'}) elif 'Along_Track_375m' in self.nc.dims: diff --git a/satpy/tests/reader_tests/test_viirs_l2_jrr.py b/satpy/tests/reader_tests/test_viirs_edr.py similarity index 98% rename from satpy/tests/reader_tests/test_viirs_l2_jrr.py rename to satpy/tests/reader_tests/test_viirs_edr.py index a462ec1416..def2af7ad2 100644 --- a/satpy/tests/reader_tests/test_viirs_l2_jrr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -27,7 +27,7 @@ import xarray as xr -from satpy.readers.viirs_l2_jrr import VIIRSJRRFileHandler +from satpy.readers.viirs_edr import VIIRSJRRFileHandler class TestVIIRSJRRReader(unittest.TestCase): From 9ff96918c47647f81028b1135fe7b986089d73d4 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 18 Jul 2023 13:29:58 -0500 Subject: [PATCH 16/59] Convert VIIRS EDR tests to pytest --- satpy/tests/reader_tests/test_viirs_edr.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index def2af7ad2..cbd036639f 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -20,17 +20,17 @@ Note: This is adapted from the test_slstr_l2.py code. """ -import unittest from datetime import datetime from unittest import mock from unittest.mock import MagicMock +import pytest import xarray as xr from satpy.readers.viirs_edr import VIIRSJRRFileHandler -class TestVIIRSJRRReader(unittest.TestCase): +class TestVIIRSJRRReader: """Test the VIIRS JRR L2 reader.""" @mock.patch('xarray.open_dataset') @@ -62,7 +62,7 @@ def test_get_dataset(self, mocked_dataset): test.get_dataset('latitude', {'file_key': 'Latitude'}) test.get_dataset('smoke_concentration', {'file_key': 'smoke_concentration'}) test.get_dataset('fire_mask', {'file_key': 'fire_mask'}) - with self.assertRaises(KeyError): + with pytest.raises(KeyError): test.get_dataset('erroneous dataset', {'file_key': 'erroneous dataset'}) mocked_dataset.assert_called() mocked_dataset.reset_mock() @@ -78,8 +78,8 @@ def test_get_startend_times(self, mocked_dataset): tmp.rename.return_value = tmp xr.open_dataset.return_value = tmp hdl = VIIRSJRRFileHandler('somedir/somefile.nc', filename_info, None) - self.assertEqual(hdl.start_time, datetime(2021, 4, 3, 12, 0, 10)) - self.assertEqual(hdl.end_time, datetime(2021, 4, 3, 12, 4, 28)) + assert hdl.start_time == datetime(2021, 4, 3, 12, 0, 10) + assert hdl.end_time == datetime(2021, 4, 3, 12, 4, 28) @mock.patch('xarray.open_dataset') def test_get_platformname(self, mocked_dataset): @@ -88,8 +88,8 @@ def test_get_platformname(self, mocked_dataset): tmp.rename.return_value = tmp xr.open_dataset.return_value = tmp hdl = VIIRSJRRFileHandler('somedir/somefile.nc', {'platform_shortname': 'npp'}, None) - self.assertEqual(hdl.platform_name, 'Suomi-NPP') + assert hdl.platform_name == 'Suomi-NPP' hdl = VIIRSJRRFileHandler('somedir/somefile.nc', {'platform_shortname': 'JPSS-1'}, None) - self.assertEqual(hdl.platform_name, 'NOAA-20') + assert hdl.platform_name == 'NOAA-20' hdl = VIIRSJRRFileHandler('somedir/somefile.nc', {'platform_shortname': 'J01'}, None) - self.assertEqual(hdl.platform_name, 'NOAA-20') + assert hdl.platform_name == 'NOAA-20' From 60deee1b605e2d00ab99903d6b48c52a52383bdb Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 18 Jul 2023 15:11:36 -0500 Subject: [PATCH 17/59] Add surface reflectance specific tests to viirs_edr --- satpy/etc/readers/viirs_edr.yaml | 6 +- satpy/tests/reader_tests/test_viirs_edr.py | 103 +++++++++++++++------ 2 files changed, 79 insertions(+), 30 deletions(-) diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index 3f268b86b5..bf8a949de8 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -9,17 +9,17 @@ reader: file_types: jrr_cloudmask: - file_reader: !!python/name:satpy.readers.viirs_l2_jrr.VIIRSJRRFileHandler + file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler variable_prefix: "" file_patterns: - 'JRR-CloudMask_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' jrr_aerosol_product: - file_reader: !!python/name:satpy.readers.viirs_l2_jrr.VIIRSJRRFileHandler + file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler variable_prefix: "" file_patterns: - 'JRR-ADP_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' jrr_surfref_product: - file_reader: !!python/name:satpy.readers.viirs_l2_jrr.VIIRSJRRFileHandler + file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler variable_prefix: "" file_patterns: - 'SurfRefl_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index cbd036639f..c84fadf500 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -19,30 +19,89 @@ Note: This is adapted from the test_slstr_l2.py code. """ +from __future__ import annotations from datetime import datetime +from pathlib import Path from unittest import mock from unittest.mock import MagicMock +import numpy as np import pytest import xarray as xr +from pyresample import SwathDefinition from satpy.readers.viirs_edr import VIIRSJRRFileHandler +I_COLS = 64 # real-world 6400 +I_ROWS = 32 # one scan +M_COLS = 32 # real-world 3200 +M_ROWS = 16 # one scan +START_TIME = datetime(2023, 5, 30, 17, 55, 41, 0) +END_TIME = datetime(2023, 5, 30, 17, 57, 5, 0) + + +@pytest.fixture(scope="module") +def surface_reflectance_file(tmp_path_factory) -> Path: + """Generate fake surface reflectance EDR file.""" + tmp_path = tmp_path_factory.mktemp("viirs_edr_tmp") + fn = f"SurfRefl_v1r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202305302025590.nc" + file_path = tmp_path / fn + sr_vars = _create_surf_refl_variables() + ds = _create_fake_dataset(sr_vars) + ds.to_netcdf(file_path) + return file_path + + +def _create_fake_dataset(vars_dict: dict[str, xr.DataArray]) -> xr.Dataset: + ds = xr.Dataset( + vars_dict, + attrs={} + ) + return ds + + +def _create_surf_refl_variables() -> dict[str, xr.DataArray]: + dim_y_750 = "Along_Track_750m" + dim_x_750 = "Along_Scan_750m" + m_dims = (dim_y_750, dim_x_750) + dim_y_375 = "Along_Track_375m" + dim_x_375 = "Along_Scan_375m" + i_dims = (dim_y_375, dim_x_375) + + lon_attrs = {"standard_name": "longitude", "units": "degrees_east", "_FillValue": -999.9} + lat_attrs = {"standard_name": "latitude", "units": "degrees_north", "_FillValue": -999.9} + sr_attrs = {"units": "unitless", "_FillValue": -9999, "scale_factor": 0.0001, "add_offset": 0.0} + + i_data = np.zeros((I_ROWS, I_COLS), dtype=np.float32) + m_data = np.zeros((M_ROWS, M_COLS), dtype=np.float32) + data_arrs = { + "Longitude_at_375m_resolution": xr.DataArray(i_data, dims=i_dims, attrs=lon_attrs), + "Latitude_at_375m_resolution": xr.DataArray(i_data, dims=i_dims, attrs=lat_attrs), + "Longitude_at_750m_resolution": xr.DataArray(i_data, dims=i_dims, attrs=lon_attrs), + "Latitude_at_750m_resolution": xr.DataArray(i_data, dims=i_dims, attrs=lat_attrs), + "375m Surface Reflectance Band I1": xr.DataArray(i_data, dims=i_dims, attrs=sr_attrs), + "750m Surface Reflectance Band M1": xr.DataArray(m_data, dims=m_dims, attrs=sr_attrs), + } + for data_arr in data_arrs.values(): + if "scale_factor" not in data_arr.attrs: + continue + data_arr.encoding["dtype"] = np.int16 + return data_arrs + class TestVIIRSJRRReader: """Test the VIIRS JRR L2 reader.""" - @mock.patch('xarray.open_dataset') - def test_instantiate(self, mocked_dataset): - """Test initialization of file handlers.""" - filename_info = {'platform_shortname': 'npp'} - tmp = MagicMock(start_time='20191120T125002Z', stop_time='20191120T125002Z') - tmp.rename.return_value = tmp - xr.open_dataset.return_value = tmp - VIIRSJRRFileHandler('somedir/somefile.nc', filename_info, None) - mocked_dataset.assert_called() - mocked_dataset.reset_mock() + def test_get_dataset_surf_refl(self, surface_reflectance_file): + """Test retrieval of datasets.""" + from satpy import Scene + scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_file]) + assert scn.start_time == START_TIME + assert scn.end_time == END_TIME + scn.load(["surf_refl_I01", "surf_refl_M01"]) + _check_surf_refl_data_arr(scn["surf_refl_I01"]) + _check_surf_refl_data_arr(scn["surf_refl_M01"]) @mock.patch('xarray.open_dataset') def test_get_dataset(self, mocked_dataset): @@ -55,8 +114,6 @@ def test_get_dataset(self, mocked_dataset): 'Latitude': xr.Dataset(), 'smoke_concentration': xr.Dataset(), 'fire_mask': xr.Dataset(), - 'surf_refl_I01': xr.Dataset(), - 'surf_refl_M05': xr.Dataset(), } test.get_dataset('longitude', {'file_key': 'Longitude'}) test.get_dataset('latitude', {'file_key': 'Latitude'}) @@ -65,21 +122,6 @@ def test_get_dataset(self, mocked_dataset): with pytest.raises(KeyError): test.get_dataset('erroneous dataset', {'file_key': 'erroneous dataset'}) mocked_dataset.assert_called() - mocked_dataset.reset_mock() - test.get_dataset('surf_refl_I01', {'file_key': 'surf_refl_I01'}) - - @mock.patch('xarray.open_dataset') - def test_get_startend_times(self, mocked_dataset): - """Test finding start and end times of granules.""" - filename_info = {'platform_shortname': 'npp', - 'start_time': datetime(2021, 4, 3, 12, 0, 10), - 'end_time': datetime(2021, 4, 3, 12, 4, 28)} - tmp = MagicMock() - tmp.rename.return_value = tmp - xr.open_dataset.return_value = tmp - hdl = VIIRSJRRFileHandler('somedir/somefile.nc', filename_info, None) - assert hdl.start_time == datetime(2021, 4, 3, 12, 0, 10) - assert hdl.end_time == datetime(2021, 4, 3, 12, 4, 28) @mock.patch('xarray.open_dataset') def test_get_platformname(self, mocked_dataset): @@ -93,3 +135,10 @@ def test_get_platformname(self, mocked_dataset): assert hdl.platform_name == 'NOAA-20' hdl = VIIRSJRRFileHandler('somedir/somefile.nc', {'platform_shortname': 'J01'}, None) assert hdl.platform_name == 'NOAA-20' + + +def _check_surf_refl_data_arr(data_arr: xr.DataArray) -> None: + assert data_arr.dims == ("y", "x") + assert isinstance(data_arr.attrs["area"], SwathDefinition) + assert np.issubdtype(data_arr.data.dtype, np.float32) + # TODO: More checks From dc1cf11020a9725dcdfe0aa785e5571628794628 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 18 Jul 2023 20:07:19 -0500 Subject: [PATCH 18/59] Comment out new VIIRS EDR test to debug Windows CI hanging --- satpy/readers/viirs_edr.py | 2 ++ satpy/tests/reader_tests/test_viirs_edr.py | 22 ++++++++++++---------- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 80a4c347c3..a68961be97 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -84,6 +84,8 @@ def __init__(self, filename, filename_info, filetype_info): def get_dataset(self, dataset_id, info): """Get the dataset.""" ds = self.nc[info['file_key']] + if ds.attrs.get("units", None) == "unitless": + ds.attrs["units"] = "1" return ds diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index c84fadf500..95d7bc4563 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -93,15 +93,15 @@ def _create_surf_refl_variables() -> dict[str, xr.DataArray]: class TestVIIRSJRRReader: """Test the VIIRS JRR L2 reader.""" - def test_get_dataset_surf_refl(self, surface_reflectance_file): - """Test retrieval of datasets.""" - from satpy import Scene - scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_file]) - assert scn.start_time == START_TIME - assert scn.end_time == END_TIME - scn.load(["surf_refl_I01", "surf_refl_M01"]) - _check_surf_refl_data_arr(scn["surf_refl_I01"]) - _check_surf_refl_data_arr(scn["surf_refl_M01"]) + # def test_get_dataset_surf_refl(self, surface_reflectance_file): + # """Test retrieval of datasets.""" + # from satpy import Scene + # scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_file]) + # assert scn.start_time == START_TIME + # assert scn.end_time == END_TIME + # scn.load(["surf_refl_I01", "surf_refl_M01"]) + # _check_surf_refl_data_arr(scn["surf_refl_I01"]) + # _check_surf_refl_data_arr(scn["surf_refl_M01"]) @mock.patch('xarray.open_dataset') def test_get_dataset(self, mocked_dataset): @@ -141,4 +141,6 @@ def _check_surf_refl_data_arr(data_arr: xr.DataArray) -> None: assert data_arr.dims == ("y", "x") assert isinstance(data_arr.attrs["area"], SwathDefinition) assert np.issubdtype(data_arr.data.dtype, np.float32) - # TODO: More checks + assert data_arr.attrs["units"] == "1" + exp_shape = (M_ROWS, M_COLS) if "M" in data_arr.attrs["name"] else (I_ROWS, I_COLS) + assert data_arr.shape == exp_shape From 74d45a45ba0d0147c3ba0571d203bff96fa977ee Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 18 Jul 2023 20:45:43 -0500 Subject: [PATCH 19/59] Comment out module scoped fixture to see if it stops Windows CI hanging --- satpy/tests/reader_tests/test_viirs_edr.py | 20 ++++++++++---------- 1 file changed, 10 insertions(+), 10 deletions(-) diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 95d7bc4563..14e7c50bfe 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -41,16 +41,16 @@ END_TIME = datetime(2023, 5, 30, 17, 57, 5, 0) -@pytest.fixture(scope="module") -def surface_reflectance_file(tmp_path_factory) -> Path: - """Generate fake surface reflectance EDR file.""" - tmp_path = tmp_path_factory.mktemp("viirs_edr_tmp") - fn = f"SurfRefl_v1r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202305302025590.nc" - file_path = tmp_path / fn - sr_vars = _create_surf_refl_variables() - ds = _create_fake_dataset(sr_vars) - ds.to_netcdf(file_path) - return file_path +# @pytest.fixture(scope="module") +# def surface_reflectance_file(tmp_path_factory) -> Path: +# """Generate fake surface reflectance EDR file.""" +# tmp_path = tmp_path_factory.mktemp("viirs_edr_tmp") +# fn = f"SurfRefl_v1r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202305302025590.nc" +# file_path = tmp_path / fn +# sr_vars = _create_surf_refl_variables() +# ds = _create_fake_dataset(sr_vars) +# ds.to_netcdf(file_path) +# return file_path def _create_fake_dataset(vars_dict: dict[str, xr.DataArray]) -> xr.Dataset: From d0943807348be4d6815a964808899557fd325210 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 18 Jul 2023 20:48:19 -0500 Subject: [PATCH 20/59] Fix unused import due to commented out code --- satpy/tests/reader_tests/test_viirs_edr.py | 1 - 1 file changed, 1 deletion(-) diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 14e7c50bfe..8f984a323d 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -22,7 +22,6 @@ from __future__ import annotations from datetime import datetime -from pathlib import Path from unittest import mock from unittest.mock import MagicMock From 17148bbffcfff9f837fabb9c8d5a15b5a4619f84 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 20 Jul 2023 12:16:16 -0500 Subject: [PATCH 21/59] Force libnetcdf to a non-hanging build (for now) --- continuous_integration/environment.yaml | 1 + satpy/tests/reader_tests/test_viirs_edr.py | 39 +++++++++++----------- 2 files changed, 21 insertions(+), 19 deletions(-) diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index 48976401a2..46096d7846 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -24,6 +24,7 @@ dependencies: - coverage - codecov - behave + - libnetcdf=4.9.2=nompi_h5902ca5_107 # [win] - netcdf4 - h5py - h5netcdf diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 8f984a323d..43647c4aab 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -22,6 +22,7 @@ from __future__ import annotations from datetime import datetime +from pathlib import Path from unittest import mock from unittest.mock import MagicMock @@ -40,16 +41,16 @@ END_TIME = datetime(2023, 5, 30, 17, 57, 5, 0) -# @pytest.fixture(scope="module") -# def surface_reflectance_file(tmp_path_factory) -> Path: -# """Generate fake surface reflectance EDR file.""" -# tmp_path = tmp_path_factory.mktemp("viirs_edr_tmp") -# fn = f"SurfRefl_v1r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202305302025590.nc" -# file_path = tmp_path / fn -# sr_vars = _create_surf_refl_variables() -# ds = _create_fake_dataset(sr_vars) -# ds.to_netcdf(file_path) -# return file_path +@pytest.fixture(scope="module") +def surface_reflectance_file(tmp_path_factory) -> Path: + """Generate fake surface reflectance EDR file.""" + tmp_path = tmp_path_factory.mktemp("viirs_edr_tmp") + fn = f"SurfRefl_v1r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202305302025590.nc" + file_path = tmp_path / fn + sr_vars = _create_surf_refl_variables() + ds = _create_fake_dataset(sr_vars) + ds.to_netcdf(file_path) + return file_path def _create_fake_dataset(vars_dict: dict[str, xr.DataArray]) -> xr.Dataset: @@ -92,15 +93,15 @@ def _create_surf_refl_variables() -> dict[str, xr.DataArray]: class TestVIIRSJRRReader: """Test the VIIRS JRR L2 reader.""" - # def test_get_dataset_surf_refl(self, surface_reflectance_file): - # """Test retrieval of datasets.""" - # from satpy import Scene - # scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_file]) - # assert scn.start_time == START_TIME - # assert scn.end_time == END_TIME - # scn.load(["surf_refl_I01", "surf_refl_M01"]) - # _check_surf_refl_data_arr(scn["surf_refl_I01"]) - # _check_surf_refl_data_arr(scn["surf_refl_M01"]) + def test_get_dataset_surf_refl(self, surface_reflectance_file): + """Test retrieval of datasets.""" + from satpy import Scene + scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_file]) + assert scn.start_time == START_TIME + assert scn.end_time == END_TIME + scn.load(["surf_refl_I01", "surf_refl_M01"]) + _check_surf_refl_data_arr(scn["surf_refl_I01"]) + _check_surf_refl_data_arr(scn["surf_refl_M01"]) @mock.patch('xarray.open_dataset') def test_get_dataset(self, mocked_dataset): From c9377f768d20f4372050e1d2498c275044c6d801 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 20 Jul 2023 12:38:39 -0500 Subject: [PATCH 22/59] Fix yaml selector for libnetcdf build on windows --- continuous_integration/environment.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index 46096d7846..0cf682e1bb 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -24,7 +24,7 @@ dependencies: - coverage - codecov - behave - - libnetcdf=4.9.2=nompi_h5902ca5_107 # [win] + - libnetcdf=4.9.2=nompi_h5902ca5_107 # [win] - netcdf4 - h5py - h5netcdf From 69645a392f47c6bb1ff97c0a4287d121d58955bf Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 20 Jul 2023 12:58:16 -0500 Subject: [PATCH 23/59] Try environment hack one more time --- .github/workflows/ci.yaml | 3 +++ continuous_integration/environment.yaml | 1 - 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index faa0aea2cc..1c18e4a5cc 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -82,6 +82,9 @@ jobs: - name: Update environment run: mamba env update -n test-environment -f continuous_integration/environment.yaml if: steps.cache.outputs.cache-hit != 'true' + - name: Update environment - libnetcdf + run: mamba install -y -n test-environment libnetcdf=4.9.2=nompi_h5902ca5_107 + if: runner.os == 'Windows' - name: Install unstable dependencies if: matrix.experimental == true diff --git a/continuous_integration/environment.yaml b/continuous_integration/environment.yaml index 0cf682e1bb..48976401a2 100644 --- a/continuous_integration/environment.yaml +++ b/continuous_integration/environment.yaml @@ -24,7 +24,6 @@ dependencies: - coverage - codecov - behave - - libnetcdf=4.9.2=nompi_h5902ca5_107 # [win] - netcdf4 - h5py - h5netcdf From fade8513d193168c61ff36aa2ccce634e20f2242 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 20 Jul 2023 14:06:28 -0500 Subject: [PATCH 24/59] Switch VIIRS EDR to modern chunk sizing --- satpy/readers/viirs_edr.py | 14 ++++++++++---- satpy/tests/reader_tests/test_viirs_edr.py | 10 ++++++++-- 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index a68961be97..f7817833fc 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -47,7 +47,7 @@ import xarray as xr from satpy.readers.file_handlers import BaseFileHandler -from satpy.utils import get_legacy_chunk_size +from satpy.utils import get_chunk_size_limit LOG = logging.getLogger(__name__) @@ -59,12 +59,18 @@ def __init__(self, filename, filename_info, filetype_info): """Initialize the geo filehandler.""" super(VIIRSJRRFileHandler, self).__init__(filename, filename_info, filetype_info) - chunk_size = get_legacy_chunk_size() + chunk_size = get_chunk_size_limit() // 4 # 32-bit floats self.nc = xr.open_dataset(self.filename, decode_cf=True, mask_and_scale=True, - chunks={'Columns': chunk_size, - 'Rows': chunk_size}) + chunks={ + 'Columns': chunk_size, + 'Rows': chunk_size, + 'Along_Scan_375m': chunk_size, + 'Along_Track_375m': chunk_size, + 'Along_Scan_750m': chunk_size, + 'Along_Track_750m': chunk_size, + }) if 'columns' in self.nc.dims: self.nc = self.nc.rename({'Columns': 'x', 'Rows': 'y'}) elif 'Along_Track_375m' in self.nc.dims: diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 43647c4aab..f049a5e288 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -26,6 +26,8 @@ from unittest import mock from unittest.mock import MagicMock +import dask +import dask.array as da import numpy as np import pytest import xarray as xr @@ -96,10 +98,11 @@ class TestVIIRSJRRReader: def test_get_dataset_surf_refl(self, surface_reflectance_file): """Test retrieval of datasets.""" from satpy import Scene - scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_file]) + with dask.config.set({"array.chunk-size": "16B"}): + scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_file]) + scn.load(["surf_refl_I01", "surf_refl_M01"]) assert scn.start_time == START_TIME assert scn.end_time == END_TIME - scn.load(["surf_refl_I01", "surf_refl_M01"]) _check_surf_refl_data_arr(scn["surf_refl_I01"]) _check_surf_refl_data_arr(scn["surf_refl_M01"]) @@ -140,7 +143,10 @@ def test_get_platformname(self, mocked_dataset): def _check_surf_refl_data_arr(data_arr: xr.DataArray) -> None: assert data_arr.dims == ("y", "x") assert isinstance(data_arr.attrs["area"], SwathDefinition) + assert isinstance(data_arr.data, da.Array) assert np.issubdtype(data_arr.data.dtype, np.float32) + assert all(c == 4 for c in data_arr.chunks[0]) + assert all(c == 4 for c in data_arr.chunks[1]) assert data_arr.attrs["units"] == "1" exp_shape = (M_ROWS, M_COLS) if "M" in data_arr.attrs["name"] else (I_ROWS, I_COLS) assert data_arr.shape == exp_shape From af68e101d4701f5b2fbeab2d67787cc93d0de8bf Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 20 Jul 2023 14:20:42 -0500 Subject: [PATCH 25/59] Update VIIRS EDR chunking to be scan-based --- satpy/readers/viirs_edr.py | 16 +++++++++------- satpy/tests/reader_tests/test_viirs_edr.py | 13 ++++++++----- 2 files changed, 17 insertions(+), 12 deletions(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index f7817833fc..8389ea8019 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -59,17 +59,19 @@ def __init__(self, filename, filename_info, filetype_info): """Initialize the geo filehandler.""" super(VIIRSJRRFileHandler, self).__init__(filename, filename_info, filetype_info) - chunk_size = get_chunk_size_limit() // 4 # 32-bit floats + # use entire scans as chunks + row_chunks_m = max(get_chunk_size_limit() // 4 // 3200, 1) # 32-bit floats + row_chunks_i = row_chunks_m * 2 self.nc = xr.open_dataset(self.filename, decode_cf=True, mask_and_scale=True, chunks={ - 'Columns': chunk_size, - 'Rows': chunk_size, - 'Along_Scan_375m': chunk_size, - 'Along_Track_375m': chunk_size, - 'Along_Scan_750m': chunk_size, - 'Along_Track_750m': chunk_size, + 'Columns': -1, + 'Rows': row_chunks_i, + 'Along_Scan_375m': -1, + 'Along_Track_375m': row_chunks_i, + 'Along_Scan_750m': -1, + 'Along_Track_750m': row_chunks_m, }) if 'columns' in self.nc.dims: self.nc = self.nc.rename({'Columns': 'x', 'Rows': 'y'}) diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index f049a5e288..6ab31c6ff6 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -98,7 +98,8 @@ class TestVIIRSJRRReader: def test_get_dataset_surf_refl(self, surface_reflectance_file): """Test retrieval of datasets.""" from satpy import Scene - with dask.config.set({"array.chunk-size": "16B"}): + bytes_in_m_row = 4 * 3200 + with dask.config.set({"array.chunk-size": f"{bytes_in_m_row * 4}B"}): scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_file]) scn.load(["surf_refl_I01", "surf_refl_M01"]) assert scn.start_time == START_TIME @@ -145,8 +146,10 @@ def _check_surf_refl_data_arr(data_arr: xr.DataArray) -> None: assert isinstance(data_arr.attrs["area"], SwathDefinition) assert isinstance(data_arr.data, da.Array) assert np.issubdtype(data_arr.data.dtype, np.float32) - assert all(c == 4 for c in data_arr.chunks[0]) - assert all(c == 4 for c in data_arr.chunks[1]) - assert data_arr.attrs["units"] == "1" - exp_shape = (M_ROWS, M_COLS) if "M" in data_arr.attrs["name"] else (I_ROWS, I_COLS) + is_m_band = "I" not in data_arr.attrs["name"] + exp_shape = (M_ROWS, M_COLS) if is_m_band else (I_ROWS, I_COLS) assert data_arr.shape == exp_shape + exp_row_chunks = 4 if is_m_band else 8 + assert all(c == exp_row_chunks for c in data_arr.chunks[0]) + assert data_arr.chunks[1] == (exp_shape[1],) + assert data_arr.attrs["units"] == "1" From ab555c2de2de52390ece06f78179a6aa251230d3 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 20 Jul 2023 15:00:05 -0500 Subject: [PATCH 26/59] Remove mocking of platform name VIIRS EDR test --- satpy/readers/viirs_edr.py | 1 + satpy/tests/reader_tests/test_viirs_edr.py | 31 +++++++++++++--------- 2 files changed, 19 insertions(+), 13 deletions(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 8389ea8019..d02d38f53e 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -94,6 +94,7 @@ def get_dataset(self, dataset_id, info): ds = self.nc[info['file_key']] if ds.attrs.get("units", None) == "unitless": ds.attrs["units"] = "1" + ds.attrs["platform_name"] = self.platform_name return ds diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 6ab31c6ff6..d1da58970d 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -21,6 +21,7 @@ """ from __future__ import annotations +import shutil from datetime import datetime from pathlib import Path from unittest import mock @@ -107,6 +108,23 @@ def test_get_dataset_surf_refl(self, surface_reflectance_file): _check_surf_refl_data_arr(scn["surf_refl_I01"]) _check_surf_refl_data_arr(scn["surf_refl_M01"]) + @pytest.mark.parametrize( + ("filename_platform", "exp_shortname"), + [ + ("npp", "Suomi-NPP"), + ("JPSS-1", "NOAA-20"), + ("J01", "NOAA-20") + ]) + def test_get_platformname(self, surface_reflectance_file, filename_platform, exp_shortname): + """Test finding start and end times of granules.""" + from satpy import Scene + new_name = str(surface_reflectance_file).replace("npp", filename_platform) + if new_name != str(surface_reflectance_file): + shutil.copy(surface_reflectance_file, new_name) + scn = Scene(reader="viirs_edr", filenames=[new_name]) + scn.load(["surf_refl_I01"]) + assert scn["surf_refl_I01"].attrs["platform_name"] == exp_shortname + @mock.patch('xarray.open_dataset') def test_get_dataset(self, mocked_dataset): """Test retrieval of datasets.""" @@ -127,19 +145,6 @@ def test_get_dataset(self, mocked_dataset): test.get_dataset('erroneous dataset', {'file_key': 'erroneous dataset'}) mocked_dataset.assert_called() - @mock.patch('xarray.open_dataset') - def test_get_platformname(self, mocked_dataset): - """Test finding start and end times of granules.""" - tmp = MagicMock() - tmp.rename.return_value = tmp - xr.open_dataset.return_value = tmp - hdl = VIIRSJRRFileHandler('somedir/somefile.nc', {'platform_shortname': 'npp'}, None) - assert hdl.platform_name == 'Suomi-NPP' - hdl = VIIRSJRRFileHandler('somedir/somefile.nc', {'platform_shortname': 'JPSS-1'}, None) - assert hdl.platform_name == 'NOAA-20' - hdl = VIIRSJRRFileHandler('somedir/somefile.nc', {'platform_shortname': 'J01'}, None) - assert hdl.platform_name == 'NOAA-20' - def _check_surf_refl_data_arr(data_arr: xr.DataArray) -> None: assert data_arr.dims == ("y", "x") From f3ea32f393dfee7ad2302c9ff7a1d8e1362a18ff Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 20 Jul 2023 15:02:00 -0500 Subject: [PATCH 27/59] Remove all-in-one VIIRS EDR test It wasn't doing much and they'll be readded later as new files are supported --- satpy/tests/reader_tests/test_viirs_edr.py | 24 ---------------------- 1 file changed, 24 deletions(-) diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index d1da58970d..a9a7d13a81 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -24,8 +24,6 @@ import shutil from datetime import datetime from pathlib import Path -from unittest import mock -from unittest.mock import MagicMock import dask import dask.array as da @@ -34,8 +32,6 @@ import xarray as xr from pyresample import SwathDefinition -from satpy.readers.viirs_edr import VIIRSJRRFileHandler - I_COLS = 64 # real-world 6400 I_ROWS = 32 # one scan M_COLS = 32 # real-world 3200 @@ -125,26 +121,6 @@ def test_get_platformname(self, surface_reflectance_file, filename_platform, exp scn.load(["surf_refl_I01"]) assert scn["surf_refl_I01"].attrs["platform_name"] == exp_shortname - @mock.patch('xarray.open_dataset') - def test_get_dataset(self, mocked_dataset): - """Test retrieval of datasets.""" - filename_info = {'platform_shortname': 'npp'} - tmp = MagicMock(start_time='20191120T125002Z', stop_time='20191120T125002Z') - xr.open_dataset.return_value = tmp - test = VIIRSJRRFileHandler('somedir/somefile.nc', filename_info, None) - test.nc = {'Longitude': xr.Dataset(), - 'Latitude': xr.Dataset(), - 'smoke_concentration': xr.Dataset(), - 'fire_mask': xr.Dataset(), - } - test.get_dataset('longitude', {'file_key': 'Longitude'}) - test.get_dataset('latitude', {'file_key': 'Latitude'}) - test.get_dataset('smoke_concentration', {'file_key': 'smoke_concentration'}) - test.get_dataset('fire_mask', {'file_key': 'fire_mask'}) - with pytest.raises(KeyError): - test.get_dataset('erroneous dataset', {'file_key': 'erroneous dataset'}) - mocked_dataset.assert_called() - def _check_surf_refl_data_arr(data_arr: xr.DataArray) -> None: assert data_arr.dims == ("y", "x") From a34c9c8079a2406c3232532cd6ac24acfb1e68d5 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 21 Jul 2023 08:58:42 -0500 Subject: [PATCH 28/59] Add NDVI/EVI optional datasets to viirs_edr reader --- satpy/etc/readers/viirs_edr.yaml | 15 ++++ satpy/readers/file_handlers.py | 4 +- satpy/readers/viirs_edr.py | 48 +++++++++-- satpy/tests/reader_tests/test_viirs_edr.py | 99 ++++++++++++++++++++++ 4 files changed, 159 insertions(+), 7 deletions(-) diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index bf8a949de8..ebcea3cd3d 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -409,3 +409,18 @@ datasets: coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 + # Swath-based vegetation indexes added to CSPP LEO output + NDVI: + name: NDVI + resolution: 375 + file_type: [jrr_surfref_product] + file_key: "NDVI" + coordinates: [longitude_375, latitude_375] + units: "1" + EVI: + name: EVI + resolution: 375 + file_type: [jrr_surfref_product] + file_key: "NDVI" + coordinates: [longitude_375, latitude_375] + units: "1" diff --git a/satpy/readers/file_handlers.py b/satpy/readers/file_handlers.py index cebab6e307..0c47553b0d 100644 --- a/satpy/readers/file_handlers.py +++ b/satpy/readers/file_handlers.py @@ -228,9 +228,9 @@ def available_datasets(self, configured_datasets=None): Args: configured_datasets (list): Series of (bool or None, dict) in the same way as is returned by this method (see below). The bool - is whether or not the dataset is available from at least one + is whether the dataset is available from at least one of the current file handlers. It can also be ``None`` if - no file handler knows before us knows how to handle it. + no file handler before us knows how to handle it. The dictionary is existing dataset metadata. The dictionaries are typically provided from a YAML configuration file and may be modified, updated, or used as a "template" for additional diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index d02d38f53e..3ddd685bab 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -91,12 +91,14 @@ def __init__(self, filename, filename_info, filetype_info): def get_dataset(self, dataset_id, info): """Get the dataset.""" - ds = self.nc[info['file_key']] - if ds.attrs.get("units", None) == "unitless": - ds.attrs["units"] = "1" - ds.attrs["platform_name"] = self.platform_name + data_arr = self.nc[info['file_key']] + if data_arr.attrs.get("units", None) == "unitless": + data_arr.attrs["units"] = "1" + if isinstance(data_arr.attrs.get('flag_meanings'), str): + data_arr.attrs['flag_meanings'] = [flag.strip() for flag in data_arr.attrs['flag_meanings'].split(' ')] + data_arr.attrs["platform_name"] = self.platform_name - return ds + return data_arr @property def start_time(self): @@ -118,3 +120,39 @@ def platform_name(self): 'JPSS-2': 'NOAA-21', 'J02': 'NOAA-21'} return platform_dict[platform_path.upper()] + + def available_datasets(self, configured_datasets=None): + """Get information of available datasets in this file. + + Args: + configured_datasets (list): Series of (bool or None, dict) in the + same way as is returned by this method (see below). The bool + is whether the dataset is available from at least one + of the current file handlers. It can also be ``None`` if + no file handler before us knows how to handle it. + The dictionary is existing dataset metadata. The dictionaries + are typically provided from a YAML configuration file and may + be modified, updated, or used as a "template" for additional + available datasets. This argument could be the result of a + previous file handler's implementation of this method. + + Returns: + Iterator of (bool or None, dict) pairs where dict is the + dataset's metadata. If the dataset is available in the current + file type then the boolean value should be ``True``, ``False`` + if we **know** about the dataset but it is unavailable, or + ``None`` if this file object is not responsible for it. + + """ + for is_avail, ds_info in (configured_datasets or []): + if is_avail is not None: + # some other file handler said it has this dataset + # we don't know any more information than the previous + # file handler so let's yield early + yield is_avail, ds_info + continue + if self.file_type_matches(ds_info['file_type']) is None: + # this is not the file type for this dataset + yield None, ds_info + file_key = ds_info.get("file_key", ds_info["name"]) + yield file_key in self.nc, ds_info diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index a9a7d13a81..de95e946ac 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -31,6 +31,7 @@ import pytest import xarray as xr from pyresample import SwathDefinition +from pytest_lazyfixture import lazy_fixture I_COLS = 64 # real-world 6400 I_ROWS = 32 # one scan @@ -38,15 +39,52 @@ M_ROWS = 16 # one scan START_TIME = datetime(2023, 5, 30, 17, 55, 41, 0) END_TIME = datetime(2023, 5, 30, 17, 57, 5, 0) +QF1_FLAG_MEANINGS = """ +\tBits are listed from the MSB (bit 7) to the LSB (bit 0): +\tBit Description +\t6-7 SUN GLINT; +\t 00 -- none +\t 01 -- geometry based +\t 10 -- wind speed based +\t 11 -- geometry & wind speed based +\t5 low sun mask; +\t 0 -- high +\t 1 -- low +\t4 day/night; +\t 0 -- day +\t 1 -- night +\t2-3 cloud detection & confidence; +\t 00 -- confident clear +\t 01 -- probably clear +\t 10 -- probably cloudy +\t 11 -- confident cloudy +\t0-1 cloud mask quality; +\t 00 -- poor +\t 01 -- low +\t 10 -- medium +\t 11 -- high +""" @pytest.fixture(scope="module") def surface_reflectance_file(tmp_path_factory) -> Path: """Generate fake surface reflectance EDR file.""" + return _create_surface_reflectance_file(tmp_path_factory, include_veg_indices=False) + + +@pytest.fixture(scope="module") +def surface_reflectance_with_veg_indices_file(tmp_path_factory) -> Path: + """Generate fake surface reflectance EDR file with vegetation indexes included.""" + return _create_surface_reflectance_file(tmp_path_factory, include_veg_indices=True) + + +def _create_surface_reflectance_file(tmp_path_factory, include_veg_indices: bool = False) -> Path: tmp_path = tmp_path_factory.mktemp("viirs_edr_tmp") fn = f"SurfRefl_v1r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202305302025590.nc" file_path = tmp_path / fn sr_vars = _create_surf_refl_variables() + if include_veg_indices: + sr_vars.update(_create_veg_index_variables()) ds = _create_fake_dataset(sr_vars) ds.to_netcdf(file_path) return file_path @@ -89,6 +127,32 @@ def _create_surf_refl_variables() -> dict[str, xr.DataArray]: return data_arrs +def _create_veg_index_variables() -> dict[str, xr.DataArray]: + dim_y_750 = "Along_Track_750m" + dim_x_750 = "Along_Scan_750m" + m_dims = (dim_y_750, dim_x_750) + dim_y_375 = "Along_Track_375m" + dim_x_375 = "Along_Scan_375m" + i_dims = (dim_y_375, dim_x_375) + + i_data = np.zeros((I_ROWS, I_COLS), dtype=np.float32) + data_arrs = { + "NDVI": xr.DataArray(i_data, dims=i_dims, attrs={"units": "unitless"}), + "EVI": xr.DataArray(i_data, dims=i_dims, attrs={"units": "unitless"}), + } + data_arrs["NDVI"].encoding["dtype"] = np.float32 + data_arrs["EVI"].encoding["dtype"] = np.float32 + + # Quality Flags are from the Surface Reflectance data, but only used for VI products in the reader + qf_data = np.zeros((M_ROWS, M_COLS), dtype=np.uint8) + for qf_num in range(1, 8): + qf_name = f"QF{qf_num} Surface Reflectance" + data_arr = xr.DataArray(qf_data, dims=m_dims, attrs={"flag_meanings": QF1_FLAG_MEANINGS}) + data_arr.encoding["dtype"] = np.uint8 + data_arrs[qf_name] = data_arr + return data_arrs + + class TestVIIRSJRRReader: """Test the VIIRS JRR L2 reader.""" @@ -104,6 +168,34 @@ def test_get_dataset_surf_refl(self, surface_reflectance_file): _check_surf_refl_data_arr(scn["surf_refl_I01"]) _check_surf_refl_data_arr(scn["surf_refl_M01"]) + def test_get_dataset_surf_refl_with_veg_idx(self, surface_reflectance_with_veg_indices_file): + """Test retrieval of vegetation indices from surface reflectance files.""" + from satpy import Scene + scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_with_veg_indices_file]) + scn.load(["NDVI", "EVI", "surf_refl_qf1"]) + _check_surf_refl_qf_data_arr(scn["surf_refl_qf1"]) + # TODO: Check NDVI/EVI attributes/dims + # TODO: Check NDVI/EVI quality flag clearing + + @pytest.mark.parametrize( + ("data_file", "exp_available"), + [ + (lazy_fixture("surface_reflectance_file"), False), + (lazy_fixture("surface_reflectance_with_veg_indices_file"), True), + ] + ) + def test_availability_veg_idx(self, data_file, exp_available): + """Test that vegetation indexes aren't available when they aren't present.""" + from satpy import Scene + scn = Scene(reader="viirs_edr", filenames=[data_file]) + avail = scn.available_dataset_names() + if exp_available: + assert "NDVI" in avail + assert "EVI" in avail + else: + assert "NDVI" not in avail + assert "EVI" not in avail + @pytest.mark.parametrize( ("filename_platform", "exp_shortname"), [ @@ -134,3 +226,10 @@ def _check_surf_refl_data_arr(data_arr: xr.DataArray) -> None: assert all(c == exp_row_chunks for c in data_arr.chunks[0]) assert data_arr.chunks[1] == (exp_shape[1],) assert data_arr.attrs["units"] == "1" + + +def _check_surf_refl_qf_data_arr(data_arr: xr.DataArray) -> None: + assert data_arr.dims == ("y", "x") + assert isinstance(data_arr.attrs["area"], SwathDefinition) + assert isinstance(data_arr.data, da.Array) + assert np.issubdtype(data_arr.data.dtype, np.uint8) From 976c1466b45a2c496dd71749c0cbcdb3d39eca40 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 21 Jul 2023 09:10:46 -0500 Subject: [PATCH 29/59] Cleanup VIIRS EDR testing --- satpy/readers/viirs_edr.py | 14 ++++++++---- satpy/tests/reader_tests/test_viirs_edr.py | 25 +++++++++++----------- 2 files changed, 23 insertions(+), 16 deletions(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 3ddd685bab..29a3d414d1 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -92,14 +92,20 @@ def __init__(self, filename, filename_info, filetype_info): def get_dataset(self, dataset_id, info): """Get the dataset.""" data_arr = self.nc[info['file_key']] - if data_arr.attrs.get("units", None) == "unitless": + units = data_arr.attrs.get("units", None) + if units is None or units == "unitless": data_arr.attrs["units"] = "1" - if isinstance(data_arr.attrs.get('flag_meanings'), str): - data_arr.attrs['flag_meanings'] = [flag.strip() for flag in data_arr.attrs['flag_meanings'].split(' ')] + self._decode_flag_meanings(data_arr) data_arr.attrs["platform_name"] = self.platform_name - return data_arr + @staticmethod + def _decode_flag_meanings(data_arr: xr.DataArray): + flag_meanings = data_arr.attrs.get("flag_meanings", None) + if isinstance(flag_meanings, str) and "\n" not in flag_meanings: + # only handle CF-standard flag meanings + data_arr.attrs['flag_meanings'] = [flag for flag in data_arr.attrs['flag_meanings'].split(' ')] + @property def start_time(self): """Get first date/time when observations were recorded.""" diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index de95e946ac..06bd49aba5 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -28,6 +28,7 @@ import dask import dask.array as da import numpy as np +import numpy.typing as npt import pytest import xarray as xr from pyresample import SwathDefinition @@ -171,10 +172,13 @@ def test_get_dataset_surf_refl(self, surface_reflectance_file): def test_get_dataset_surf_refl_with_veg_idx(self, surface_reflectance_with_veg_indices_file): """Test retrieval of vegetation indices from surface reflectance files.""" from satpy import Scene - scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_with_veg_indices_file]) - scn.load(["NDVI", "EVI", "surf_refl_qf1"]) + bytes_in_m_row = 4 * 3200 + with dask.config.set({"array.chunk-size": f"{bytes_in_m_row * 4}B"}): + scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_with_veg_indices_file]) + scn.load(["NDVI", "EVI", "surf_refl_qf1"]) + _check_surf_refl_data_arr(scn["NDVI"]) + _check_surf_refl_data_arr(scn["EVI"]) _check_surf_refl_qf_data_arr(scn["surf_refl_qf1"]) - # TODO: Check NDVI/EVI attributes/dims # TODO: Check NDVI/EVI quality flag clearing @pytest.mark.parametrize( @@ -214,22 +218,19 @@ def test_get_platformname(self, surface_reflectance_file, filename_platform, exp assert scn["surf_refl_I01"].attrs["platform_name"] == exp_shortname -def _check_surf_refl_data_arr(data_arr: xr.DataArray) -> None: +def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.float32) -> None: assert data_arr.dims == ("y", "x") assert isinstance(data_arr.attrs["area"], SwathDefinition) assert isinstance(data_arr.data, da.Array) - assert np.issubdtype(data_arr.data.dtype, np.float32) - is_m_band = "I" not in data_arr.attrs["name"] - exp_shape = (M_ROWS, M_COLS) if is_m_band else (I_ROWS, I_COLS) + assert np.issubdtype(data_arr.data.dtype, dtype) + is_mband_res = "I" not in data_arr.attrs["name"] # includes NDVI and EVI + exp_shape = (M_ROWS, M_COLS) if is_mband_res else (I_ROWS, I_COLS) assert data_arr.shape == exp_shape - exp_row_chunks = 4 if is_m_band else 8 + exp_row_chunks = 4 if is_mband_res else 8 assert all(c == exp_row_chunks for c in data_arr.chunks[0]) assert data_arr.chunks[1] == (exp_shape[1],) assert data_arr.attrs["units"] == "1" def _check_surf_refl_qf_data_arr(data_arr: xr.DataArray) -> None: - assert data_arr.dims == ("y", "x") - assert isinstance(data_arr.attrs["area"], SwathDefinition) - assert isinstance(data_arr.data, da.Array) - assert np.issubdtype(data_arr.data.dtype, np.uint8) + _check_surf_refl_data_arr(data_arr, dtype=np.uint8) From 1db0e4855f2b82b2ea38456524fe346344f8f3f0 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 21 Jul 2023 09:47:35 -0500 Subject: [PATCH 30/59] Fix sensor in VIIRS EDR --- satpy/readers/viirs_edr.py | 1 + satpy/tests/reader_tests/test_viirs_edr.py | 2 ++ 2 files changed, 3 insertions(+) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 29a3d414d1..2d26b4dd35 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -97,6 +97,7 @@ def get_dataset(self, dataset_id, info): data_arr.attrs["units"] = "1" self._decode_flag_meanings(data_arr) data_arr.attrs["platform_name"] = self.platform_name + data_arr.attrs["sensor"] = self.sensor_name return data_arr @staticmethod diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 06bd49aba5..be1cf8dc6a 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -229,7 +229,9 @@ def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.floa exp_row_chunks = 4 if is_mband_res else 8 assert all(c == exp_row_chunks for c in data_arr.chunks[0]) assert data_arr.chunks[1] == (exp_shape[1],) + assert data_arr.attrs["units"] == "1" + assert data_arr.attrs["sensor"] == "viirs" def _check_surf_refl_qf_data_arr(data_arr: xr.DataArray) -> None: From 7b183ca7147c970076a3fa7032002f878cbfdedf Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 21 Jul 2023 16:04:24 -0500 Subject: [PATCH 31/59] Fix fill value handling and add valid_range YAML handling for VI products --- satpy/etc/readers/viirs_edr.yaml | 11 +++------- satpy/readers/viirs_edr.py | 15 +++++++++++++ satpy/tests/reader_tests/test_viirs_edr.py | 25 ++++++++++++++-------- 3 files changed, 34 insertions(+), 17 deletions(-) diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index ebcea3cd3d..23d07f4e07 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -360,7 +360,6 @@ datasets: file_key: "QF1 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' - _FillValue: -9999 surf_refl_qf2: name: surf_refl_qf2 resolution: 750 @@ -368,7 +367,6 @@ datasets: file_key: "QF2 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' - _FillValue: -9999 surf_refl_qf3: name: surf_refl_qf3 resolution: 750 @@ -376,7 +374,6 @@ datasets: file_key: "QF3 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' - _FillValue: -9999 surf_refl_qf4: name: surf_refl_qf4 resolution: 750 @@ -384,7 +381,6 @@ datasets: file_key: "QF4 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' - _FillValue: -9999 surf_refl_qf5: name: surf_refl_qf5 resolution: 750 @@ -392,7 +388,6 @@ datasets: file_key: "QF5 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' - _FillValue: -9999 surf_refl_qf6: name: surf_refl_qf6 resolution: 750 @@ -400,7 +395,6 @@ datasets: file_key: "QF6 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' - _FillValue: -9999 surf_refl_qf7: name: surf_refl_qf7 resolution: 750 @@ -408,7 +402,6 @@ datasets: file_key: "QF7 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' - _FillValue: -9999 # Swath-based vegetation indexes added to CSPP LEO output NDVI: name: NDVI @@ -417,10 +410,12 @@ datasets: file_key: "NDVI" coordinates: [longitude_375, latitude_375] units: "1" + valid_range: [-1.0, 1.0] EVI: name: EVI resolution: 375 file_type: [jrr_surfref_product] - file_key: "NDVI" + file_key: "EVI" coordinates: [longitude_375, latitude_375] units: "1" + valid_range: [-1.0, 1.0] diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 2d26b4dd35..dc58cd406e 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -44,6 +44,7 @@ import logging +import numpy as np import xarray as xr from satpy.readers.file_handlers import BaseFileHandler @@ -92,6 +93,7 @@ def __init__(self, filename, filename_info, filetype_info): def get_dataset(self, dataset_id, info): """Get the dataset.""" data_arr = self.nc[info['file_key']] + data_arr = self._mask_invalid(data_arr, info) units = data_arr.attrs.get("units", None) if units is None or units == "unitless": data_arr.attrs["units"] = "1" @@ -100,6 +102,19 @@ def get_dataset(self, dataset_id, info): data_arr.attrs["sensor"] = self.sensor_name return data_arr + def _mask_invalid(self, data_arr: xr.DataArray, ds_info: dict) -> xr.DataArray: + fill_value = data_arr.encoding.get("_FillValue") + if fill_value is not None and not np.isnan(fill_value): + # xarray auto mask and scale handled this + return data_arr + yaml_fill = ds_info.get("_FillValue") + if yaml_fill is not None: + return data_arr.where(data_arr != yaml_fill) + valid_range = ds_info.get("valid_range", data_arr.attrs.get("valid_range")) + if valid_range is not None: + return data_arr.where((valid_range[0] <= data_arr) & (data_arr <= valid_range[1])) + return data_arr + @staticmethod def _decode_flag_meanings(data_arr: xr.DataArray): flag_meanings = data_arr.attrs.get("flag_meanings", None) diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index be1cf8dc6a..69d03f6db9 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -136,10 +136,11 @@ def _create_veg_index_variables() -> dict[str, xr.DataArray]: dim_x_375 = "Along_Scan_375m" i_dims = (dim_y_375, dim_x_375) - i_data = np.zeros((I_ROWS, I_COLS), dtype=np.float32) + vi_data = np.zeros((I_ROWS, I_COLS), dtype=np.float32) + vi_data[0, :7] = [-2.0, -1.0, -0.5, 0.0, 0.5, 1.0, 1.5] data_arrs = { - "NDVI": xr.DataArray(i_data, dims=i_dims, attrs={"units": "unitless"}), - "EVI": xr.DataArray(i_data, dims=i_dims, attrs={"units": "unitless"}), + "NDVI": xr.DataArray(vi_data, dims=i_dims, attrs={"units": "unitless"}), + "EVI": xr.DataArray(vi_data, dims=i_dims, attrs={"units": "unitless"}), } data_arrs["NDVI"].encoding["dtype"] = np.float32 data_arrs["EVI"].encoding["dtype"] = np.float32 @@ -176,8 +177,8 @@ def test_get_dataset_surf_refl_with_veg_idx(self, surface_reflectance_with_veg_i with dask.config.set({"array.chunk-size": f"{bytes_in_m_row * 4}B"}): scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_with_veg_indices_file]) scn.load(["NDVI", "EVI", "surf_refl_qf1"]) - _check_surf_refl_data_arr(scn["NDVI"]) - _check_surf_refl_data_arr(scn["EVI"]) + _check_vi_data_arr(scn["NDVI"]) + _check_vi_data_arr(scn["EVI"]) _check_surf_refl_qf_data_arr(scn["surf_refl_qf1"]) # TODO: Check NDVI/EVI quality flag clearing @@ -218,6 +219,16 @@ def test_get_platformname(self, surface_reflectance_file, filename_platform, exp assert scn["surf_refl_I01"].attrs["platform_name"] == exp_shortname +def _check_surf_refl_qf_data_arr(data_arr: xr.DataArray) -> None: + _check_surf_refl_data_arr(data_arr, dtype=np.uint8) + + +def _check_vi_data_arr(data_arr: xr.DataArray) -> None: + _check_surf_refl_data_arr(data_arr) + data = data_arr.data.compute() + np.testing.assert_allclose(data[0, :7], [np.nan, -1.0, -0.5, 0.0, 0.5, 1.0, np.nan]) + + def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.float32) -> None: assert data_arr.dims == ("y", "x") assert isinstance(data_arr.attrs["area"], SwathDefinition) @@ -232,7 +243,3 @@ def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.floa assert data_arr.attrs["units"] == "1" assert data_arr.attrs["sensor"] == "viirs" - - -def _check_surf_refl_qf_data_arr(data_arr: xr.DataArray) -> None: - _check_surf_refl_data_arr(data_arr, dtype=np.uint8) From 2864f98e2c37f3acc868c09008ac43d5688eafb7 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sat, 22 Jul 2023 14:15:57 -0500 Subject: [PATCH 32/59] Add basic vegetation quality masking --- satpy/readers/viirs_edr.py | 35 ++++++++++++++++++++++ satpy/tests/reader_tests/test_viirs_edr.py | 20 +++++++++++-- 2 files changed, 53 insertions(+), 2 deletions(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index dc58cd406e..df151a1fd9 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -94,6 +94,9 @@ def get_dataset(self, dataset_id, info): """Get the dataset.""" data_arr = self.nc[info['file_key']] data_arr = self._mask_invalid(data_arr, info) + if info["file_key"] in ("NDVI", "EVI"): + good_mask = self._get_veg_index_good_mask() + data_arr = data_arr.where(good_mask) units = data_arr.attrs.get("units", None) if units is None or units == "unitless": data_arr.attrs["units"] = "1" @@ -115,6 +118,38 @@ def _mask_invalid(self, data_arr: xr.DataArray, ds_info: dict) -> xr.DataArray: return data_arr.where((valid_range[0] <= data_arr) & (data_arr <= valid_range[1])) return data_arr + def _get_veg_index_good_mask(self) -> xr.DataArray: + # each mask array should be TRUE when pixels are UNACCEPTABLE + qf1 = self.nc['QF1 Surface Reflectance'] + has_sun_glint = (qf1 & 0b11000000) > 0 + is_cloudy = (qf1 & 0b00001100) > 0 # mask everything but "confident clear" + cloud_quality = (qf1 & 0b00000011) < 0b10 + + qf2 = self.nc['QF2 Surface Reflectance'] + has_snow_or_ice = (qf2 & 0b00100000) > 0 + has_cloud_shadow = (qf2 & 0b00001000) > 0 + water_mask = (qf2 & 0b00000111) + has_water = (water_mask <= 0b010) | (water_mask == 0b101) # shallow water, deep ocean, arctic + + qf7 = self.nc['QF7 Surface Reflectance'] + has_aerosols = (qf7 & 0b00001100) > 0b1000 # high aerosol quantity + adjacent_to_cloud = (qf7 & 0b00000010) > 0 + + bad_mask = ( + has_sun_glint | + is_cloudy | + cloud_quality | + has_snow_or_ice | + has_cloud_shadow | + has_water | + has_aerosols | + adjacent_to_cloud + ) + # upscale from M-band resolution to I-band resolution + bad_mask_iband_dask = bad_mask.data.repeat(2, axis=1).repeat(2, axis=0) + good_mask_iband = xr.DataArray(~bad_mask_iband_dask, dims=qf1.dims) + return good_mask_iband + @staticmethod def _decode_flag_meanings(data_arr: xr.DataArray): flag_meanings = data_arr.attrs.get("flag_meanings", None) diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 69d03f6db9..6d380cd017 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -146,9 +146,24 @@ def _create_veg_index_variables() -> dict[str, xr.DataArray]: data_arrs["EVI"].encoding["dtype"] = np.float32 # Quality Flags are from the Surface Reflectance data, but only used for VI products in the reader - qf_data = np.zeros((M_ROWS, M_COLS), dtype=np.uint8) for qf_num in range(1, 8): qf_name = f"QF{qf_num} Surface Reflectance" + qf_data = np.zeros((M_ROWS, M_COLS), dtype=np.uint8) + bad_qf_start = 4 # 0.5x the last test pixel set in "vi_data" above (I-band versus M-band index) + if qf_num == 1: + qf_data[:, :] |= 0b00000010 # medium cloud mask quality everywhere + qf_data[0, bad_qf_start] |= 0b11000000 # sun glint + qf_data[0, bad_qf_start + 1] |= 0b00001100 # cloudy + qf_data[0, bad_qf_start + 2] = 0b00000001 # low cloud mask quality + elif qf_num == 2: + qf_data[:, :] |= 0b00000011 # desert everywhere + qf_data[0, bad_qf_start + 3] |= 0b00100000 # snow or ice + qf_data[0, bad_qf_start + 4] |= 0b00001000 # cloud shadow + qf_data[0, bad_qf_start + 5] = 0b00000001 # deep ocean + elif qf_num == 7: + qf_data[0, bad_qf_start + 6] |= 0b00001100 # high aerosol + qf_data[0, bad_qf_start + 7] |= 0b00000010 # adjacent to cloud + data_arr = xr.DataArray(qf_data, dims=m_dims, attrs={"flag_meanings": QF1_FLAG_MEANINGS}) data_arr.encoding["dtype"] = np.uint8 data_arrs[qf_name] = data_arr @@ -180,7 +195,6 @@ def test_get_dataset_surf_refl_with_veg_idx(self, surface_reflectance_with_veg_i _check_vi_data_arr(scn["NDVI"]) _check_vi_data_arr(scn["EVI"]) _check_surf_refl_qf_data_arr(scn["surf_refl_qf1"]) - # TODO: Check NDVI/EVI quality flag clearing @pytest.mark.parametrize( ("data_file", "exp_available"), @@ -227,6 +241,8 @@ def _check_vi_data_arr(data_arr: xr.DataArray) -> None: _check_surf_refl_data_arr(data_arr) data = data_arr.data.compute() np.testing.assert_allclose(data[0, :7], [np.nan, -1.0, -0.5, 0.0, 0.5, 1.0, np.nan]) + np.testing.assert_allclose(data[0, 8:8 + 16], np.nan) + np.testing.assert_allclose(data[0, 8 + 16:], 0.0) def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.float32) -> None: From b9a5f4c321a33db1ad06f3c261680e03ad1891e7 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sat, 22 Jul 2023 14:29:12 -0500 Subject: [PATCH 33/59] Refactor viirs edr reader to surface reflectance is separate --- satpy/etc/readers/viirs_edr.yaml | 2 +- satpy/readers/viirs_edr.py | 78 ++++++++++++++++++-------------- 2 files changed, 44 insertions(+), 36 deletions(-) diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index 23d07f4e07..9e362c18af 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -19,7 +19,7 @@ file_types: file_patterns: - 'JRR-ADP_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' jrr_surfref_product: - file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler + file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSSurfaceReflectanceWithVIHandler variable_prefix: "" file_patterns: - 'SurfRefl_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index df151a1fd9..9ad841754d 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -94,9 +94,6 @@ def get_dataset(self, dataset_id, info): """Get the dataset.""" data_arr = self.nc[info['file_key']] data_arr = self._mask_invalid(data_arr, info) - if info["file_key"] in ("NDVI", "EVI"): - good_mask = self._get_veg_index_good_mask() - data_arr = data_arr.where(good_mask) units = data_arr.attrs.get("units", None) if units is None or units == "unitless": data_arr.attrs["units"] = "1" @@ -118,38 +115,6 @@ def _mask_invalid(self, data_arr: xr.DataArray, ds_info: dict) -> xr.DataArray: return data_arr.where((valid_range[0] <= data_arr) & (data_arr <= valid_range[1])) return data_arr - def _get_veg_index_good_mask(self) -> xr.DataArray: - # each mask array should be TRUE when pixels are UNACCEPTABLE - qf1 = self.nc['QF1 Surface Reflectance'] - has_sun_glint = (qf1 & 0b11000000) > 0 - is_cloudy = (qf1 & 0b00001100) > 0 # mask everything but "confident clear" - cloud_quality = (qf1 & 0b00000011) < 0b10 - - qf2 = self.nc['QF2 Surface Reflectance'] - has_snow_or_ice = (qf2 & 0b00100000) > 0 - has_cloud_shadow = (qf2 & 0b00001000) > 0 - water_mask = (qf2 & 0b00000111) - has_water = (water_mask <= 0b010) | (water_mask == 0b101) # shallow water, deep ocean, arctic - - qf7 = self.nc['QF7 Surface Reflectance'] - has_aerosols = (qf7 & 0b00001100) > 0b1000 # high aerosol quantity - adjacent_to_cloud = (qf7 & 0b00000010) > 0 - - bad_mask = ( - has_sun_glint | - is_cloudy | - cloud_quality | - has_snow_or_ice | - has_cloud_shadow | - has_water | - has_aerosols | - adjacent_to_cloud - ) - # upscale from M-band resolution to I-band resolution - bad_mask_iband_dask = bad_mask.data.repeat(2, axis=1).repeat(2, axis=0) - good_mask_iband = xr.DataArray(~bad_mask_iband_dask, dims=qf1.dims) - return good_mask_iband - @staticmethod def _decode_flag_meanings(data_arr: xr.DataArray): flag_meanings = data_arr.attrs.get("flag_meanings", None) @@ -213,3 +178,46 @@ def available_datasets(self, configured_datasets=None): yield None, ds_info file_key = ds_info.get("file_key", ds_info["name"]) yield file_key in self.nc, ds_info + + +class VIIRSSurfaceReflectanceWithVIHandler(VIIRSJRRFileHandler): + """File handler for surface reflectance files with optional vegetation indexes.""" + + def _mask_invalid(self, data_arr: xr.DataArray, ds_info: dict) -> xr.DataArray: + new_data_arr = super()._mask_invalid(data_arr, ds_info) + if ds_info["file_key"] in ("NDVI", "EVI"): + good_mask = self._get_veg_index_good_mask() + new_data_arr = new_data_arr.where(good_mask) + return new_data_arr + + def _get_veg_index_good_mask(self) -> xr.DataArray: + # each mask array should be TRUE when pixels are UNACCEPTABLE + qf1 = self.nc['QF1 Surface Reflectance'] + has_sun_glint = (qf1 & 0b11000000) > 0 + is_cloudy = (qf1 & 0b00001100) > 0 # mask everything but "confident clear" + cloud_quality = (qf1 & 0b00000011) < 0b10 + + qf2 = self.nc['QF2 Surface Reflectance'] + has_snow_or_ice = (qf2 & 0b00100000) > 0 + has_cloud_shadow = (qf2 & 0b00001000) > 0 + water_mask = (qf2 & 0b00000111) + has_water = (water_mask <= 0b010) | (water_mask == 0b101) # shallow water, deep ocean, arctic + + qf7 = self.nc['QF7 Surface Reflectance'] + has_aerosols = (qf7 & 0b00001100) > 0b1000 # high aerosol quantity + adjacent_to_cloud = (qf7 & 0b00000010) > 0 + + bad_mask = ( + has_sun_glint | + is_cloudy | + cloud_quality | + has_snow_or_ice | + has_cloud_shadow | + has_water | + has_aerosols | + adjacent_to_cloud + ) + # upscale from M-band resolution to I-band resolution + bad_mask_iband_dask = bad_mask.data.repeat(2, axis=1).repeat(2, axis=0) + good_mask_iband = xr.DataArray(~bad_mask_iband_dask, dims=qf1.dims) + return good_mask_iband From 5b1573fc6f7cf794fb45c388362e9b86ef59d6b8 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sat, 22 Jul 2023 15:26:22 -0500 Subject: [PATCH 34/59] Add rows_per_scan to viirs_edr metadata --- satpy/readers/viirs_edr.py | 5 +++++ satpy/tests/reader_tests/test_viirs_edr.py | 5 +++-- 2 files changed, 8 insertions(+), 2 deletions(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 9ad841754d..472e4c29ea 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -90,6 +90,10 @@ def __init__(self, filename, filename_info, filetype_info): self.algorithm_version = filename_info['platform_shortname'] self.sensor_name = 'viirs' + def rows_per_scans(self, data_arr: xr.DataArray) -> int: + """Get number of array rows per instrument scan based on data resolution.""" + return 32 if data_arr.shape[1] == 6400 else 16 + def get_dataset(self, dataset_id, info): """Get the dataset.""" data_arr = self.nc[info['file_key']] @@ -100,6 +104,7 @@ def get_dataset(self, dataset_id, info): self._decode_flag_meanings(data_arr) data_arr.attrs["platform_name"] = self.platform_name data_arr.attrs["sensor"] = self.sensor_name + data_arr.attrs["rows_per_scan"] = self.rows_per_scans(data_arr) return data_arr def _mask_invalid(self, data_arr: xr.DataArray, ds_info: dict) -> xr.DataArray: diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 6d380cd017..7f2659ec61 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -34,9 +34,9 @@ from pyresample import SwathDefinition from pytest_lazyfixture import lazy_fixture -I_COLS = 64 # real-world 6400 +I_COLS = 6400 I_ROWS = 32 # one scan -M_COLS = 32 # real-world 3200 +M_COLS = 3200 M_ROWS = 16 # one scan START_TIME = datetime(2023, 5, 30, 17, 55, 41, 0) END_TIME = datetime(2023, 5, 30, 17, 57, 5, 0) @@ -259,3 +259,4 @@ def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.floa assert data_arr.attrs["units"] == "1" assert data_arr.attrs["sensor"] == "viirs" + assert data_arr.attrs["rows_per_scan"] == 16 if is_mband_res else 32 From d7121bb1315360ac44c384435784785d32cb5530 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sun, 23 Jul 2023 06:43:34 -0500 Subject: [PATCH 35/59] Add standard_names from YAML --- satpy/etc/readers/viirs_edr.yaml | 22 ++++++++++++++++++++ satpy/readers/viirs_edr.py | 5 ++++- satpy/tests/reader_tests/test_viirs_edr.py | 24 +++++++++++++++++++--- 3 files changed, 47 insertions(+), 4 deletions(-) diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index 9e362c18af..b93c2fdd3d 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -254,6 +254,7 @@ datasets: coordinates: [longitude_375, latitude_375] units: '1' _FillValue: -9999 + standard_name: "surface_bidirectional_reflectance" surf_refl_I02: name: surf_refl_I02 resolution: 375 @@ -263,6 +264,7 @@ datasets: coordinates: [longitude_375, latitude_375] units: '1' _FillValue: -9999 + standard_name: "surface_bidirectional_reflectance" surf_refl_I03: name: surf_refl_I03 resolution: 375 @@ -272,6 +274,7 @@ datasets: coordinates: [longitude_375, latitude_375] units: '1' _FillValue: -9999 + standard_name: "surface_bidirectional_reflectance" surf_refl_M01: name: surf_refl_M01 resolution: 750 @@ -281,6 +284,7 @@ datasets: coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 + standard_name: "surface_bidirectional_reflectance" surf_refl_M02: name: surf_refl_M02 resolution: 750 @@ -290,6 +294,7 @@ datasets: coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 + standard_name: "surface_bidirectional_reflectance" surf_refl_M03: name: surf_refl_M03 resolution: 750 @@ -299,6 +304,7 @@ datasets: coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 + standard_name: "surface_bidirectional_reflectance" surf_refl_M04: name: surf_refl_M04 resolution: 750 @@ -308,6 +314,7 @@ datasets: coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 + standard_name: "surface_bidirectional_reflectance" surf_refl_M05: name: surf_refl_M05 resolution: 750 @@ -317,6 +324,7 @@ datasets: coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 + standard_name: "surface_bidirectional_reflectance" surf_refl_M06: name: surf_refl_M06 resolution: 750 @@ -326,6 +334,7 @@ datasets: coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 + standard_name: "surface_bidirectional_reflectance" surf_refl_M07: name: surf_refl_M07 resolution: 750 @@ -335,6 +344,7 @@ datasets: coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 + standard_name: "surface_bidirectional_reflectance" surf_refl_M08: name: surf_refl_M08 resolution: 750 @@ -344,6 +354,7 @@ datasets: coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 + standard_name: "surface_bidirectional_reflectance" surf_refl_M10: name: surf_refl_M10 resolution: 750 @@ -353,6 +364,7 @@ datasets: coordinates: [longitude_750, latitude_750] units: '1' _FillValue: -9999 + standard_name: "surface_bidirectional_reflectance" surf_refl_qf1: name: surf_refl_qf1 resolution: 750 @@ -360,6 +372,7 @@ datasets: file_key: "QF1 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' + standard_name: "quality_flag" surf_refl_qf2: name: surf_refl_qf2 resolution: 750 @@ -367,6 +380,7 @@ datasets: file_key: "QF2 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' + standard_name: "quality_flag" surf_refl_qf3: name: surf_refl_qf3 resolution: 750 @@ -374,6 +388,7 @@ datasets: file_key: "QF3 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' + standard_name: "quality_flag" surf_refl_qf4: name: surf_refl_qf4 resolution: 750 @@ -381,6 +396,7 @@ datasets: file_key: "QF4 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' + standard_name: "quality_flag" surf_refl_qf5: name: surf_refl_qf5 resolution: 750 @@ -388,6 +404,7 @@ datasets: file_key: "QF5 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' + standard_name: "quality_flag" surf_refl_qf6: name: surf_refl_qf6 resolution: 750 @@ -395,6 +412,7 @@ datasets: file_key: "QF6 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' + standard_name: "quality_flag" surf_refl_qf7: name: surf_refl_qf7 resolution: 750 @@ -402,6 +420,8 @@ datasets: file_key: "QF7 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' + standard_name: "quality_flag" + # Swath-based vegetation indexes added to CSPP LEO output NDVI: name: NDVI @@ -411,6 +431,7 @@ datasets: coordinates: [longitude_375, latitude_375] units: "1" valid_range: [-1.0, 1.0] + standard_name: "normalized_difference_vegetation_index" EVI: name: EVI resolution: 375 @@ -419,3 +440,4 @@ datasets: coordinates: [longitude_375, latitude_375] units: "1" valid_range: [-1.0, 1.0] + standard_name: "normalized_difference_vegetation_index" diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 472e4c29ea..ee9ba66953 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -98,9 +98,12 @@ def get_dataset(self, dataset_id, info): """Get the dataset.""" data_arr = self.nc[info['file_key']] data_arr = self._mask_invalid(data_arr, info) - units = data_arr.attrs.get("units", None) + units = info.get("units", data_arr.attrs.get("units", None)) if units is None or units == "unitless": data_arr.attrs["units"] = "1" + data_arr.attrs["units"] = units + if "standard_name" in info: + data_arr.attrs["standard_name"] = info["standard_name"] self._decode_flag_meanings(data_arr) data_arr.attrs["platform_name"] = self.platform_name data_arr.attrs["sensor"] = self.sensor_name diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 7f2659ec61..ff1b646737 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -234,11 +234,16 @@ def test_get_platformname(self, surface_reflectance_file, filename_platform, exp def _check_surf_refl_qf_data_arr(data_arr: xr.DataArray) -> None: - _check_surf_refl_data_arr(data_arr, dtype=np.uint8) + _array_checks(data_arr, dtype=np.uint8) + _shared_metadata_checks(data_arr) + assert data_arr.attrs["standard_name"] == "quality_flag" def _check_vi_data_arr(data_arr: xr.DataArray) -> None: - _check_surf_refl_data_arr(data_arr) + _array_checks(data_arr) + _shared_metadata_checks(data_arr) + assert data_arr.attrs["standard_name"] == "normalized_difference_vegetation_index" + data = data_arr.data.compute() np.testing.assert_allclose(data[0, :7], [np.nan, -1.0, -0.5, 0.0, 0.5, 1.0, np.nan]) np.testing.assert_allclose(data[0, 8:8 + 16], np.nan) @@ -246,17 +251,30 @@ def _check_vi_data_arr(data_arr: xr.DataArray) -> None: def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.float32) -> None: + _array_checks(data_arr, dtype) + _shared_metadata_checks(data_arr) + assert data_arr.attrs["standard_name"] == "surface_bidirectional_reflectance" + + +def _array_checks(data_arr: xr.DataArray, dtype: npt.Dtype = np.float32) -> None: assert data_arr.dims == ("y", "x") assert isinstance(data_arr.attrs["area"], SwathDefinition) assert isinstance(data_arr.data, da.Array) assert np.issubdtype(data_arr.data.dtype, dtype) - is_mband_res = "I" not in data_arr.attrs["name"] # includes NDVI and EVI + is_mband_res = _is_mband_res(data_arr) exp_shape = (M_ROWS, M_COLS) if is_mband_res else (I_ROWS, I_COLS) assert data_arr.shape == exp_shape exp_row_chunks = 4 if is_mband_res else 8 assert all(c == exp_row_chunks for c in data_arr.chunks[0]) assert data_arr.chunks[1] == (exp_shape[1],) + +def _shared_metadata_checks(data_arr: xr.DataArray) -> None: + is_mband_res = _is_mband_res(data_arr) assert data_arr.attrs["units"] == "1" assert data_arr.attrs["sensor"] == "viirs" assert data_arr.attrs["rows_per_scan"] == 16 if is_mband_res else 32 + + +def _is_mband_res(data_arr: xr.DataArray) -> bool: + return "I" not in data_arr.attrs["name"] # includes NDVI and EVI From 891c4c70ae5f0f4267c7e3132835478a427d902f Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sun, 23 Jul 2023 07:06:57 -0500 Subject: [PATCH 36/59] Change surface reflectances to percentage for consistency --- satpy/etc/enhancements/generic.yaml | 9 ++++++++ satpy/etc/readers/viirs_edr.yaml | 24 +++++++++++----------- satpy/readers/viirs_edr.py | 9 +++++--- satpy/tests/reader_tests/test_viirs_edr.py | 13 +++++++++--- 4 files changed, 37 insertions(+), 18 deletions(-) diff --git a/satpy/etc/enhancements/generic.yaml b/satpy/etc/enhancements/generic.yaml index 37b375f36c..967f47e2f1 100644 --- a/satpy/etc/enhancements/generic.yaml +++ b/satpy/etc/enhancements/generic.yaml @@ -15,6 +15,15 @@ enhancements: - name: gamma method: !!python/name:satpy.enhancements.gamma kwargs: {gamma: 1.5} + surface_reflectance_default: + standard_name: surface_bidirectional_reflectance + operations: + - name: linear_stretch + method: !!python/name:satpy.enhancements.stretch + kwargs: {stretch: 'crude', min_stretch: 0.0, max_stretch: 100.} + - name: gamma + method: !!python/name:satpy.enhancements.gamma + kwargs: {gamma: 1.5} true_color_default: standard_name: true_color operations: diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index b93c2fdd3d..fc6d49d2cf 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -252,7 +252,7 @@ datasets: file_type: [jrr_surfref_product] file_key: "375m Surface Reflectance Band I1" coordinates: [longitude_375, latitude_375] - units: '1' + units: '%' _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_I02: @@ -262,7 +262,7 @@ datasets: file_type: [jrr_surfref_product] file_key: "375m Surface Reflectance Band I2" coordinates: [longitude_375, latitude_375] - units: '1' + units: '%' _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_I03: @@ -272,7 +272,7 @@ datasets: file_type: [jrr_surfref_product] file_key: "375m Surface Reflectance Band I3" coordinates: [longitude_375, latitude_375] - units: '1' + units: '%' _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M01: @@ -282,7 +282,7 @@ datasets: file_type: [jrr_surfref_product] file_key: "750m Surface Reflectance Band M1" coordinates: [longitude_750, latitude_750] - units: '1' + units: '%' _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M02: @@ -292,7 +292,7 @@ datasets: file_type: [jrr_surfref_product] file_key: "750m Surface Reflectance Band M2" coordinates: [longitude_750, latitude_750] - units: '1' + units: '%' _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M03: @@ -302,7 +302,7 @@ datasets: file_type: [jrr_surfref_product] file_key: "750m Surface Reflectance Band M3" coordinates: [longitude_750, latitude_750] - units: '1' + units: '%' _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M04: @@ -312,7 +312,7 @@ datasets: file_type: [jrr_surfref_product] file_key: "750m Surface Reflectance Band M4" coordinates: [longitude_750, latitude_750] - units: '1' + units: '%' _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M05: @@ -322,7 +322,7 @@ datasets: file_type: [jrr_surfref_product] file_key: "750m Surface Reflectance Band M5" coordinates: [longitude_750, latitude_750] - units: '1' + units: '%' _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M06: @@ -332,7 +332,7 @@ datasets: file_type: [jrr_surfref_product] file_key: "750m Surface Reflectance Band M6" coordinates: [longitude_750, latitude_750] - units: '1' + units: '%' _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M07: @@ -342,7 +342,7 @@ datasets: file_type: [jrr_surfref_product] file_key: "750m Surface Reflectance Band M7" coordinates: [longitude_750, latitude_750] - units: '1' + units: '%' _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M08: @@ -352,7 +352,7 @@ datasets: file_type: [jrr_surfref_product] file_key: "750m Surface Reflectance Band M8" coordinates: [longitude_750, latitude_750] - units: '1' + units: '%' _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M10: @@ -362,7 +362,7 @@ datasets: file_type: [jrr_surfref_product] file_key: "750m Surface Reflectance Band M10" coordinates: [longitude_750, latitude_750] - units: '1' + units: '%' _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_qf1: diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index ee9ba66953..64edbd48c1 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -47,6 +47,7 @@ import numpy as np import xarray as xr +from satpy import DataID from satpy.readers.file_handlers import BaseFileHandler from satpy.utils import get_chunk_size_limit @@ -94,13 +95,15 @@ def rows_per_scans(self, data_arr: xr.DataArray) -> int: """Get number of array rows per instrument scan based on data resolution.""" return 32 if data_arr.shape[1] == 6400 else 16 - def get_dataset(self, dataset_id, info): + def get_dataset(self, dataset_id: DataID, info: dict) -> xr.DataArray: """Get the dataset.""" data_arr = self.nc[info['file_key']] data_arr = self._mask_invalid(data_arr, info) - units = info.get("units", data_arr.attrs.get("units", None)) + units = info.get("units", data_arr.attrs.get("units")) if units is None or units == "unitless": - data_arr.attrs["units"] = "1" + units = "1" + if units == "%" and data_arr.attrs.get("units") in ("1", "unitless"): + data_arr *= 100.0 # turn into percentages data_arr.attrs["units"] = units if "standard_name" in info: data_arr.attrs["standard_name"] = info["standard_name"] diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index ff1b646737..6a40778505 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -111,8 +111,8 @@ def _create_surf_refl_variables() -> dict[str, xr.DataArray]: lat_attrs = {"standard_name": "latitude", "units": "degrees_north", "_FillValue": -999.9} sr_attrs = {"units": "unitless", "_FillValue": -9999, "scale_factor": 0.0001, "add_offset": 0.0} - i_data = np.zeros((I_ROWS, I_COLS), dtype=np.float32) - m_data = np.zeros((M_ROWS, M_COLS), dtype=np.float32) + i_data = np.random.random_sample((I_ROWS, I_COLS)).astype(np.float32) + m_data = np.random.random_sample((M_ROWS, M_COLS)).astype(np.float32) data_arrs = { "Longitude_at_375m_resolution": xr.DataArray(i_data, dims=i_dims, attrs=lon_attrs), "Latitude_at_375m_resolution": xr.DataArray(i_data, dims=i_dims, attrs=lat_attrs), @@ -125,6 +125,8 @@ def _create_surf_refl_variables() -> dict[str, xr.DataArray]: if "scale_factor" not in data_arr.attrs: continue data_arr.encoding["dtype"] = np.int16 + data_arr.encoding["scale_factor"] = data_arr.attrs.pop("scale_factor") + data_arr.encoding["add_offset"] = data_arr.attrs.pop("add_offset") return data_arrs @@ -236,12 +238,14 @@ def test_get_platformname(self, surface_reflectance_file, filename_platform, exp def _check_surf_refl_qf_data_arr(data_arr: xr.DataArray) -> None: _array_checks(data_arr, dtype=np.uint8) _shared_metadata_checks(data_arr) + assert data_arr.attrs["units"] == "1" assert data_arr.attrs["standard_name"] == "quality_flag" def _check_vi_data_arr(data_arr: xr.DataArray) -> None: _array_checks(data_arr) _shared_metadata_checks(data_arr) + assert data_arr.attrs["units"] == "1" assert data_arr.attrs["standard_name"] == "normalized_difference_vegetation_index" data = data_arr.data.compute() @@ -252,7 +256,11 @@ def _check_vi_data_arr(data_arr: xr.DataArray) -> None: def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.float32) -> None: _array_checks(data_arr, dtype) + data = data_arr.data.compute() + assert data.max() > 1.0 # random 0-1 test data multiplied by 100 + _shared_metadata_checks(data_arr) + assert data_arr.attrs["units"] == "%" assert data_arr.attrs["standard_name"] == "surface_bidirectional_reflectance" @@ -271,7 +279,6 @@ def _array_checks(data_arr: xr.DataArray, dtype: npt.Dtype = np.float32) -> None def _shared_metadata_checks(data_arr: xr.DataArray) -> None: is_mband_res = _is_mband_res(data_arr) - assert data_arr.attrs["units"] == "1" assert data_arr.attrs["sensor"] == "viirs" assert data_arr.attrs["rows_per_scan"] == 16 if is_mband_res else 32 From e4ee636ece198e976fc7cff34d49f8ab4be3b6a5 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sun, 23 Jul 2023 20:29:23 -0500 Subject: [PATCH 37/59] Fix true color surface name Addresses reviewer comment --- satpy/etc/composites/viirs.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/etc/composites/viirs.yaml b/satpy/etc/composites/viirs.yaml index 994a09c960..9c7269862b 100644 --- a/satpy/etc/composites/viirs.yaml +++ b/satpy/etc/composites/viirs.yaml @@ -322,7 +322,7 @@ composites: - name: surf_refl_M05 standard_name: natural_color - true_color_mband_nocorr: + true_color_mband_surf_nocorr: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - name: surf_refl_M05 From fee489f5b58c85a44dc4851ab498eb94f0908932 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sun, 23 Jul 2023 20:31:09 -0500 Subject: [PATCH 38/59] Remove redundant night_overview for VIIRS Closes #1964 --- satpy/etc/composites/viirs.yaml | 8 -------- 1 file changed, 8 deletions(-) diff --git a/satpy/etc/composites/viirs.yaml b/satpy/etc/composites/viirs.yaml index 9c7269862b..541c4dff10 100644 --- a/satpy/etc/composites/viirs.yaml +++ b/satpy/etc/composites/viirs.yaml @@ -352,14 +352,6 @@ composites: modifiers: [sunz_corrected] standard_name: true_color - night_overview: - compositor: !!python/name:satpy.composites.GenericCompositor - prerequisites: - - DNB - - DNB - - M15 - standard_name: night_overview - overview: compositor: !!python/name:satpy.composites.GenericCompositor prerequisites: From ca587f9e77ed209ed29e81cc4a534be34c7ecbe4 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Sun, 23 Jul 2023 21:10:24 -0500 Subject: [PATCH 39/59] Rename surface reflectance based composites and add sharpened true color --- satpy/etc/composites/viirs.yaml | 17 ++++++++++++++--- 1 file changed, 14 insertions(+), 3 deletions(-) diff --git a/satpy/etc/composites/viirs.yaml b/satpy/etc/composites/viirs.yaml index 541c4dff10..6fcdad4e1e 100644 --- a/satpy/etc/composites/viirs.yaml +++ b/satpy/etc/composites/viirs.yaml @@ -306,7 +306,7 @@ composites: modifiers: [sunz_corrected_iband] standard_name: natural_color - natural_color_iband_surf_nocorr: + natural_color_surf: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - name: surf_refl_I03 @@ -314,7 +314,7 @@ composites: - name: surf_refl_I01 standard_name: natural_color - natural_color_mband_surf_nocorr: + natural_color_lowres_surf: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - name: surf_refl_M10 @@ -322,7 +322,7 @@ composites: - name: surf_refl_M05 standard_name: natural_color - true_color_mband_surf_nocorr: + true_color_lowres_surf: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: - name: surf_refl_M05 @@ -330,6 +330,17 @@ composites: - name: surf_refl_M03 standard_name: true_color + true_color_surf: + compositor: !!python/name:satpy.composites.RatioSharpenedRGB + prerequisites: + - name: surf_refl_M05 + - name: surf_refl_M04 + - name: surf_refl_M03 + optional_prerequisites: + - name: surf_refl_I01 + standard_name: true_color + high_resolution_band: red + natural_color_sun_lowres: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: From a913e73cd161ff819a4856e167630cfb55989a6d Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 24 Jul 2023 10:38:52 -0500 Subject: [PATCH 40/59] Remove inaccurate VIIRS EDR mask information --- satpy/etc/readers/viirs_edr.yaml | 60 ++------------------------------ 1 file changed, 2 insertions(+), 58 deletions(-) diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index fc6d49d2cf..3ef014a88e 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -77,172 +77,116 @@ datasets: file_type: jrr_cloudmask file_key: "CloudMask" coordinates: [longitude, latitude] - units: '1' - flag_meanings: ['Clear', 'Probably Clear', 'Probably Cloudy', 'Cloudy'] - flag_values: [0, 1, 2, 3] - _FillValue: -128 cloud_mask_binary: name: cloud_mask_binary resolution: 750 file_type: [jrr_cloudmask] file_key: "CloudMaskBinary" coordinates: [longitude, latitude] - units: '1' - flag_meanings: ['Clear', 'Cloudy'] - flag_values: [0, 1] - _FillValue: -128 cloud_probability: name: cloud_probability resolution: 750 file_type: [jrr_cloudmask] file_key: "CloudProbability" coordinates: [longitude, latitude] - units: '1' - _FillValue: -999. dust_mask: name: dust_mask resolution: 750 file_type: [jrr_cloudmask] file_key: "Dust_Mask" coordinates: [longitude, latitude] - units: '1' - flag_meanings: ['Clear', 'Dusty'] - flag_values: [0, 1] - _FillValue: -128 fire_mask: name: fire_mask resolution: 750 file_type: [jrr_cloudmask] file_key: "Fire_Mask" coordinates: [longitude, latitude] - units: '1' - flag_meanings: ['No fire', 'Fire'] - flag_values: [0, 1] - _FillValue: -128 smoke_mask: name: smoke_mask resolution: 750 file_type: [jrr_cloudmask] file_key: "Smoke_Mask" coordinates: [longitude, latitude] - units: '1' - flag_meanings: ['Clear', 'Smoky'] - flag_values: [0, 1] - _FillValue: -128 - # Aerosol optical depth product datasets + # Aerosol detection product datasets ash_mask: name: ash_mask resolution: 750 file_type: [jrr_aerosol_product] file_key: "Ash" coordinates: [longitude, latitude] - units: '1' - flag_meanings: ['Clear', 'Ash'] - flag_values: [0, 1] - _FillValue: -128 cloud_mask_adp: name: cloud_mask_adp resolution: 750 file_type: [jrr_aerosol_product] file_key: "Cloud" coordinates: [longitude, latitude] - units: '1' - flag_meanings: ['Clear', 'Probably Clear', 'Probably Cloudy', 'Cloudy'] - flag_values: [0, 1, 2, 3] - _FillValue: -128 dust_smoke_discrimination_index: name: dust_smoke_discrimination_index resolution: 750 file_type: [jrr_aerosol_product] file_key: "DSDI" coordinates: [longitude, latitude] - units: '1' - _FillValue: -999 nuc: name: nuc resolution: 750 file_type: [jrr_aerosol_product] file_key: "NUC" coordinates: [longitude, latitude] - units: '1' - flag_meanings: ['No', 'Yes'] - flag_values: [0, 1] - _FillValue: -128 pqi1: name: pqi1 resolution: 750 file_type: [jrr_aerosol_product] file_key: "PQI1" coordinates: [longitude, latitude] - units: '1' - _FillValue: -128 pqi2: name: pqi2 resolution: 750 file_type: [jrr_aerosol_product] file_key: "PQI2" coordinates: [longitude, latitude] - units: '1' - _FillValue: -128 pqi3: name: pqi3 resolution: 750 file_type: [jrr_aerosol_product] file_key: "PQI3" coordinates: [longitude, latitude] - units: '1' - _FillValue: -128 pqi4: name: pqi4 resolution: 750 file_type: [jrr_aerosol_product] file_key: "PQI4" coordinates: [longitude, latitude] - units: '1' - _FillValue: -128 qcflag: name: qcflag resolution: 750 file_type: [jrr_aerosol_product] file_key: "QC_Flag" coordinates: [longitude, latitude] - units: '1' - _FillValue: -128 saai: name: saai resolution: 750 file_type: [jrr_aerosol_product] file_key: "SAAI" coordinates: [longitude, latitude] - units: '1' - _FillValue: -999 smoke: name: smoke resolution: 750 file_type: [jrr_aerosol_product] file_key: "Smoke" coordinates: [longitude, latitude] - units: '1' - _FillValue: -999 smoke_concentration: name: smoke_concentration resolution: 750 file_type: [jrr_aerosol_product] file_key: "SmokeCon" coordinates: [longitude, latitude] - units: 'ug/m^3' - _FillValue: -999 snow_ice: name: snow_ice resolution: 750 file_type: [jrr_aerosol_product] file_key: "SnowIce" coordinates: [longitude, latitude] - units: '1' - flag_meanings: ['No', 'Yes'] - flag_values: [0, 1] - _FillValue: -128 # Surface reflectance products surf_refl_I01: @@ -422,7 +366,7 @@ datasets: units: '1' standard_name: "quality_flag" - # Swath-based vegetation indexes added to CSPP LEO output + # Swath-based vegetation indexes added to CSPP LEO surface reflectance files NDVI: name: NDVI resolution: 375 From a3f1f4e152dd712f69a192435d901d234341c59d Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 24 Jul 2023 12:24:23 -0500 Subject: [PATCH 41/59] Removal of unnecessary fill values in YAML --- satpy/etc/readers/viirs_edr.yaml | 12 ------------ 1 file changed, 12 deletions(-) diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index 3ef014a88e..645c22a898 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -197,7 +197,6 @@ datasets: file_key: "375m Surface Reflectance Band I1" coordinates: [longitude_375, latitude_375] units: '%' - _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_I02: name: surf_refl_I02 @@ -207,7 +206,6 @@ datasets: file_key: "375m Surface Reflectance Band I2" coordinates: [longitude_375, latitude_375] units: '%' - _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_I03: name: surf_refl_I03 @@ -217,7 +215,6 @@ datasets: file_key: "375m Surface Reflectance Band I3" coordinates: [longitude_375, latitude_375] units: '%' - _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M01: name: surf_refl_M01 @@ -227,7 +224,6 @@ datasets: file_key: "750m Surface Reflectance Band M1" coordinates: [longitude_750, latitude_750] units: '%' - _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M02: name: surf_refl_M02 @@ -237,7 +233,6 @@ datasets: file_key: "750m Surface Reflectance Band M2" coordinates: [longitude_750, latitude_750] units: '%' - _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M03: name: surf_refl_M03 @@ -247,7 +242,6 @@ datasets: file_key: "750m Surface Reflectance Band M3" coordinates: [longitude_750, latitude_750] units: '%' - _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M04: name: surf_refl_M04 @@ -257,7 +251,6 @@ datasets: file_key: "750m Surface Reflectance Band M4" coordinates: [longitude_750, latitude_750] units: '%' - _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M05: name: surf_refl_M05 @@ -267,7 +260,6 @@ datasets: file_key: "750m Surface Reflectance Band M5" coordinates: [longitude_750, latitude_750] units: '%' - _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M06: name: surf_refl_M06 @@ -277,7 +269,6 @@ datasets: file_key: "750m Surface Reflectance Band M6" coordinates: [longitude_750, latitude_750] units: '%' - _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M07: name: surf_refl_M07 @@ -287,7 +278,6 @@ datasets: file_key: "750m Surface Reflectance Band M7" coordinates: [longitude_750, latitude_750] units: '%' - _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M08: name: surf_refl_M08 @@ -297,7 +287,6 @@ datasets: file_key: "750m Surface Reflectance Band M8" coordinates: [longitude_750, latitude_750] units: '%' - _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_M10: name: surf_refl_M10 @@ -307,7 +296,6 @@ datasets: file_key: "750m Surface Reflectance Band M10" coordinates: [longitude_750, latitude_750] units: '%' - _FillValue: -9999 standard_name: "surface_bidirectional_reflectance" surf_refl_qf1: name: surf_refl_qf1 From c6f6cc616c1b3d74d4dcc6619d822677703eb752 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 24 Jul 2023 13:12:52 -0500 Subject: [PATCH 42/59] Add missing M11 surface reflectance product and Polar2Grid false_color_surf product --- satpy/etc/composites/viirs.yaml | 16 ++++++++++++++-- satpy/etc/readers/viirs_edr.yaml | 9 +++++++++ 2 files changed, 23 insertions(+), 2 deletions(-) diff --git a/satpy/etc/composites/viirs.yaml b/satpy/etc/composites/viirs.yaml index 6fcdad4e1e..bebf6c5833 100644 --- a/satpy/etc/composites/viirs.yaml +++ b/satpy/etc/composites/viirs.yaml @@ -231,10 +231,10 @@ composites: - name: M05 modifiers: [sunz_corrected, rayleigh_corrected] optional_prerequisites: - - name: I01 + - name: I02 modifiers: [sunz_corrected_iband, rayleigh_corrected_iband] standard_name: false_color - high_resolution_band: blue + high_resolution_band: green fire_temperature: # CIRA: Original VIIRS @@ -322,6 +322,18 @@ composites: - name: surf_refl_M05 standard_name: natural_color + false_color_surf: + compositor: !!python/name:satpy.composites.RatioSharpenedRGB + prerequisites: + - name: surf_refl_M11 + - name: surf_refl_M07 + - name: surf_refl_M05 + optional_prerequisites: + - name: surf_refl_I02 + standard_name: false_color + high_resolution_band: green + + true_color_lowres_surf: compositor: !!python/name:satpy.composites.RGBCompositor prerequisites: diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index 645c22a898..953c508660 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -297,6 +297,15 @@ datasets: coordinates: [longitude_750, latitude_750] units: '%' standard_name: "surface_bidirectional_reflectance" + surf_refl_M11: + name: surf_refl_M11 + resolution: 750 + wavelength: [2.225, 2.250, 2.275] + file_type: [jrr_surfref_product] + file_key: "750m Surface Reflectance Band M11" + coordinates: [longitude_750, latitude_750] + units: '%' + standard_name: "surface_bidirectional_reflectance" surf_refl_qf1: name: surf_refl_qf1 resolution: 750 From 25b4f51207fb055f7d37ac63be7adbecb7a0e753 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 24 Jul 2023 14:03:48 -0500 Subject: [PATCH 43/59] Add a few cloud height VIIRS EDR products --- satpy/etc/readers/viirs_edr.yaml | 29 ++++++++- satpy/readers/viirs_edr.py | 4 +- satpy/tests/reader_tests/test_viirs_edr.py | 69 +++++++++++++++++++--- 3 files changed, 90 insertions(+), 12 deletions(-) diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index 953c508660..8bcc094b3b 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -23,6 +23,11 @@ file_types: variable_prefix: "" file_patterns: - 'SurfRefl_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' + jrr_cloudheight_product: + file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler + variable_prefix: "" + file_patterns: + - 'JRR-CloudHeight_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' datasets: @@ -30,14 +35,14 @@ datasets: longitude: name: longitude standard_name: longitude - file_type: [jrr_cloudmask, jrr_aerosol_product] + file_type: [jrr_cloudmask, jrr_aerosol_product, jrr_cloudheight_product] file_key: "Longitude" units: 'degrees_east' resolution: 750 latitude: name: latitude standard_name: latitude - file_type: [jrr_cloudmask, jrr_aerosol_product] + file_type: [jrr_cloudmask, jrr_aerosol_product, jrr_cloudheight_product] file_key: "Latitude" units: 'degrees_north' resolution: 750 @@ -382,3 +387,23 @@ datasets: units: "1" valid_range: [-1.0, 1.0] standard_name: "normalized_difference_vegetation_index" + + # Cloud Height products + cloud_top_temp: + name: CldTopTemp + file_key: "CldTopTemp" + file_type: [jrr_cloudheight_product] + resolution: 750 + coordinates: [longitude, latitude] + cloud_top_height: + name: CldTopHght + file_key: "CldTopHght" + file_type: [jrr_cloudheight_product] + resolution: 750 + coordinates: [longitude, latitude] + cloud_top_pressure: + name: CldTopPres + file_key: "CldTopPres" + file_type: [jrr_cloudheight_product] + resolution: 750 + coordinates: [longitude, latitude] diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 64edbd48c1..6dececa9c1 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -69,13 +69,13 @@ def __init__(self, filename, filename_info, filetype_info): mask_and_scale=True, chunks={ 'Columns': -1, - 'Rows': row_chunks_i, + 'Rows': row_chunks_m, 'Along_Scan_375m': -1, 'Along_Track_375m': row_chunks_i, 'Along_Scan_750m': -1, 'Along_Track_750m': row_chunks_m, }) - if 'columns' in self.nc.dims: + if 'Columns' in self.nc.dims: self.nc = self.nc.rename({'Columns': 'x', 'Rows': 'y'}) elif 'Along_Track_375m' in self.nc.dims: self.nc = self.nc.rename({'Along_Scan_375m': 'x', 'Along_Track_375m': 'y'}) diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 6a40778505..459f058495 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -91,14 +91,6 @@ def _create_surface_reflectance_file(tmp_path_factory, include_veg_indices: bool return file_path -def _create_fake_dataset(vars_dict: dict[str, xr.DataArray]) -> xr.Dataset: - ds = xr.Dataset( - vars_dict, - attrs={} - ) - return ds - - def _create_surf_refl_variables() -> dict[str, xr.DataArray]: dim_y_750 = "Along_Track_750m" dim_x_750 = "Along_Scan_750m" @@ -172,6 +164,51 @@ def _create_veg_index_variables() -> dict[str, xr.DataArray]: return data_arrs +@pytest.fixture(scope="module") +def cloud_height_file(tmp_path_factory) -> Path: + """Generate fake CloudHeight VIIRS EDR file.""" + tmp_path = tmp_path_factory.mktemp("viirs_edr_tmp") + fn = f"JRR-CloudHeight_v3r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202307231023395.nc" + file_path = tmp_path / fn + ch_vars = _create_cloud_height_variables() + ds = _create_fake_dataset(ch_vars) + ds.to_netcdf(file_path) + return file_path + + +def _create_cloud_height_variables() -> dict[str, xr.DataArray]: + dims = ("Rows", "Columns") + + lon_attrs = {"standard_name": "longitude", "units": "degrees_east", "_FillValue": -999.9} + lat_attrs = {"standard_name": "latitude", "units": "degrees_north", "_FillValue": -999.9} + cont_attrs = {"units": "Kelvin", "_FillValue": -9999, "scale_factor": 0.0001, "add_offset": 0.0} + + m_data = np.random.random_sample((M_ROWS, M_COLS)).astype(np.float32) + data_arrs = { + "Longitude": xr.DataArray(m_data, dims=dims, attrs=lon_attrs), + "Latitude": xr.DataArray(m_data, dims=dims, attrs=lat_attrs), + } + for var_name in ("CldTopTemp", "CldTopHght", "CldTopPres"): + data_arrs[var_name] = xr.DataArray(m_data, dims=dims, attrs=cont_attrs) + for data_arr in data_arrs.values(): + if "_FillValue" in data_arr.attrs: + data_arr.encoding["_FillValue"] = data_arr.attrs.pop("_FillValue") + if "scale_factor" not in data_arr.attrs: + continue + data_arr.encoding["dtype"] = np.int16 + data_arr.encoding["scale_factor"] = data_arr.attrs.pop("scale_factor") + data_arr.encoding["add_offset"] = data_arr.attrs.pop("add_offset") + return data_arrs + + +def _create_fake_dataset(vars_dict: dict[str, xr.DataArray]) -> xr.Dataset: + ds = xr.Dataset( + vars_dict, + attrs={} + ) + return ds + + class TestVIIRSJRRReader: """Test the VIIRS JRR L2 reader.""" @@ -198,6 +235,17 @@ def test_get_dataset_surf_refl_with_veg_idx(self, surface_reflectance_with_veg_i _check_vi_data_arr(scn["EVI"]) _check_surf_refl_qf_data_arr(scn["surf_refl_qf1"]) + def test_get_dataset_cloud_height(self, cloud_height_file): + """Test datasets from cloud height files.""" + from satpy import Scene + bytes_in_m_row = 4 * 3200 + with dask.config.set({"array.chunk-size": f"{bytes_in_m_row * 4}B"}): + scn = Scene(reader="viirs_edr", filenames=[cloud_height_file]) + scn.load(["CldTopTemp", "CldTopHght", "CldTopPres"]) + _check_cloud_height_data_arr(scn["CldTopTemp"]) + _check_cloud_height_data_arr(scn["CldTopHght"]) + _check_cloud_height_data_arr(scn["CldTopPres"]) + @pytest.mark.parametrize( ("data_file", "exp_available"), [ @@ -264,6 +312,11 @@ def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.floa assert data_arr.attrs["standard_name"] == "surface_bidirectional_reflectance" +def _check_cloud_height_data_arr(data_arr: xr.DataArray) -> None: + _array_checks(data_arr) + _shared_metadata_checks(data_arr) + + def _array_checks(data_arr: xr.DataArray, dtype: npt.Dtype = np.float32) -> None: assert data_arr.dims == ("y", "x") assert isinstance(data_arr.attrs["area"], SwathDefinition) From c0b6e83c7316b1d9e78d96d874012148c8f667cf Mon Sep 17 00:00:00 2001 From: David Hoese Date: Mon, 24 Jul 2023 15:08:18 -0500 Subject: [PATCH 44/59] Add VIIRs EDR AOD product --- satpy/etc/readers/viirs_edr.yaml | 17 ++++++- satpy/tests/reader_tests/test_viirs_edr.py | 59 ++++++++++++++-------- 2 files changed, 54 insertions(+), 22 deletions(-) diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index 8bcc094b3b..0004751f5d 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -28,6 +28,11 @@ file_types: variable_prefix: "" file_patterns: - 'JRR-CloudHeight_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' + jrr_aod_product: + file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler + variable_prefix: "" + file_patterns: + - 'JRR-AOD_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' datasets: @@ -35,14 +40,14 @@ datasets: longitude: name: longitude standard_name: longitude - file_type: [jrr_cloudmask, jrr_aerosol_product, jrr_cloudheight_product] + file_type: [jrr_cloudmask, jrr_aerosol_product, jrr_cloudheight_product, jrr_aod_product] file_key: "Longitude" units: 'degrees_east' resolution: 750 latitude: name: latitude standard_name: latitude - file_type: [jrr_cloudmask, jrr_aerosol_product, jrr_cloudheight_product] + file_type: [jrr_cloudmask, jrr_aerosol_product, jrr_cloudheight_product, jrr_aod_product] file_key: "Latitude" units: 'degrees_north' resolution: 750 @@ -407,3 +412,11 @@ datasets: file_type: [jrr_cloudheight_product] resolution: 750 coordinates: [longitude, latitude] + + # Aerosol Optical Depth products + aod550: + name: AOD550 + file_key: AOD550 + file_type: [jrr_aod_product] + resolution: 750 + coordinates: [longitude, latitude] diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 459f058495..53d24e7d11 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -24,6 +24,7 @@ import shutil from datetime import datetime from pathlib import Path +from typing import Iterable import dask import dask.array as da @@ -80,15 +81,11 @@ def surface_reflectance_with_veg_indices_file(tmp_path_factory) -> Path: def _create_surface_reflectance_file(tmp_path_factory, include_veg_indices: bool = False) -> Path: - tmp_path = tmp_path_factory.mktemp("viirs_edr_tmp") fn = f"SurfRefl_v1r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202305302025590.nc" - file_path = tmp_path / fn sr_vars = _create_surf_refl_variables() if include_veg_indices: sr_vars.update(_create_veg_index_variables()) - ds = _create_fake_dataset(sr_vars) - ds.to_netcdf(file_path) - return file_path + return _create_fake_file(tmp_path_factory, fn, sr_vars) def _create_surf_refl_variables() -> dict[str, xr.DataArray]: @@ -167,16 +164,24 @@ def _create_veg_index_variables() -> dict[str, xr.DataArray]: @pytest.fixture(scope="module") def cloud_height_file(tmp_path_factory) -> Path: """Generate fake CloudHeight VIIRS EDR file.""" - tmp_path = tmp_path_factory.mktemp("viirs_edr_tmp") fn = f"JRR-CloudHeight_v3r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202307231023395.nc" - file_path = tmp_path / fn - ch_vars = _create_cloud_height_variables() - ds = _create_fake_dataset(ch_vars) - ds.to_netcdf(file_path) - return file_path + data_vars = _create_continuous_variables( + ("CldTopTemp", "CldTopHght", "CldTopPres") + ) + return _create_fake_file(tmp_path_factory, fn, data_vars) + + +@pytest.fixture(scope="module") +def aod_file(tmp_path_factory) -> Path: + """Generate fake AOD VIIRs EDR file.""" + fn = f"JRR-AOD_v3r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202307231023395.nc" + data_vars = _create_continuous_variables( + ("AOD550",) + ) + return _create_fake_file(tmp_path_factory, fn, data_vars) -def _create_cloud_height_variables() -> dict[str, xr.DataArray]: +def _create_continuous_variables(var_names: Iterable[str]) -> dict[str, xr.DataArray]: dims = ("Rows", "Columns") lon_attrs = {"standard_name": "longitude", "units": "degrees_east", "_FillValue": -999.9} @@ -188,7 +193,7 @@ def _create_cloud_height_variables() -> dict[str, xr.DataArray]: "Longitude": xr.DataArray(m_data, dims=dims, attrs=lon_attrs), "Latitude": xr.DataArray(m_data, dims=dims, attrs=lat_attrs), } - for var_name in ("CldTopTemp", "CldTopHght", "CldTopPres"): + for var_name in var_names: data_arrs[var_name] = xr.DataArray(m_data, dims=dims, attrs=cont_attrs) for data_arr in data_arrs.values(): if "_FillValue" in data_arr.attrs: @@ -201,6 +206,14 @@ def _create_cloud_height_variables() -> dict[str, xr.DataArray]: return data_arrs +def _create_fake_file(tmp_path_factory, filename: str, data_arrs: dict[str, xr.DataArray]) -> Path: + tmp_path = tmp_path_factory.mktemp("viirs_edr_tmp") + file_path = tmp_path / filename + ds = _create_fake_dataset(data_arrs) + ds.to_netcdf(file_path) + return file_path + + def _create_fake_dataset(vars_dict: dict[str, xr.DataArray]) -> xr.Dataset: ds = xr.Dataset( vars_dict, @@ -235,16 +248,22 @@ def test_get_dataset_surf_refl_with_veg_idx(self, surface_reflectance_with_veg_i _check_vi_data_arr(scn["EVI"]) _check_surf_refl_qf_data_arr(scn["surf_refl_qf1"]) - def test_get_dataset_cloud_height(self, cloud_height_file): + @pytest.mark.parametrize( + ("var_names", "data_file"), + [ + (("CldTopTemp", "CldTopHght", "CldTopPres"), lazy_fixture("cloud_height_file")), + (("AOD550",), lazy_fixture("aod_file")), + ] + ) + def test_get_dataset_generic(self, var_names, data_file): """Test datasets from cloud height files.""" from satpy import Scene bytes_in_m_row = 4 * 3200 with dask.config.set({"array.chunk-size": f"{bytes_in_m_row * 4}B"}): - scn = Scene(reader="viirs_edr", filenames=[cloud_height_file]) - scn.load(["CldTopTemp", "CldTopHght", "CldTopPres"]) - _check_cloud_height_data_arr(scn["CldTopTemp"]) - _check_cloud_height_data_arr(scn["CldTopHght"]) - _check_cloud_height_data_arr(scn["CldTopPres"]) + scn = Scene(reader="viirs_edr", filenames=[data_file]) + scn.load(var_names) + for var_name in var_names: + _check_continuous_data_arr(scn[var_name]) @pytest.mark.parametrize( ("data_file", "exp_available"), @@ -312,7 +331,7 @@ def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.floa assert data_arr.attrs["standard_name"] == "surface_bidirectional_reflectance" -def _check_cloud_height_data_arr(data_arr: xr.DataArray) -> None: +def _check_continuous_data_arr(data_arr: xr.DataArray) -> None: _array_checks(data_arr) _shared_metadata_checks(data_arr) From 31988b63ae75427fed7b3e96e05c33dcf10a276b Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 26 Jul 2023 06:55:37 -0500 Subject: [PATCH 45/59] Update viirs_edr module docstring --- satpy/readers/viirs_edr.py | 33 +++++++++++++++++++-------------- 1 file changed, 19 insertions(+), 14 deletions(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 6dececa9c1..58458b925c 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -1,6 +1,6 @@ #!/usr/bin/env python # -*- coding: utf-8 -*- -# Copyright (c) 2022 Satpy developers +# Copyright (c) 2022-2023 Satpy developers # # This file is part of satpy. # @@ -15,30 +15,35 @@ # # You should have received a copy of the GNU General Public License along with # satpy. If not, see . -"""VIIRS NOAA enterprise L2 product reader. +"""VIIRS NOAA enterprise EDR product reader. This module defines the :class:`VIIRSJRRFileHandler` file handler, to -be used for reading VIIRS Level 2 products generated by the NOAA enterprise -suite, which are downloadable via NOAA CLASS. -A wide variety of such products exist and, at present, only three are -supported here, showing example filenames: +be used for reading VIIRS EDR products generated by the NOAA enterprise +suite, which are downloadable via NOAA CLASS or on NOAA's AWS buckets. + +A wide variety of such products exist and, at present, only a subset are supported. + - Cloud mask: JRR-CloudMask_v2r3_j01_s202112250807275_e202112250808520_c202112250837300.nc - - Aerosol properties: JRR-ADP_v2r3_j01_s202112250807275_e202112250808520_c202112250839550.nc + - Cloud products: JRR-CloudHeight_v2r3_j01_s202112250807275_e202112250808520_c202112250837300.nc + - Aerosol detection: JRR-ADP_v2r3_j01_s202112250807275_e202112250808520_c202112250839550.nc + - Aerosol optical depth: JRR-AOD_v2r3_j01_s202112250807275_e202112250808520_c202112250839550.nc - Surface reflectance: SurfRefl_v1r1_j01_s202112250807275_e202112250808520_c202112250845080.nc -All products use the same base reader `viirs_l2_jrr` and can be read through satpy with:: + +All products use the same base reader ``viirs_edr`` and can be read through satpy with:: import satpy import glob filenames = glob.glob('JRR-ADP*.nc') - scene = satpy.Scene(filenames, - reader='viirs_l2_jrr') + scene = satpy.Scene(filenames, reader='viirs_edr') scene.load(['smoke_concentration']) -NOTE: -Multiple products contain datasets with the same name! For example, both the cloud mask -and aerosol files contain a cloud mask, but these are not identical. -For clarity, the aerosol file cloudmask is named `cloud_mask_adp` in this reader. +.. note:: + + Multiple products contain datasets with the same name! For example, both the cloud mask + and aerosol detection files contain a cloud mask, but these are not identical. + For clarity, the aerosol file cloudmask is named `cloud_mask_adp` in this reader. + """ From 8957d02c18065dd6bfaefe79e32f0c790a93d484 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 27 Jul 2023 09:22:00 -0500 Subject: [PATCH 46/59] Add LST support --- satpy/etc/readers/viirs_edr.yaml | 21 +++++++---- satpy/readers/viirs_edr.py | 16 +++++++++ satpy/tests/reader_tests/test_viirs_edr.py | 41 ++++++++++++++++++---- 3 files changed, 65 insertions(+), 13 deletions(-) diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index 0004751f5d..0aa39b69db 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -10,29 +10,28 @@ reader: file_types: jrr_cloudmask: file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler - variable_prefix: "" file_patterns: - 'JRR-CloudMask_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' jrr_aerosol_product: file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler - variable_prefix: "" file_patterns: - 'JRR-ADP_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' jrr_surfref_product: file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSSurfaceReflectanceWithVIHandler - variable_prefix: "" file_patterns: - 'SurfRefl_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' jrr_cloudheight_product: file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler - variable_prefix: "" file_patterns: - 'JRR-CloudHeight_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' jrr_aod_product: file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler - variable_prefix: "" file_patterns: - 'JRR-AOD_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' + jrr_lst_product: + file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSLSTHandler + file_patterns: + - 'LST_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' datasets: @@ -40,14 +39,14 @@ datasets: longitude: name: longitude standard_name: longitude - file_type: [jrr_cloudmask, jrr_aerosol_product, jrr_cloudheight_product, jrr_aod_product] + file_type: [jrr_cloudmask, jrr_aerosol_product, jrr_cloudheight_product, jrr_aod_product, jrr_lst_product] file_key: "Longitude" units: 'degrees_east' resolution: 750 latitude: name: latitude standard_name: latitude - file_type: [jrr_cloudmask, jrr_aerosol_product, jrr_cloudheight_product, jrr_aod_product] + file_type: [jrr_cloudmask, jrr_aerosol_product, jrr_cloudheight_product, jrr_aod_product, jrr_lst_product] file_key: "Latitude" units: 'degrees_north' resolution: 750 @@ -420,3 +419,11 @@ datasets: file_type: [jrr_aod_product] resolution: 750 coordinates: [longitude, latitude] + + # Land Surface Temperature + vlst: + name: VLST + file_key: VLST + file_type: [jrr_lst_product] + resolution: 750 + coordinates: [longitude, latitude] diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 58458b925c..d90b0359f6 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -28,6 +28,7 @@ - Aerosol detection: JRR-ADP_v2r3_j01_s202112250807275_e202112250808520_c202112250839550.nc - Aerosol optical depth: JRR-AOD_v2r3_j01_s202112250807275_e202112250808520_c202112250839550.nc - Surface reflectance: SurfRefl_v1r1_j01_s202112250807275_e202112250808520_c202112250845080.nc + - Land Surface Temperature: LST_v2r0_npp_s202307241724558_e202307241726200_c202307241854058.nc All products use the same base reader ``viirs_edr`` and can be read through satpy with:: @@ -237,3 +238,18 @@ def _get_veg_index_good_mask(self) -> xr.DataArray: bad_mask_iband_dask = bad_mask.data.repeat(2, axis=1).repeat(2, axis=0) good_mask_iband = xr.DataArray(~bad_mask_iband_dask, dims=qf1.dims) return good_mask_iband + + +class VIIRSLSTHandler(VIIRSJRRFileHandler): + """File handler to handle LST file scale factor and offset weirdness.""" + + def __init__(self, *args, **kwargs): + """Initialize the file handler and unscale necessary variables.""" + super().__init__(*args, **kwargs) + + # Update variables with external scale factor and offset + lst_data_arr = self.nc["VLST"] + scale_factor = self.nc["LST_ScaleFact"] + add_offset = self.nc["LST_Offset"] + lst_data_arr.data = lst_data_arr.data * scale_factor.data + add_offset.data + self.nc["VLST"] = lst_data_arr diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 53d24e7d11..cc06e16647 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -33,6 +33,7 @@ import pytest import xarray as xr from pyresample import SwathDefinition +from pytest import TempPathFactory from pytest_lazyfixture import lazy_fixture I_COLS = 6400 @@ -69,18 +70,18 @@ @pytest.fixture(scope="module") -def surface_reflectance_file(tmp_path_factory) -> Path: +def surface_reflectance_file(tmp_path_factory: TempPathFactory) -> Path: """Generate fake surface reflectance EDR file.""" return _create_surface_reflectance_file(tmp_path_factory, include_veg_indices=False) @pytest.fixture(scope="module") -def surface_reflectance_with_veg_indices_file(tmp_path_factory) -> Path: +def surface_reflectance_with_veg_indices_file(tmp_path_factory: TempPathFactory) -> Path: """Generate fake surface reflectance EDR file with vegetation indexes included.""" return _create_surface_reflectance_file(tmp_path_factory, include_veg_indices=True) -def _create_surface_reflectance_file(tmp_path_factory, include_veg_indices: bool = False) -> Path: +def _create_surface_reflectance_file(tmp_path_factory: TempPathFactory, include_veg_indices: bool = False) -> Path: fn = f"SurfRefl_v1r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202305302025590.nc" sr_vars = _create_surf_refl_variables() if include_veg_indices: @@ -162,7 +163,7 @@ def _create_veg_index_variables() -> dict[str, xr.DataArray]: @pytest.fixture(scope="module") -def cloud_height_file(tmp_path_factory) -> Path: +def cloud_height_file(tmp_path_factory: TempPathFactory) -> Path: """Generate fake CloudHeight VIIRS EDR file.""" fn = f"JRR-CloudHeight_v3r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202307231023395.nc" data_vars = _create_continuous_variables( @@ -172,7 +173,7 @@ def cloud_height_file(tmp_path_factory) -> Path: @pytest.fixture(scope="module") -def aod_file(tmp_path_factory) -> Path: +def aod_file(tmp_path_factory: TempPathFactory) -> Path: """Generate fake AOD VIIRs EDR file.""" fn = f"JRR-AOD_v3r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202307231023395.nc" data_vars = _create_continuous_variables( @@ -181,6 +182,27 @@ def aod_file(tmp_path_factory) -> Path: return _create_fake_file(tmp_path_factory, fn, data_vars) +@pytest.fixture(scope="module") +def lst_file(tmp_path_factory: TempPathFactory) -> Path: + """Generate fake VLST EDR file.""" + fn = f"LST_v2r0_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202307241854058.nc" + data_vars = _create_lst_variables() + return _create_fake_file(tmp_path_factory, fn, data_vars) + + +def _create_lst_variables() -> dict[str, xr.DataArray]: + data_vars = _create_continuous_variables(("VLST",)) + + # VLST scale factors + data_vars["VLST"].data = (data_vars["VLST"].data / 0.0001).astype(np.int16) + data_vars["VLST"].encoding.pop("scale_factor") + data_vars["VLST"].encoding.pop("add_offset") + data_vars["LST_ScaleFact"] = xr.DataArray(np.float32(0.0001)) + data_vars["LST_Offset"] = xr.DataArray(np.float32(0.0)) + + return data_vars + + def _create_continuous_variables(var_names: Iterable[str]) -> dict[str, xr.DataArray]: dims = ("Rows", "Columns") @@ -206,7 +228,7 @@ def _create_continuous_variables(var_names: Iterable[str]) -> dict[str, xr.DataA return data_arrs -def _create_fake_file(tmp_path_factory, filename: str, data_arrs: dict[str, xr.DataArray]) -> Path: +def _create_fake_file(tmp_path_factory: TempPathFactory, filename: str, data_arrs: dict[str, xr.DataArray]) -> Path: tmp_path = tmp_path_factory.mktemp("viirs_edr_tmp") file_path = tmp_path / filename ds = _create_fake_dataset(data_arrs) @@ -253,6 +275,7 @@ def test_get_dataset_surf_refl_with_veg_idx(self, surface_reflectance_with_veg_i [ (("CldTopTemp", "CldTopHght", "CldTopPres"), lazy_fixture("cloud_height_file")), (("AOD550",), lazy_fixture("aod_file")), + (("VLST",), lazy_fixture("lst_file")), ] ) def test_get_dataset_generic(self, var_names, data_file): @@ -333,6 +356,12 @@ def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.floa def _check_continuous_data_arr(data_arr: xr.DataArray) -> None: _array_checks(data_arr) + + # random sample should be between 0 and 1 only if factor/offset applied + data = data_arr.data.compute() + assert not (data < 0).any() + assert not (data > 1).any() + _shared_metadata_checks(data_arr) From db09792900673e45c5b5fa9f11cef12f8a757a90 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 9 Aug 2023 10:17:33 -0500 Subject: [PATCH 47/59] Restructure LST scaling to be more flexible --- satpy/readers/viirs_edr.py | 24 +++++++++++++++++++----- 1 file changed, 19 insertions(+), 5 deletions(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index d90b0359f6..1c7ba034ef 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -243,13 +243,27 @@ def _get_veg_index_good_mask(self) -> xr.DataArray: class VIIRSLSTHandler(VIIRSJRRFileHandler): """File handler to handle LST file scale factor and offset weirdness.""" + _manual_scalings = { + "VLST": ("LST_ScaleFact", "LST_Offset"), + "emis_m15": ("LSE_ScaleFact", "LSE_Offset"), + "emis_m16": ("LSE_ScaleFact", "LSE_Offset"), + "emis_bbe": ("LSE_ScaleFact", "LSE_Offset"), + "Satellite_Azimuth_Angle": ("AZI_ScaleFact", "AZI_Offset"), + } + def __init__(self, *args, **kwargs): """Initialize the file handler and unscale necessary variables.""" super().__init__(*args, **kwargs) # Update variables with external scale factor and offset - lst_data_arr = self.nc["VLST"] - scale_factor = self.nc["LST_ScaleFact"] - add_offset = self.nc["LST_Offset"] - lst_data_arr.data = lst_data_arr.data * scale_factor.data + add_offset.data - self.nc["VLST"] = lst_data_arr + self._scale_data() + + def _scale_data(self): + for var_name in list(self.nc.variables.keys()): + if var_name not in self._manual_scalings: + continue + data_arr = self.nc[var_name] + scale_factor = self.nc[self._manual_scalings[var_name][0]] + add_offset = self.nc[self._manual_scalings[var_name][1]] + data_arr.data = data_arr.data * scale_factor.data + add_offset.data + self.nc[var_name] = data_arr From 4e56372d4605ec486b240adc19b54dd2349e22fe Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 9 Aug 2023 13:03:37 -0500 Subject: [PATCH 48/59] Switch "viirs_edr" reader to dynamic variable loading --- satpy/etc/readers/viirs_edr.yaml | 240 ++++----------------- satpy/readers/viirs_edr.py | 44 +++- satpy/tests/reader_tests/test_viirs_edr.py | 5 +- 3 files changed, 84 insertions(+), 205 deletions(-) diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index 0aa39b69db..4f33bcc184 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -12,197 +12,71 @@ file_types: file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler file_patterns: - 'JRR-CloudMask_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' - jrr_aerosol_product: + jrr_aerosol: file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler file_patterns: - 'JRR-ADP_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' - jrr_surfref_product: + jrr_surfref: file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSSurfaceReflectanceWithVIHandler file_patterns: - 'SurfRefl_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' - jrr_cloudheight_product: + jrr_cloudheight: file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler file_patterns: - 'JRR-CloudHeight_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' - jrr_aod_product: + jrr_aod: file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSJRRFileHandler file_patterns: - 'JRR-AOD_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' - jrr_lst_product: + jrr_lst: file_reader: !!python/name:satpy.readers.viirs_edr.VIIRSLSTHandler file_patterns: - 'LST_{version}_{platform_shortname}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time}.nc' datasets: - # Geolocation datasets - longitude: - name: longitude - standard_name: longitude - file_type: [jrr_cloudmask, jrr_aerosol_product, jrr_cloudheight_product, jrr_aod_product, jrr_lst_product] - file_key: "Longitude" - units: 'degrees_east' - resolution: 750 - latitude: - name: latitude - standard_name: latitude - file_type: [jrr_cloudmask, jrr_aerosol_product, jrr_cloudheight_product, jrr_aod_product, jrr_lst_product] - file_key: "Latitude" - units: 'degrees_north' - resolution: 750 + # NOTE: All non-surface reflectance file variables are dynamically loaded + # from the variable names inside the file. All 2D variables are + # supported and use the exact name of the variable in the NetCDF file. + # Files mentioned above in "file_types" are supported. + # To see a full list of loadable variables, create a Scene object with + # data files and run ``scn.available_dataset_names()``. + + # Surface reflectance products longitude_375: name: longitude_375 standard_name: longitude - file_type: jrr_surfref_product + file_type: jrr_surfref file_key: "Longitude_at_375m_resolution" units: 'degrees_east' resolution: 375 latitude_375: name: latitude_375 standard_name: latitude - file_type: jrr_surfref_product + file_type: jrr_surfref file_key: "Latitude_at_375m_resolution" units: 'degrees_north' resolution: 375 longitude_750: name: longitude_750 standard_name: longitude - file_type: jrr_surfref_product + file_type: jrr_surfref file_key: "Longitude_at_750m_resolution" units: 'degrees_east' resolution: 750 latitude_750: name: latitude_750 standard_name: latitude - file_type: jrr_surfref_product + file_type: jrr_surfref file_key: "Latitude_at_750m_resolution" units: 'degrees_north' resolution: 750 - # Cloudmask product datasets - cloud_mask: - name: cloud_mask - resolution: 750 - file_type: jrr_cloudmask - file_key: "CloudMask" - coordinates: [longitude, latitude] - cloud_mask_binary: - name: cloud_mask_binary - resolution: 750 - file_type: [jrr_cloudmask] - file_key: "CloudMaskBinary" - coordinates: [longitude, latitude] - cloud_probability: - name: cloud_probability - resolution: 750 - file_type: [jrr_cloudmask] - file_key: "CloudProbability" - coordinates: [longitude, latitude] - dust_mask: - name: dust_mask - resolution: 750 - file_type: [jrr_cloudmask] - file_key: "Dust_Mask" - coordinates: [longitude, latitude] - fire_mask: - name: fire_mask - resolution: 750 - file_type: [jrr_cloudmask] - file_key: "Fire_Mask" - coordinates: [longitude, latitude] - smoke_mask: - name: smoke_mask - resolution: 750 - file_type: [jrr_cloudmask] - file_key: "Smoke_Mask" - coordinates: [longitude, latitude] - - # Aerosol detection product datasets - ash_mask: - name: ash_mask - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "Ash" - coordinates: [longitude, latitude] - cloud_mask_adp: - name: cloud_mask_adp - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "Cloud" - coordinates: [longitude, latitude] - dust_smoke_discrimination_index: - name: dust_smoke_discrimination_index - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "DSDI" - coordinates: [longitude, latitude] - nuc: - name: nuc - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "NUC" - coordinates: [longitude, latitude] - pqi1: - name: pqi1 - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "PQI1" - coordinates: [longitude, latitude] - pqi2: - name: pqi2 - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "PQI2" - coordinates: [longitude, latitude] - pqi3: - name: pqi3 - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "PQI3" - coordinates: [longitude, latitude] - pqi4: - name: pqi4 - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "PQI4" - coordinates: [longitude, latitude] - qcflag: - name: qcflag - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "QC_Flag" - coordinates: [longitude, latitude] - saai: - name: saai - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "SAAI" - coordinates: [longitude, latitude] - smoke: - name: smoke - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "Smoke" - coordinates: [longitude, latitude] - smoke_concentration: - name: smoke_concentration - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "SmokeCon" - coordinates: [longitude, latitude] - snow_ice: - name: snow_ice - resolution: 750 - file_type: [jrr_aerosol_product] - file_key: "SnowIce" - coordinates: [longitude, latitude] - - # Surface reflectance products surf_refl_I01: name: surf_refl_I01 resolution: 375 wavelength: [0.600, 0.640, 0.680] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "375m Surface Reflectance Band I1" coordinates: [longitude_375, latitude_375] units: '%' @@ -211,7 +85,7 @@ datasets: name: surf_refl_I02 resolution: 375 wavelength: [0.845, 0.865, 0.884] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "375m Surface Reflectance Band I2" coordinates: [longitude_375, latitude_375] units: '%' @@ -220,7 +94,7 @@ datasets: name: surf_refl_I03 resolution: 375 wavelength: [1.580, 1.610, 1.640] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "375m Surface Reflectance Band I3" coordinates: [longitude_375, latitude_375] units: '%' @@ -229,7 +103,7 @@ datasets: name: surf_refl_M01 resolution: 750 wavelength: [0.402, 0.412, 0.422] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "750m Surface Reflectance Band M1" coordinates: [longitude_750, latitude_750] units: '%' @@ -238,7 +112,7 @@ datasets: name: surf_refl_M02 resolution: 750 wavelength: [0.436, 0.445, 0.454] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "750m Surface Reflectance Band M2" coordinates: [longitude_750, latitude_750] units: '%' @@ -247,7 +121,7 @@ datasets: name: surf_refl_M03 resolution: 750 wavelength: [0.478, 0.488, 0.498] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "750m Surface Reflectance Band M3" coordinates: [longitude_750, latitude_750] units: '%' @@ -256,7 +130,7 @@ datasets: name: surf_refl_M04 resolution: 750 wavelength: [0.545, 0.555, 0.565] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "750m Surface Reflectance Band M4" coordinates: [longitude_750, latitude_750] units: '%' @@ -265,7 +139,7 @@ datasets: name: surf_refl_M05 resolution: 750 wavelength: [0.662, 0.672, 0.682] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "750m Surface Reflectance Band M5" coordinates: [longitude_750, latitude_750] units: '%' @@ -274,7 +148,7 @@ datasets: name: surf_refl_M06 resolution: 750 wavelength: [0.739, 0.746, 0.754] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "750m Surface Reflectance Band M6" coordinates: [longitude_750, latitude_750] units: '%' @@ -283,7 +157,7 @@ datasets: name: surf_refl_M07 resolution: 750 wavelength: [0.846, 0.865, 0.885] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "750m Surface Reflectance Band M7" coordinates: [longitude_750, latitude_750] units: '%' @@ -292,7 +166,7 @@ datasets: name: surf_refl_M08 resolution: 750 wavelength: [1.230, 1.240, 1.250] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "750m Surface Reflectance Band M8" coordinates: [longitude_750, latitude_750] units: '%' @@ -301,7 +175,7 @@ datasets: name: surf_refl_M10 resolution: 750 wavelength: [1.580, 1.610, 1.640] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "750m Surface Reflectance Band M10" coordinates: [longitude_750, latitude_750] units: '%' @@ -310,7 +184,7 @@ datasets: name: surf_refl_M11 resolution: 750 wavelength: [2.225, 2.250, 2.275] - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "750m Surface Reflectance Band M11" coordinates: [longitude_750, latitude_750] units: '%' @@ -318,7 +192,7 @@ datasets: surf_refl_qf1: name: surf_refl_qf1 resolution: 750 - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "QF1 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' @@ -326,7 +200,7 @@ datasets: surf_refl_qf2: name: surf_refl_qf2 resolution: 750 - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "QF2 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' @@ -334,7 +208,7 @@ datasets: surf_refl_qf3: name: surf_refl_qf3 resolution: 750 - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "QF3 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' @@ -342,7 +216,7 @@ datasets: surf_refl_qf4: name: surf_refl_qf4 resolution: 750 - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "QF4 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' @@ -350,7 +224,7 @@ datasets: surf_refl_qf5: name: surf_refl_qf5 resolution: 750 - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "QF5 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' @@ -358,7 +232,7 @@ datasets: surf_refl_qf6: name: surf_refl_qf6 resolution: 750 - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "QF6 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' @@ -366,7 +240,7 @@ datasets: surf_refl_qf7: name: surf_refl_qf7 resolution: 750 - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "QF7 Surface Reflectance" coordinates: [longitude_750, latitude_750] units: '1' @@ -376,7 +250,7 @@ datasets: NDVI: name: NDVI resolution: 375 - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "NDVI" coordinates: [longitude_375, latitude_375] units: "1" @@ -385,45 +259,9 @@ datasets: EVI: name: EVI resolution: 375 - file_type: [jrr_surfref_product] + file_type: [jrr_surfref] file_key: "EVI" coordinates: [longitude_375, latitude_375] units: "1" valid_range: [-1.0, 1.0] standard_name: "normalized_difference_vegetation_index" - - # Cloud Height products - cloud_top_temp: - name: CldTopTemp - file_key: "CldTopTemp" - file_type: [jrr_cloudheight_product] - resolution: 750 - coordinates: [longitude, latitude] - cloud_top_height: - name: CldTopHght - file_key: "CldTopHght" - file_type: [jrr_cloudheight_product] - resolution: 750 - coordinates: [longitude, latitude] - cloud_top_pressure: - name: CldTopPres - file_key: "CldTopPres" - file_type: [jrr_cloudheight_product] - resolution: 750 - coordinates: [longitude, latitude] - - # Aerosol Optical Depth products - aod550: - name: AOD550 - file_key: AOD550 - file_type: [jrr_aod_product] - resolution: 750 - coordinates: [longitude, latitude] - - # Land Surface Temperature - vlst: - name: VLST - file_key: VLST - file_type: [jrr_lst_product] - resolution: 750 - coordinates: [longitude, latitude] diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 1c7ba034ef..d1ef69db96 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -58,6 +58,7 @@ from satpy.utils import get_chunk_size_limit LOG = logging.getLogger(__name__) +M_COLS = 3200 class VIIRSJRRFileHandler(BaseFileHandler): @@ -68,7 +69,7 @@ def __init__(self, filename, filename_info, filetype_info): super(VIIRSJRRFileHandler, self).__init__(filename, filename_info, filetype_info) # use entire scans as chunks - row_chunks_m = max(get_chunk_size_limit() // 4 // 3200, 1) # 32-bit floats + row_chunks_m = max(get_chunk_size_limit() // 4 // M_COLS, 1) # 32-bit floats row_chunks_i = row_chunks_m * 2 self.nc = xr.open_dataset(self.filename, decode_cf=True, @@ -99,7 +100,7 @@ def __init__(self, filename, filename_info, filetype_info): def rows_per_scans(self, data_arr: xr.DataArray) -> int: """Get number of array rows per instrument scan based on data resolution.""" - return 32 if data_arr.shape[1] == 6400 else 16 + return 16 if data_arr.shape[1] == M_COLS else 32 def get_dataset(self, dataset_id: DataID, info: dict) -> xr.DataArray: """Get the dataset.""" @@ -183,6 +184,9 @@ def available_datasets(self, configured_datasets=None): ``None`` if this file object is not responsible for it. """ + # keep track of what variables the YAML has configured, so we don't + # duplicate entries for them in the dynamic portion + handled_var_names = set() for is_avail, ds_info in (configured_datasets or []): if is_avail is not None: # some other file handler said it has this dataset @@ -194,8 +198,44 @@ def available_datasets(self, configured_datasets=None): # this is not the file type for this dataset yield None, ds_info file_key = ds_info.get("file_key", ds_info["name"]) + handled_var_names.add(file_key) yield file_key in self.nc, ds_info + ftype = self.filetype_info["file_type"] + m_lon_name = f"longitude_{ftype}" + m_lat_name = f"latitude_{ftype}" + m_coords = (m_lon_name, m_lat_name) + i_lon_name = f"longitude_i_{ftype}" + i_lat_name = f"latitude_i_{ftype}" + i_coords = (i_lon_name, i_lat_name) + for var_name, data_arr in self.nc.items(): + is_lon = "longitude" in var_name.lower() + is_lat = "latitude" in var_name.lower() + if var_name in handled_var_names and not (is_lon or is_lat): + # skip variables that YAML had configured, but allow lon/lats + # to be reprocessed due to our dynamic coordinate naming + continue + if data_arr.ndim != 2: + # only 2D arrays supported at this time + continue + res = 750 if data_arr.shape[1] == M_COLS else 375 + ds_info = { + "file_key": var_name, + "file_type": ftype, + "name": var_name, + "resolution": res, + "coordinates": m_coords if res == 750 else i_coords, + } + if is_lon: + ds_info["standard_name"] = "longitude" + ds_info["units"] = "degrees_east" + ds_info["name"] = m_lon_name if res == 750 else i_lon_name + elif is_lat: + ds_info["standard_name"] = "latitude" + ds_info["units"] = "degrees_north" + ds_info["name"] = m_lat_name if res == 750 else i_lat_name + yield True, ds_info + class VIIRSSurfaceReflectanceWithVIHandler(VIIRSJRRFileHandler): """File handler for surface reflectance files with optional vegetation indexes.""" diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index cc06e16647..09cc2769b6 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -106,8 +106,8 @@ def _create_surf_refl_variables() -> dict[str, xr.DataArray]: data_arrs = { "Longitude_at_375m_resolution": xr.DataArray(i_data, dims=i_dims, attrs=lon_attrs), "Latitude_at_375m_resolution": xr.DataArray(i_data, dims=i_dims, attrs=lat_attrs), - "Longitude_at_750m_resolution": xr.DataArray(i_data, dims=i_dims, attrs=lon_attrs), - "Latitude_at_750m_resolution": xr.DataArray(i_data, dims=i_dims, attrs=lat_attrs), + "Longitude_at_750m_resolution": xr.DataArray(m_data, dims=m_dims, attrs=lon_attrs), + "Latitude_at_750m_resolution": xr.DataArray(m_data, dims=m_dims, attrs=lat_attrs), "375m Surface Reflectance Band I1": xr.DataArray(i_data, dims=i_dims, attrs=sr_attrs), "750m Surface Reflectance Band M1": xr.DataArray(m_data, dims=m_dims, attrs=sr_attrs), } @@ -368,6 +368,7 @@ def _check_continuous_data_arr(data_arr: xr.DataArray) -> None: def _array_checks(data_arr: xr.DataArray, dtype: npt.Dtype = np.float32) -> None: assert data_arr.dims == ("y", "x") assert isinstance(data_arr.attrs["area"], SwathDefinition) + assert data_arr.attrs["area"].shape == data_arr.shape assert isinstance(data_arr.data, da.Array) assert np.issubdtype(data_arr.data.dtype, dtype) is_mband_res = _is_mband_res(data_arr) From 65511ecf98a89346ab161c749e135ddebfcd38fe Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 11 Aug 2023 14:02:29 -0500 Subject: [PATCH 49/59] Deprecate "viirs_l2_cloud_mask_nc" reader --- satpy/readers/__init__.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/satpy/readers/__init__.py b/satpy/readers/__init__.py index 7efab8d904..2b1bbc37ba 100644 --- a/satpy/readers/__init__.py +++ b/satpy/readers/__init__.py @@ -37,7 +37,7 @@ # Old Name -> New Name -PENDING_OLD_READER_NAMES = {'fci_l1c_fdhsi': 'fci_l1c_nc'} +PENDING_OLD_READER_NAMES = {'fci_l1c_fdhsi': 'fci_l1c_nc', 'viirs_l2_cloud_mask_nc': 'viirs_edr'} OLD_READER_NAMES: dict[str, str] = {} From fbd2802c0747adde8a7582894d3c18f495bfc7ce Mon Sep 17 00:00:00 2001 From: David Hoese Date: Fri, 11 Aug 2023 14:15:55 -0500 Subject: [PATCH 50/59] Add flag to control QF filtering of vegetation indexes --- satpy/readers/viirs_edr.py | 18 +++++++++++++++++- satpy/tests/reader_tests/test_viirs_edr.py | 22 ++++++++++++++-------- 2 files changed, 31 insertions(+), 9 deletions(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index d1ef69db96..9b73d272ce 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -45,6 +45,17 @@ and aerosol detection files contain a cloud mask, but these are not identical. For clarity, the aerosol file cloudmask is named `cloud_mask_adp` in this reader. +Vegetation Indexes +^^^^^^^^^^^^^^^^^^ + +The NDVI and EVI products can be loaded from CSPP-produced Surface Reflectance +files. By default, these products are filtered based on the Surface Reflectance +Quality Flags. This is used to remove/mask pixels in certain cloud or water +regions. This behavior can be disabled by providing the reader keyword argument +``filter_veg`` and setting it to ``False``. For example:: + + scene = satpy.Scene(filenames, reader='viirs_edr', reader_kwargs={"filter_veg": False}) + """ @@ -240,9 +251,14 @@ def available_datasets(self, configured_datasets=None): class VIIRSSurfaceReflectanceWithVIHandler(VIIRSJRRFileHandler): """File handler for surface reflectance files with optional vegetation indexes.""" + def __init__(self, *args, filter_veg: bool = True, **kwargs) -> None: + """Initialize file handler and keep track of vegetation index filtering.""" + super().__init__(*args, **kwargs) + self._filter_veg = filter_veg + def _mask_invalid(self, data_arr: xr.DataArray, ds_info: dict) -> xr.DataArray: new_data_arr = super()._mask_invalid(data_arr, ds_info) - if ds_info["file_key"] in ("NDVI", "EVI"): + if ds_info["file_key"] in ("NDVI", "EVI") and self._filter_veg: good_mask = self._get_veg_index_good_mask() new_data_arr = new_data_arr.where(good_mask) return new_data_arr diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 09cc2769b6..c9caf3ab85 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -259,15 +259,17 @@ def test_get_dataset_surf_refl(self, surface_reflectance_file): _check_surf_refl_data_arr(scn["surf_refl_I01"]) _check_surf_refl_data_arr(scn["surf_refl_M01"]) - def test_get_dataset_surf_refl_with_veg_idx(self, surface_reflectance_with_veg_indices_file): + @pytest.mark.parametrize("filter_veg", [False, True]) + def test_get_dataset_surf_refl_with_veg_idx(self, surface_reflectance_with_veg_indices_file, filter_veg): """Test retrieval of vegetation indices from surface reflectance files.""" from satpy import Scene bytes_in_m_row = 4 * 3200 with dask.config.set({"array.chunk-size": f"{bytes_in_m_row * 4}B"}): - scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_with_veg_indices_file]) + scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_with_veg_indices_file], + reader_kwargs={"filter_veg": filter_veg}) scn.load(["NDVI", "EVI", "surf_refl_qf1"]) - _check_vi_data_arr(scn["NDVI"]) - _check_vi_data_arr(scn["EVI"]) + _check_vi_data_arr(scn["NDVI"], filter_veg) + _check_vi_data_arr(scn["EVI"], filter_veg) _check_surf_refl_qf_data_arr(scn["surf_refl_qf1"]) @pytest.mark.parametrize( @@ -332,16 +334,20 @@ def _check_surf_refl_qf_data_arr(data_arr: xr.DataArray) -> None: assert data_arr.attrs["standard_name"] == "quality_flag" -def _check_vi_data_arr(data_arr: xr.DataArray) -> None: +def _check_vi_data_arr(data_arr: xr.DataArray, is_filtered: bool) -> None: _array_checks(data_arr) _shared_metadata_checks(data_arr) assert data_arr.attrs["units"] == "1" assert data_arr.attrs["standard_name"] == "normalized_difference_vegetation_index" data = data_arr.data.compute() - np.testing.assert_allclose(data[0, :7], [np.nan, -1.0, -0.5, 0.0, 0.5, 1.0, np.nan]) - np.testing.assert_allclose(data[0, 8:8 + 16], np.nan) - np.testing.assert_allclose(data[0, 8 + 16:], 0.0) + if is_filtered: + np.testing.assert_allclose(data[0, :7], [np.nan, -1.0, -0.5, 0.0, 0.5, 1.0, np.nan]) + np.testing.assert_allclose(data[0, 8:8 + 16], np.nan) + np.testing.assert_allclose(data[0, 8 + 16:], 0.0) + else: + np.testing.assert_allclose(data[0, :7], [np.nan, -1.0, -0.5, 0.0, 0.5, 1.0, np.nan]) + np.testing.assert_allclose(data[0, 8:], 0.0) def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.float32) -> None: From d5b1caca16a9442f6ddb3b40bd1f83041530f086 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Tue, 15 Aug 2023 20:35:25 -0500 Subject: [PATCH 51/59] Refactor viirs_edr available_datasets --- satpy/readers/viirs_edr.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 9b73d272ce..f95f6a901c 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -57,9 +57,10 @@ scene = satpy.Scene(filenames, reader='viirs_edr', reader_kwargs={"filter_veg": False}) """ - +from __future__ import annotations import logging +from typing import Iterable import numpy as np import xarray as xr @@ -212,6 +213,9 @@ def available_datasets(self, configured_datasets=None): handled_var_names.add(file_key) yield file_key in self.nc, ds_info + yield from self._dynamic_variables_from_file(handled_var_names) + + def _dynamic_variables_from_file(self, handled_var_names: set) -> Iterable[tuple[bool, dict]]: ftype = self.filetype_info["file_type"] m_lon_name = f"longitude_{ftype}" m_lat_name = f"latitude_{ftype}" From aa3f904cc0795eb4a1af2da3d1794e7bca5037cd Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 16 Aug 2023 10:57:00 -0500 Subject: [PATCH 52/59] Fix coordinate variables not being dynamically included --- satpy/readers/viirs_edr.py | 3 ++- satpy/tests/reader_tests/test_viirs_edr.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index f95f6a901c..8b3cf99d27 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -223,7 +223,8 @@ def _dynamic_variables_from_file(self, handled_var_names: set) -> Iterable[tuple i_lon_name = f"longitude_i_{ftype}" i_lat_name = f"latitude_i_{ftype}" i_coords = (i_lon_name, i_lat_name) - for var_name, data_arr in self.nc.items(): + for var_name in self.nc.variables.keys(): + data_arr = self.nc[var_name] is_lon = "longitude" in var_name.lower() is_lat = "latitude" in var_name.lower() if var_name in handled_var_names and not (is_lon or is_lat): diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index c9caf3ab85..f4f5799444 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -225,6 +225,7 @@ def _create_continuous_variables(var_names: Iterable[str]) -> dict[str, xr.DataA data_arr.encoding["dtype"] = np.int16 data_arr.encoding["scale_factor"] = data_arr.attrs.pop("scale_factor") data_arr.encoding["add_offset"] = data_arr.attrs.pop("add_offset") + data_arr.encoding["coordinates"] = "Longitude Latitude" return data_arrs From 4fbeeaa72cfbd1704ac422c662c07631d7defa1e Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 16 Aug 2023 11:17:41 -0500 Subject: [PATCH 53/59] Fix YAML definitions being ignored when multiple files are provided --- satpy/readers/viirs_edr.py | 4 +- satpy/tests/reader_tests/test_viirs_edr.py | 63 ++++++++++++++++------ 2 files changed, 48 insertions(+), 19 deletions(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index 8b3cf99d27..b007e710b3 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -200,6 +200,8 @@ def available_datasets(self, configured_datasets=None): # duplicate entries for them in the dynamic portion handled_var_names = set() for is_avail, ds_info in (configured_datasets or []): + file_key = ds_info.get("file_key", ds_info["name"]) + handled_var_names.add(file_key) if is_avail is not None: # some other file handler said it has this dataset # we don't know any more information than the previous @@ -209,8 +211,6 @@ def available_datasets(self, configured_datasets=None): if self.file_type_matches(ds_info['file_type']) is None: # this is not the file type for this dataset yield None, ds_info - file_key = ds_info.get("file_key", ds_info["name"]) - handled_var_names.add(file_key) yield file_key in self.nc, ds_info yield from self._dynamic_variables_from_file(handled_var_names) diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index f4f5799444..5286ed3461 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -245,33 +245,57 @@ def _create_fake_dataset(vars_dict: dict[str, xr.DataArray]) -> xr.Dataset: return ds +def _copy_to_second_granule(first_granule_path: Path) -> Path: + # hack to make multiple time steps + second_fn = Path(str(first_granule_path).replace("0.nc", "1.nc")) + shutil.copy(first_granule_path, second_fn) + return second_fn + + class TestVIIRSJRRReader: """Test the VIIRS JRR L2 reader.""" - def test_get_dataset_surf_refl(self, surface_reflectance_file): + @pytest.mark.parametrize("multiple_files", [False, True]) + def test_get_dataset_surf_refl(self, surface_reflectance_file, multiple_files): """Test retrieval of datasets.""" from satpy import Scene + + files = [surface_reflectance_file] + if multiple_files: + files.append(_copy_to_second_granule(surface_reflectance_file)) + bytes_in_m_row = 4 * 3200 with dask.config.set({"array.chunk-size": f"{bytes_in_m_row * 4}B"}): - scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_file]) + scn = Scene(reader="viirs_edr", filenames=files) scn.load(["surf_refl_I01", "surf_refl_M01"]) assert scn.start_time == START_TIME assert scn.end_time == END_TIME - _check_surf_refl_data_arr(scn["surf_refl_I01"]) - _check_surf_refl_data_arr(scn["surf_refl_M01"]) + _check_surf_refl_data_arr(scn["surf_refl_I01"], multiple_files=multiple_files) + _check_surf_refl_data_arr(scn["surf_refl_M01"], multiple_files=multiple_files) @pytest.mark.parametrize("filter_veg", [False, True]) - def test_get_dataset_surf_refl_with_veg_idx(self, surface_reflectance_with_veg_indices_file, filter_veg): + @pytest.mark.parametrize("multiple_files", [False, True]) + def test_get_dataset_surf_refl_with_veg_idx( + self, + surface_reflectance_with_veg_indices_file, + filter_veg, + multiple_files + ): """Test retrieval of vegetation indices from surface reflectance files.""" from satpy import Scene + + files = [surface_reflectance_with_veg_indices_file] + if multiple_files: + files.append(_copy_to_second_granule(surface_reflectance_with_veg_indices_file)) + bytes_in_m_row = 4 * 3200 with dask.config.set({"array.chunk-size": f"{bytes_in_m_row * 4}B"}): - scn = Scene(reader="viirs_edr", filenames=[surface_reflectance_with_veg_indices_file], + scn = Scene(reader="viirs_edr", filenames=files, reader_kwargs={"filter_veg": filter_veg}) scn.load(["NDVI", "EVI", "surf_refl_qf1"]) - _check_vi_data_arr(scn["NDVI"], filter_veg) - _check_vi_data_arr(scn["EVI"], filter_veg) - _check_surf_refl_qf_data_arr(scn["surf_refl_qf1"]) + _check_vi_data_arr(scn["NDVI"], filter_veg, multiple_files) + _check_vi_data_arr(scn["EVI"], filter_veg, multiple_files) + _check_surf_refl_qf_data_arr(scn["surf_refl_qf1"], multiple_files) @pytest.mark.parametrize( ("var_names", "data_file"), @@ -328,15 +352,15 @@ def test_get_platformname(self, surface_reflectance_file, filename_platform, exp assert scn["surf_refl_I01"].attrs["platform_name"] == exp_shortname -def _check_surf_refl_qf_data_arr(data_arr: xr.DataArray) -> None: - _array_checks(data_arr, dtype=np.uint8) +def _check_surf_refl_qf_data_arr(data_arr: xr.DataArray, multiple_files: bool) -> None: + _array_checks(data_arr, dtype=np.uint8, multiple_files=multiple_files) _shared_metadata_checks(data_arr) assert data_arr.attrs["units"] == "1" assert data_arr.attrs["standard_name"] == "quality_flag" -def _check_vi_data_arr(data_arr: xr.DataArray, is_filtered: bool) -> None: - _array_checks(data_arr) +def _check_vi_data_arr(data_arr: xr.DataArray, is_filtered: bool, multiple_files: bool) -> None: + _array_checks(data_arr, multiple_files=multiple_files) _shared_metadata_checks(data_arr) assert data_arr.attrs["units"] == "1" assert data_arr.attrs["standard_name"] == "normalized_difference_vegetation_index" @@ -351,8 +375,12 @@ def _check_vi_data_arr(data_arr: xr.DataArray, is_filtered: bool) -> None: np.testing.assert_allclose(data[0, 8:], 0.0) -def _check_surf_refl_data_arr(data_arr: xr.DataArray, dtype: npt.DType = np.float32) -> None: - _array_checks(data_arr, dtype) +def _check_surf_refl_data_arr( + data_arr: xr.DataArray, + dtype: npt.DType = np.float32, + multiple_files: bool = False +) -> None: + _array_checks(data_arr, dtype, multiple_files=multiple_files) data = data_arr.data.compute() assert data.max() > 1.0 # random 0-1 test data multiplied by 100 @@ -372,14 +400,15 @@ def _check_continuous_data_arr(data_arr: xr.DataArray) -> None: _shared_metadata_checks(data_arr) -def _array_checks(data_arr: xr.DataArray, dtype: npt.Dtype = np.float32) -> None: +def _array_checks(data_arr: xr.DataArray, dtype: npt.Dtype = np.float32, multiple_files: bool = False) -> None: assert data_arr.dims == ("y", "x") assert isinstance(data_arr.attrs["area"], SwathDefinition) assert data_arr.attrs["area"].shape == data_arr.shape assert isinstance(data_arr.data, da.Array) assert np.issubdtype(data_arr.data.dtype, dtype) is_mband_res = _is_mband_res(data_arr) - exp_shape = (M_ROWS, M_COLS) if is_mband_res else (I_ROWS, I_COLS) + shape_multiplier = 1 + int(multiple_files) + exp_shape = (M_ROWS * shape_multiplier, M_COLS) if is_mband_res else (I_ROWS * shape_multiplier, I_COLS) assert data_arr.shape == exp_shape exp_row_chunks = 4 if is_mband_res else 8 assert all(c == exp_row_chunks for c in data_arr.chunks[0]) From 8d664a654bb219dd8a85ff5bc60b50f37c16dceb Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 16 Aug 2023 11:18:38 -0500 Subject: [PATCH 54/59] Remove old VIIRS L2 Cloud Mask reader --- satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml | 50 -------- satpy/readers/viirs_l2.py | 120 ------------------ 2 files changed, 170 deletions(-) delete mode 100644 satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml delete mode 100644 satpy/readers/viirs_l2.py diff --git a/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml b/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml deleted file mode 100644 index 0f2650bdc1..0000000000 --- a/satpy/etc/readers/viirs_l2_cloud_mask_nc.yaml +++ /dev/null @@ -1,50 +0,0 @@ -reader: - name: viirs_l2_cloud_mask_nc - short_name: VIIRS CSPP Cloud Mask - long_name: VIIRS CSPP Cloud Mask data in NetCDF4 format - description: VIIRS CSPP Cloud Mask reader - status: beta - supports_fsspec: false - reader: !!python/name:satpy.readers.yaml_reader.FileYAMLReader - sensors: [viirs] - -file_types: - cspp_cloud_mask_file: - file_reader: !!python/name:satpy.readers.viirs_l2.VIIRSCloudMaskFileHandler - file_patterns: ['JRR-CloudMask_{delivery_package:4s}_{platform_shortname:3s}_s{start_time:%Y%m%d%H%M%S%f}_e{end_time:%Y%m%d%H%M%S%f}_c{creation_time:%Y%m%d%H%M%S%f}.nc'] - # Example filenames - # JRR-CloudMask_v3r0_npp_s202212070726217_e202212070727459_c202212071917430.nc - -datasets: - longitude: - name: longitude - resolution: 750 - file_type: cspp_cloud_mask_file - file_key: Longitude - file_units: "degrees_east" - standard_name: longitude - coordinates: [longitude, latitude] - latitude: - name: latitude - resolution: 750 - file_type: cspp_cloud_mask_file - file_key: Latitude - file_units: "degrees_north" - standard_name: latitude - coordinates: [longitude, latitude] - cloud_mask: - name: cloud_mask - resolution: 750 - file_type: cspp_cloud_mask_file - file_key: CloudMask - file_units: "1" - standard_name: cloud_mask - coordinates: [longitude, latitude] - cloud_mask_binary: - name: cloud_mask_binary - resolution: 750 - file_type: cspp_cloud_mask_file - file_key: CloudMaskBinary - file_units: "1" - standard_name: cloud_mask_binary - coordinates: [longitude, latitude] diff --git a/satpy/readers/viirs_l2.py b/satpy/readers/viirs_l2.py deleted file mode 100644 index 90d272504a..0000000000 --- a/satpy/readers/viirs_l2.py +++ /dev/null @@ -1,120 +0,0 @@ -# Copyright (c) 2022-2023 Satpy developers -# -# This file is part of satpy. -# -# satpy is free software: you can redistribute it and/or modify it under the -# terms of the GNU General Public License as published by the Free Software -# Foundation, either version 3 of the License, or (at your option) any later -# version. -# -# satpy is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# satpy. If not, see . -"""Interface to VIIRS L2 files.""" - -from datetime import datetime - -from satpy.readers.netcdf_utils import NetCDF4FileHandler - - -class VIIRSCloudMaskFileHandler(NetCDF4FileHandler): - """VIIRS L2 Cloud Mask reader.""" - - def __init__(self, filename, filename_info, filetype_info): - """Initialize the file handler.""" - super().__init__(filename, filename_info, filetype_info, cache_handle=True) - - def _parse_datetime(self, datestr): - """Parse datetime.""" - return datetime.strptime(datestr, "%Y-%m-%dT%H:%M:%SZ") - - @property - def start_orbit_number(self): - """Get start orbit number.""" - return int(self['/attr/start_orbit_number']) - - @property - def end_orbit_number(self): - """Get end orbit number.""" - return int(self['/attr/end_orbit_number']) - - @property - def platform_name(self): - """Get platform name.""" - res = self.filename_info['platform_shortname'] - - return { - 'npp': 'Suomi-NPP', - 'j01': 'NOAA-20', - 'j02': 'NOAA-21', - }.get(res, res) - - @property - def sensor_name(self): - """Get sensor name.""" - return self['/attr/instrument_name'].lower() - - def get_shape(self, ds_id, ds_info): - """Get shape.""" - return self.get(ds_id['name'] + '/shape', 1) - - @property - def start_time(self): - """Get start time.""" - return self._parse_datetime(self['/attr/time_coverage_start']) - - @property - def end_time(self): - """Get end time.""" - return self._parse_datetime(self['/attr/time_coverage_end']) - - def get_metadata(self, dataset_id, ds_info): - """Get metadata.""" - var_path = ds_info['file_key'] - shape = self.get_shape(dataset_id, ds_info) - file_units = ds_info.get('file_units') - - attr = getattr(self[var_path], 'attrs', {}) - attr.update(ds_info) - attr.update(dataset_id.to_dict()) - attr.update({ - "shape": shape, - "units": ds_info.get("units", file_units), - "file_units": file_units, - "platform_name": self.platform_name, - "sensor": self.sensor_name, - "start_orbit": self.start_orbit_number, - "end_orbit": self.end_orbit_number, - }) - attr.update(dataset_id.to_dict()) - return attr - - def get_dataset(self, dataset_id, ds_info): - """Get dataset.""" - var_path = ds_info['file_key'] - metadata = self.get_metadata(dataset_id, ds_info) - - valid_min, valid_max = self._get_dataset_valid_range(var_path) - data = self[var_path] - data.attrs.update(metadata) - - if valid_min is not None and valid_max is not None: - data = data.where((data >= valid_min) & (data <= valid_max)) - - if isinstance(data.attrs.get('flag_meanings'), str): - data.attrs['flag_meanings'] = data.attrs['flag_meanings'].split(' ') - - # rename dimensions to correspond to satpy's 'y' and 'x' standard - if 'Rows' in data.dims: - data = data.rename({'Rows': 'y', 'Columns': 'x'}) - return data - - def _get_dataset_valid_range(self, var_path): - valid_range = self.get(var_path + '/attr/valid_range') - valid_min = valid_range[0] - valid_max = valid_range[1] - - return valid_min, valid_max From f7b60b457fdd166796e1aec28451bc59589e95f4 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 16 Aug 2023 11:20:09 -0500 Subject: [PATCH 55/59] Remove old VIIRS L2 reader tests --- satpy/tests/reader_tests/test_viirs_l2.py | 142 ---------------------- 1 file changed, 142 deletions(-) delete mode 100644 satpy/tests/reader_tests/test_viirs_l2.py diff --git a/satpy/tests/reader_tests/test_viirs_l2.py b/satpy/tests/reader_tests/test_viirs_l2.py deleted file mode 100644 index 5d6e8ffb4f..0000000000 --- a/satpy/tests/reader_tests/test_viirs_l2.py +++ /dev/null @@ -1,142 +0,0 @@ -# Copyright (c) 2022 Satpy developers -# -# This file is part of satpy. -# -# satpy is free software: you can redistribute it and/or modify it under the -# terms of the GNU General Public License as published by the Free Software -# Foundation, either version 3 of the License, or (at your option) any later -# version. -# -# satpy is distributed in the hope that it will be useful, but WITHOUT ANY -# WARRANTY; without even the implied warranty of MERCHANTABILITY or FITNESS FOR -# A PARTICULAR PURPOSE. See the GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License along with -# satpy. If not, see . -"""Tests for the VIIRS CSPP L2 readers.""" - -import numpy as np -import pytest -import xarray as xr - -from satpy import Scene - -# NOTE: -# The following Pytest fixtures are not defined in this file, but are used and injected by Pytest: -# - tmp_path - -CLOUD_MASK_FILE = "JRR-CloudMask_v3r0_npp_s202212070905565_e202212070907207_c202212071932513.nc" -NUM_COLUMNS = 3200 -NUM_ROWS = 768 -DATASETS = ['Latitude', 'Longitude', 'CloudMask', 'CloudMaskBinary'] - - -@pytest.fixture -def cloud_mask_file(tmp_path): - """Create a temporary JRR CloudMask file as a fixture.""" - file_path = tmp_path / CLOUD_MASK_FILE - _write_cloud_mask_file(file_path) - yield file_path - - -def _write_cloud_mask_file(file_path): - dset = xr.Dataset() - dset.attrs = _get_global_attrs() - dset['Latitude'] = _get_lat_arr() - dset['Longitude'] = _get_lon_arr() - dset['CloudMask'] = _get_cloud_mask_arr() - dset['CloudMaskBinary'] = _get_cloud_mask_binary_arr() - dset.to_netcdf(file_path, 'w') - - -def _get_global_attrs(): - return { - 'time_coverage_start': '2022-12-07T09:05:56Z', - 'time_coverage_end': '2022-12-07T09:07:20Z', - 'start_orbit_number': np.array(57573), - 'end_orbit_number': np.array(57573), - 'instrument_name': 'VIIRS', - } - - -def _get_lat_arr(): - arr = np.zeros((NUM_ROWS, NUM_COLUMNS), dtype=np.float32) - attrs = { - 'long_name': 'Latitude', - 'units': 'degrees_north', - 'valid_range': np.array([-90, 90], dtype=np.float32), - '_FillValue': -999. - } - return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns')) - - -def _get_lon_arr(): - arr = np.zeros((NUM_ROWS, NUM_COLUMNS), dtype=np.float32) - attrs = { - 'long_name': 'Longitude', - 'units': 'degrees_east', - 'valid_range': np.array([-180, 180], dtype=np.float32), - '_FillValue': -999. - } - return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns')) - - -def _get_cloud_mask_arr(): - arr = np.random.randint(0, 4, (NUM_ROWS, NUM_COLUMNS), dtype=np.byte) - attrs = { - 'long_name': 'Cloud Mask', - '_FillValue': np.byte(-128), - 'valid_range': np.array([0, 3], dtype=np.byte), - 'units': '1', - 'flag_values': np.array([0, 1, 2, 3], dtype=np.byte), - 'flag_meanings': 'clear probably_clear probably_cloudy cloudy', - } - return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns')) - - -def _get_cloud_mask_binary_arr(): - arr = np.random.randint(0, 2, (NUM_ROWS, NUM_COLUMNS), dtype=np.byte) - attrs = { - 'long_name': 'Cloud Mask Binary', - '_FillValue': np.byte(-128), - 'valid_range': np.array([0, 1], dtype=np.byte), - 'units': '1', - } - return xr.DataArray(arr, attrs=attrs, dims=('Rows', 'Columns')) - - -def test_cloud_mask_read_latitude(cloud_mask_file): - """Test reading latitude dataset.""" - data = _read_viirs_l2_cloud_mask_nc_data(cloud_mask_file, 'latitude') - _assert_common(data) - - -def test_cloud_mask_read_longitude(cloud_mask_file): - """Test reading longitude dataset.""" - data = _read_viirs_l2_cloud_mask_nc_data(cloud_mask_file, 'longitude') - _assert_common(data) - - -def test_cloud_mask_read_cloud_mask(cloud_mask_file): - """Test reading cloud mask dataset.""" - data = _read_viirs_l2_cloud_mask_nc_data(cloud_mask_file, 'cloud_mask') - _assert_common(data) - np.testing.assert_equal(data.attrs['flag_values'], [0, 1, 2, 3]) - assert data.attrs['flag_meanings'] == ['clear', 'probably_clear', 'probably_cloudy', 'cloudy'] - - -def test_cloud_mas_read_binary_cloud_mask(cloud_mask_file): - """Test reading binary cloud mask dataset.""" - data = _read_viirs_l2_cloud_mask_nc_data(cloud_mask_file, 'cloud_mask_binary') - _assert_common(data) - - -def _read_viirs_l2_cloud_mask_nc_data(fname, dset_name): - scn = Scene(reader="viirs_l2_cloud_mask_nc", filenames=[fname]) - scn.load([dset_name]) - return scn[dset_name] - - -def _assert_common(data): - assert data.dims == ('y', 'x') - assert "units" in data.attrs From 9e813f62f3678ce59373778085ea545ca22304b3 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 16 Aug 2023 13:21:36 -0500 Subject: [PATCH 56/59] Change multiple files tests to use a pytest fixture --- satpy/tests/reader_tests/test_viirs_edr.py | 97 +++++++++++++++------- 1 file changed, 65 insertions(+), 32 deletions(-) diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 5286ed3461..2acbd0d55a 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -22,7 +22,7 @@ from __future__ import annotations import shutil -from datetime import datetime +from datetime import datetime, timedelta from pathlib import Path from typing import Iterable @@ -72,17 +72,48 @@ @pytest.fixture(scope="module") def surface_reflectance_file(tmp_path_factory: TempPathFactory) -> Path: """Generate fake surface reflectance EDR file.""" - return _create_surface_reflectance_file(tmp_path_factory, include_veg_indices=False) + return _create_surface_reflectance_file(tmp_path_factory, START_TIME, include_veg_indices=False) + + +@pytest.fixture(scope="module") +def surface_reflectance_file2(tmp_path_factory: TempPathFactory) -> Path: + """Generate fake surface reflectance EDR file.""" + return _create_surface_reflectance_file(tmp_path_factory, START_TIME + timedelta(minutes=5), + include_veg_indices=False) + + +@pytest.fixture(scope="module") +def multiple_surface_reflectance_files(surface_reflectance_file, surface_reflectance_file2) -> list[Path]: + """Get two multiple surface reflectance files.""" + return [surface_reflectance_file, surface_reflectance_file2] @pytest.fixture(scope="module") def surface_reflectance_with_veg_indices_file(tmp_path_factory: TempPathFactory) -> Path: """Generate fake surface reflectance EDR file with vegetation indexes included.""" - return _create_surface_reflectance_file(tmp_path_factory, include_veg_indices=True) + return _create_surface_reflectance_file(tmp_path_factory, START_TIME, include_veg_indices=True) + + +@pytest.fixture(scope="module") +def surface_reflectance_with_veg_indices_file2(tmp_path_factory: TempPathFactory) -> Path: + """Generate fake surface reflectance EDR file with vegetation indexes included.""" + return _create_surface_reflectance_file(tmp_path_factory, START_TIME + timedelta(minutes=5), + include_veg_indices=True) -def _create_surface_reflectance_file(tmp_path_factory: TempPathFactory, include_veg_indices: bool = False) -> Path: - fn = f"SurfRefl_v1r2_npp_s{START_TIME:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202305302025590.nc" +@pytest.fixture(scope="module") +def multiple_surface_reflectance_files_with_veg_indices(surface_reflectance_with_veg_indices_file, + surface_reflectance_with_veg_indices_file2) -> list[Path]: + """Get two multiple surface reflectance files with vegetation indexes included.""" + return [surface_reflectance_with_veg_indices_file, surface_reflectance_with_veg_indices_file2] + + +def _create_surface_reflectance_file( + tmp_path_factory: TempPathFactory, + start_time: datetime, + include_veg_indices: bool = False, +) -> Path: + fn = f"SurfRefl_v1r2_npp_s{start_time:%Y%m%d%H%M%S}0_e{END_TIME:%Y%m%d%H%M%S}0_c202305302025590.nc" sr_vars = _create_surf_refl_variables() if include_veg_indices: sr_vars.update(_create_veg_index_variables()) @@ -245,57 +276,59 @@ def _create_fake_dataset(vars_dict: dict[str, xr.DataArray]) -> xr.Dataset: return ds -def _copy_to_second_granule(first_granule_path: Path) -> Path: - # hack to make multiple time steps - second_fn = Path(str(first_granule_path).replace("0.nc", "1.nc")) - shutil.copy(first_granule_path, second_fn) - return second_fn - - class TestVIIRSJRRReader: """Test the VIIRS JRR L2 reader.""" - @pytest.mark.parametrize("multiple_files", [False, True]) - def test_get_dataset_surf_refl(self, surface_reflectance_file, multiple_files): + @pytest.mark.parametrize( + "data_files", + [ + lazy_fixture("surface_reflectance_file"), + lazy_fixture("multiple_surface_reflectance_files"), + ], + ) + def test_get_dataset_surf_refl(self, data_files): """Test retrieval of datasets.""" from satpy import Scene - files = [surface_reflectance_file] - if multiple_files: - files.append(_copy_to_second_granule(surface_reflectance_file)) - + if not isinstance(data_files, list): + data_files = [data_files] + is_multiple = len(data_files) > 1 bytes_in_m_row = 4 * 3200 with dask.config.set({"array.chunk-size": f"{bytes_in_m_row * 4}B"}): - scn = Scene(reader="viirs_edr", filenames=files) + scn = Scene(reader="viirs_edr", filenames=data_files) scn.load(["surf_refl_I01", "surf_refl_M01"]) assert scn.start_time == START_TIME assert scn.end_time == END_TIME - _check_surf_refl_data_arr(scn["surf_refl_I01"], multiple_files=multiple_files) - _check_surf_refl_data_arr(scn["surf_refl_M01"], multiple_files=multiple_files) + _check_surf_refl_data_arr(scn["surf_refl_I01"], multiple_files=is_multiple) + _check_surf_refl_data_arr(scn["surf_refl_M01"], multiple_files=is_multiple) @pytest.mark.parametrize("filter_veg", [False, True]) - @pytest.mark.parametrize("multiple_files", [False, True]) + @pytest.mark.parametrize( + "data_files", + [ + lazy_fixture("surface_reflectance_with_veg_indices_file2"), + lazy_fixture("multiple_surface_reflectance_files_with_veg_indices"), + ], + ) def test_get_dataset_surf_refl_with_veg_idx( self, - surface_reflectance_with_veg_indices_file, + data_files, filter_veg, - multiple_files ): """Test retrieval of vegetation indices from surface reflectance files.""" from satpy import Scene - files = [surface_reflectance_with_veg_indices_file] - if multiple_files: - files.append(_copy_to_second_granule(surface_reflectance_with_veg_indices_file)) - + if not isinstance(data_files, list): + data_files = [data_files] + is_multiple = len(data_files) > 1 bytes_in_m_row = 4 * 3200 with dask.config.set({"array.chunk-size": f"{bytes_in_m_row * 4}B"}): - scn = Scene(reader="viirs_edr", filenames=files, + scn = Scene(reader="viirs_edr", filenames=data_files, reader_kwargs={"filter_veg": filter_veg}) scn.load(["NDVI", "EVI", "surf_refl_qf1"]) - _check_vi_data_arr(scn["NDVI"], filter_veg, multiple_files) - _check_vi_data_arr(scn["EVI"], filter_veg, multiple_files) - _check_surf_refl_qf_data_arr(scn["surf_refl_qf1"], multiple_files) + _check_vi_data_arr(scn["NDVI"], filter_veg, is_multiple) + _check_vi_data_arr(scn["EVI"], filter_veg, is_multiple) + _check_surf_refl_qf_data_arr(scn["surf_refl_qf1"], is_multiple) @pytest.mark.parametrize( ("var_names", "data_file"), From e5be4e3887f67c865a83590d04c5582a81bad270 Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 16 Aug 2023 15:15:41 -0500 Subject: [PATCH 57/59] Fix old colormap definition for VIIRS water detection --- satpy/etc/enhancements/viirs.yaml | 129 +++++++++++++++--------------- 1 file changed, 65 insertions(+), 64 deletions(-) diff --git a/satpy/etc/enhancements/viirs.yaml b/satpy/etc/enhancements/viirs.yaml index c740a2a6e6..8b3751167d 100644 --- a/satpy/etc/enhancements/viirs.yaml +++ b/satpy/etc/enhancements/viirs.yaml @@ -14,67 +14,68 @@ enhancements: - name: WaterDetection method: !!python/name:satpy.enhancements.viirs.water_detection kwargs: - palettes: {colors: - [[14, [0.0, 0.0, 0.0]], - [15, [0.0, 0.0, 0.39215686274509803]], - [16, [0.7686274509803922, 0.6352941176470588, 0.4470588235294118]], - [17, [0.7686274509803922, 0.6352941176470588, 0.4470588235294118]], - [18, [0.0, 0.0, 1.0]], - [20, [1.0, 1.0, 1.0]], - [27, [0.0, 1.0, 1.0]], - [30, [0.7843137254901961, 0.7843137254901961, 0.7843137254901961]], - [31, [0.39215686274509803, 0.39215686274509803, 0.39215686274509803]], - [88, [0.7058823529411765, 0.0, 0.9019607843137255]], - [100, [0.19607843137254902, 1.0, 0.39215686274509803]], - [120, [0.19607843137254902, 1.0, 0.39215686274509803]], - [121, [0.0, 1.0, 0.0]], - [130, [0.0, 1.0, 0.0]], - [131, [0.7843137254901961, 1.0, 0.0]], - [140, [0.7843137254901961, 1.0, 0.0]], - [141, [1.0, 1.0, 0.5882352941176471]], - [150, [1.0, 1.0, 0.5882352941176471]], - [151, [1.0, 1.0, 0.0]], - [160, [1.0, 1.0, 0.0]], - [161, [1.0, 0.7843137254901961, 0.0]], - [170, [1.0, 0.7843137254901961, 0.0]], - [171, [1.0, 0.5882352941176471, 0.19607843137254902]], - [180, [1.0, 0.5882352941176471, 0.19607843137254902]], - [181, [1.0, 0.39215686274509803, 0.0]], - [190, [1.0, 0.39215686274509803, 0.0]], - [191, [1.0, 0.0, 0.0]], - [200, [1.0, 0.0, 0.0]], - [201, [0.0, 0.0, 0.0]]], - min_value: 0, - max_value: 201} -# palettes: {colors: -# [[14, [0.0, 0.0, 0.0, 0.0]], -# [15, [0.0, 0.0, 0.39215686274509803, 1.0]], -# [16, [0.7686274509803922, 0.6352941176470588, 0.4470588235294118, 1.0]], -# [17, [0.7686274509803922, 0.6352941176470588, 0.4470588235294118, 1.0]], -# [18, [0.0, 0.0, 1.0, 1.0]], -# [20, [1.0, 1.0, 1.0, 1.0]], -# [27, [0.0, 1.0, 1.0, 1.0]], -# [30, [0.7843137254901961, 0.7843137254901961, 0.7843137254901961, 1.0]], -# [31, [0.39215686274509803, 0.39215686274509803, 0.39215686274509803, 1.0]], -# [88, [0.7058823529411765, 0.0, 0.9019607843137255, 1.0]], -# [100, [0.19607843137254902, 1.0, 0.39215686274509803, 1.0]], -# [120, [0.19607843137254902, 1.0, 0.39215686274509803, 1.0]], -# [121, [0.0, 1.0, 0.0, 1.0]], -# [130, [0.0, 1.0, 0.0, 1.0]], -# [131, [0.7843137254901961, 1.0, 0.0, 1.0]], -# [140, [0.7843137254901961, 1.0, 0.0, 1.0]], -# [141, [1.0, 1.0, 0.5882352941176471, 1.0]], -# [150, [1.0, 1.0, 0.5882352941176471, 1.0]], -# [151, [1.0, 1.0, 0.0, 1.0]], -# [160, [1.0, 1.0, 0.0, 1.0]], -# [161, [1.0, 0.7843137254901961, 0.0, 1.0]], -# [170, [1.0, 0.7843137254901961, 0.0, 1.0]], -# [171, [1.0, 0.5882352941176471, 0.19607843137254902, 1.0]], -# [180, [1.0, 0.5882352941176471, 0.19607843137254902, 1.0]], -# [181, [1.0, 0.39215686274509803, 0.0, 1.0]], -# [190, [1.0, 0.39215686274509803, 0.0, 1.0]], -# [191, [1.0, 0.0, 0.0, 1.0]], -# [200, [1.0, 0.0, 0.0, 1.0]], -# [201, [0.0, 0.0, 0.0, 0.0]]], -# min_value: 0, -# max_value: 201} + palettes: { + values: [ + 14, + 15, + 16, + 17, + 18, + 20, + 27, + 30, + 31, + 88, + 100, + 120, + 121, + 130, + 131, + 140, + 141, + 150, + 151, + 160, + 161, + 170, + 171, + 180, + 181, + 190, + 191, + 200, + 201, + ], + colors: [ + [0.0, 0.0, 0.0], + [0.0, 0.0, 0.39215686274509803], + [0.7686274509803922, 0.6352941176470588, 0.4470588235294118], + [0.7686274509803922, 0.6352941176470588, 0.4470588235294118], + [0.0, 0.0, 1.0], + [1.0, 1.0, 1.0], + [0.0, 1.0, 1.0], + [0.7843137254901961, 0.7843137254901961, 0.7843137254901961], + [0.39215686274509803, 0.39215686274509803, 0.39215686274509803], + [0.7058823529411765, 0.0, 0.9019607843137255], + [0.19607843137254902, 1.0, 0.39215686274509803], + [0.19607843137254902, 1.0, 0.39215686274509803], + [0.0, 1.0, 0.0], + [0.0, 1.0, 0.0], + [0.7843137254901961, 1.0, 0.0], + [0.7843137254901961, 1.0, 0.0], + [1.0, 1.0, 0.5882352941176471], + [1.0, 1.0, 0.5882352941176471], + [1.0, 1.0, 0.0], + [1.0, 1.0, 0.0], + [1.0, 0.7843137254901961, 0.0], + [1.0, 0.7843137254901961, 0.0], + [1.0, 0.5882352941176471, 0.19607843137254902], + [1.0, 0.5882352941176471, 0.19607843137254902], + [1.0, 0.39215686274509803, 0.0], + [1.0, 0.39215686274509803, 0.0], + [1.0, 0.0, 0.0], + [1.0, 0.0, 0.0], + [0.0, 0.0, 0.0], + ], + min_value: 0, + max_value: 201} From 04fe3e05a025892c52b450f1a3139ba853c00b1e Mon Sep 17 00:00:00 2001 From: David Hoese Date: Wed, 16 Aug 2023 15:16:24 -0500 Subject: [PATCH 58/59] Remove non-existent M06 surface reflectance product --- satpy/etc/readers/viirs_edr.yaml | 9 --------- 1 file changed, 9 deletions(-) diff --git a/satpy/etc/readers/viirs_edr.yaml b/satpy/etc/readers/viirs_edr.yaml index 4f33bcc184..c078e754aa 100644 --- a/satpy/etc/readers/viirs_edr.yaml +++ b/satpy/etc/readers/viirs_edr.yaml @@ -144,15 +144,6 @@ datasets: coordinates: [longitude_750, latitude_750] units: '%' standard_name: "surface_bidirectional_reflectance" - surf_refl_M06: - name: surf_refl_M06 - resolution: 750 - wavelength: [0.739, 0.746, 0.754] - file_type: [jrr_surfref] - file_key: "750m Surface Reflectance Band M6" - coordinates: [longitude_750, latitude_750] - units: '%' - standard_name: "surface_bidirectional_reflectance" surf_refl_M07: name: surf_refl_M07 resolution: 750 From 3b078daf487677edb6f2fbc8f95c2b1e69926adf Mon Sep 17 00:00:00 2001 From: David Hoese Date: Thu, 17 Aug 2023 09:24:57 -0500 Subject: [PATCH 59/59] Fix swath definitions not having all lon/lat metadata --- satpy/readers/viirs_edr.py | 13 +++++++++++++ satpy/tests/reader_tests/test_viirs_edr.py | 8 +++++++- 2 files changed, 20 insertions(+), 1 deletion(-) diff --git a/satpy/readers/viirs_edr.py b/satpy/readers/viirs_edr.py index b007e710b3..da61114fca 100644 --- a/satpy/readers/viirs_edr.py +++ b/satpy/readers/viirs_edr.py @@ -130,6 +130,11 @@ def get_dataset(self, dataset_id: DataID, info: dict) -> xr.DataArray: data_arr.attrs["platform_name"] = self.platform_name data_arr.attrs["sensor"] = self.sensor_name data_arr.attrs["rows_per_scan"] = self.rows_per_scans(data_arr) + if data_arr.attrs.get("standard_name") in ("longitude", "latitude"): + # recursive swath definitions are a problem for the base reader right now + # delete the coordinates here so the base reader doesn't try to + # make a SwathDefinition + data_arr = data_arr.reset_coords(drop=True) return data_arr def _mask_invalid(self, data_arr: xr.DataArray, ds_info: dict) -> xr.DataArray: @@ -201,6 +206,10 @@ def available_datasets(self, configured_datasets=None): handled_var_names = set() for is_avail, ds_info in (configured_datasets or []): file_key = ds_info.get("file_key", ds_info["name"]) + # we must add all variables here even if another file handler has + # claimed the variable. It could be another instance of this file + # type and we don't want to add that variable dynamically if the + # other file handler defined it by the YAML definition. handled_var_names.add(file_key) if is_avail is not None: # some other file handler said it has this dataset @@ -246,10 +255,14 @@ def _dynamic_variables_from_file(self, handled_var_names: set) -> Iterable[tuple ds_info["standard_name"] = "longitude" ds_info["units"] = "degrees_east" ds_info["name"] = m_lon_name if res == 750 else i_lon_name + # recursive coordinate/SwathDefinitions are not currently handled well in the base reader + del ds_info["coordinates"] elif is_lat: ds_info["standard_name"] = "latitude" ds_info["units"] = "degrees_north" ds_info["name"] = m_lat_name if res == 750 else i_lat_name + # recursive coordinate/SwathDefinitions are not currently handled well in the base reader + del ds_info["coordinates"] yield True, ds_info diff --git a/satpy/tests/reader_tests/test_viirs_edr.py b/satpy/tests/reader_tests/test_viirs_edr.py index 2acbd0d55a..146a0cd2c4 100644 --- a/satpy/tests/reader_tests/test_viirs_edr.py +++ b/satpy/tests/reader_tests/test_viirs_edr.py @@ -450,8 +450,14 @@ def _array_checks(data_arr: xr.DataArray, dtype: npt.Dtype = np.float32, multipl def _shared_metadata_checks(data_arr: xr.DataArray) -> None: is_mband_res = _is_mband_res(data_arr) + exp_rps = 16 if is_mband_res else 32 assert data_arr.attrs["sensor"] == "viirs" - assert data_arr.attrs["rows_per_scan"] == 16 if is_mband_res else 32 + assert data_arr.attrs["rows_per_scan"] == exp_rps + + lons = data_arr.attrs["area"].lons + lats = data_arr.attrs["area"].lats + assert lons.attrs["rows_per_scan"] == exp_rps + assert lats.attrs["rows_per_scan"] == exp_rps def _is_mband_res(data_arr: xr.DataArray) -> bool: