From 25893fcfb6bab653cbfe7814d41d2e3e9931332e Mon Sep 17 00:00:00 2001 From: Simon Proud Date: Fri, 6 Sep 2024 21:45:36 +0200 Subject: [PATCH 1/6] Update S2/MSI RSR reader now that XLRD can't handle XLSX files. Add support for S2C. --- rsr_convert_scripts/msi_reader.py | 115 +++++++++++------------------- 1 file changed, 41 insertions(+), 74 deletions(-) diff --git a/rsr_convert_scripts/msi_reader.py b/rsr_convert_scripts/msi_reader.py index ab4ac3d0..2d9116e1 100644 --- a/rsr_convert_scripts/msi_reader.py +++ b/rsr_convert_scripts/msi_reader.py @@ -28,48 +28,35 @@ import logging import os +import pandas as pd import numpy as np -from xlrd import open_workbook from pyspectral.raw_reader import InstrumentRSR from pyspectral.utils import convert2hdf5 as tohdf5 LOG = logging.getLogger(__name__) - -MSI_BAND_NAMES = {} -MSI_BAND_NAMES['S2A'] = {'S2A_SR_AV_B1': 'B01', - 'S2A_SR_AV_B2': 'B02', - 'S2A_SR_AV_B3': 'B03', - 'S2A_SR_AV_B4': 'B04', - 'S2A_SR_AV_B5': 'B05', - 'S2A_SR_AV_B6': 'B06', - 'S2A_SR_AV_B7': 'B07', - 'S2A_SR_AV_B8': 'B08', - 'S2A_SR_AV_B8A': 'B8A', - 'S2A_SR_AV_B9': 'B09', - 'S2A_SR_AV_B10': 'B10', - 'S2A_SR_AV_B11': 'B11', - 'S2A_SR_AV_B12': 'B12'} -MSI_BAND_NAMES['S2B'] = {'S2B_SR_AV_B1': 'B01', - 'S2B_SR_AV_B2': 'B02', - 'S2B_SR_AV_B3': 'B03', - 'S2B_SR_AV_B4': 'B04', - 'S2B_SR_AV_B5': 'B05', - 'S2B_SR_AV_B6': 'B06', - 'S2B_SR_AV_B7': 'B07', - 'S2B_SR_AV_B8': 'B08', - 'S2B_SR_AV_B8A': 'B8A', - 'S2B_SR_AV_B9': 'B09', - 'S2B_SR_AV_B10': 'B10', - 'S2B_SR_AV_B11': 'B11', - 'S2B_SR_AV_B12': 'B12'} - -SHEET_HEADERS = {'Spectral Responses (S2A)': 'S2A', - 'Spectral Responses (S2B)': 'S2B'} - -PLATFORM_SHORT_NAME = {'Sentinel-2A': 'S2A', - 'Sentinel-2B': 'S2B'} +MSI_BAND_NAMES = {"B01": "B1", + "B02": "B2", + "B03": "B3", + "B04": "B4", + "B05": "B5", + "B06": "B6", + "B07": "B7", + "B08": "B8", + "B8A": "B8A", + "B09": "B9", + "B10": "B10", + "B11": "B11", + "B12": "B12"} + +SHEET_HEADERS = {"S2A": "Spectral Responses (S2A)", + "S2B": "Spectral Responses (S2B)", + "S2C": "Spectral Responses (S2C)"} + +PLATFORM_SHORT_NAME = {"Sentinel-2A": "S2A", + "Sentinel-2B": "S2B", + "Sentinel-2C": "S2C"} class MsiRSR(InstrumentRSR): @@ -79,56 +66,36 @@ def __init__(self, bandname, platform_name): """Read the Sentinel-2 MSI relative spectral responses for all channels.""" super(MsiRSR, self).__init__(bandname, platform_name) - self.instrument = 'msi' + self.instrument = "msi" + self.platform_name = platform_name + self.short_plat = PLATFORM_SHORT_NAME[platform_name] self._get_options_from_config() - LOG.debug("Filename: %s", str(self.path)) + LOG.debug(f"Filename: {self.path}") if os.path.exists(self.path): - self._load() + self._load(platform_name) else: - raise IOError("Couldn't find an existing file for this band: " + - str(self.bandname)) + raise IOError(f"Couldn't find an existing file for this band: {str(self.bandname)}") def _load(self, scale=0.001): """Load the Sentinel-2 MSI relative spectral responses.""" - with open_workbook(self.path) as wb_: - for sheet in wb_.sheets(): - if sheet.name not in SHEET_HEADERS.keys(): - continue - - plt_short_name = PLATFORM_SHORT_NAME.get(self.platform_name) - if plt_short_name != SHEET_HEADERS.get(sheet.name): - continue - wvl = sheet.col_values(0, 1) - for idx in range(1, sheet.row_len(0)): - ch_name = MSI_BAND_NAMES[plt_short_name].get(str(sheet.col_values(idx, 0, 1)[0])) - if ch_name != self.bandname: - continue + bname = MSI_BAND_NAMES.get(self.bandname) + df = pd.read_excel(self.path, engine='openpyxl', sheet_name=SHEET_HEADERS[self.short_plat]) + wvl = np.array(df['SR_WL']) + resp = np.array(df[f"{self.short_plat}_SR_AV_{bname}"]) - resp = sheet.col_values(idx, 1) - resp = np.array(resp) - resp = np.where(resp == '', 0, resp).astype('float32') - mask = np.less_equal(resp, 0.00001) - wvl0 = np.ma.masked_array(wvl, mask=mask) - wvl_mask = np.ma.masked_outside(wvl, wvl0.min() - 2, wvl0.max() + 2) + mask = np.less_equal(resp, 0.00001) + wvl0 = np.ma.masked_array(wvl, mask=mask) + wvl_mask = np.ma.masked_outside(wvl, wvl0.min() - 2, wvl0.max() + 2) - wvl = wvl_mask.compressed() - resp = np.ma.masked_array(resp, mask=wvl_mask.mask).compressed() - self.rsr = {'wavelength': wvl / 1000., 'response': resp} + wvl = wvl_mask.compressed() + resp = np.ma.masked_array(resp, mask=wvl_mask.mask).compressed() + self.rsr = {"wavelength": wvl / 1000., "response": resp} - break - - break if __name__ == "__main__": - bands = MSI_BAND_NAMES['S2A'].values() - bands.sort() - for platform_name in ['Sentinel-2A', ]: - tohdf5(MsiRSR, platform_name, bands) - - bands = MSI_BAND_NAMES['S2B'].values() - bands.sort() - for platform_name in ['Sentinel-2B', ]: - tohdf5(MsiRSR, platform_name, bands) + + for plat_name in ["Sentinel-2A", "Sentinel-2B", "Sentinel-2C"]: + tohdf5(MsiRSR, plat_name, sorted(MSI_BAND_NAMES)) From ce0544eacce420310094d172156b8221284c1e3f Mon Sep 17 00:00:00 2001 From: Simon Proud Date: Fri, 6 Sep 2024 21:46:12 +0200 Subject: [PATCH 2/6] Add support for S2C. --- pyspectral/etc/pyspectral.yaml | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/pyspectral/etc/pyspectral.yaml b/pyspectral/etc/pyspectral.yaml index a6f3aebb..06d0aca6 100644 --- a/pyspectral/etc/pyspectral.yaml +++ b/pyspectral/etc/pyspectral.yaml @@ -154,10 +154,12 @@ download_from_internet: True # Arctica-M-N1-msu-gsa: # path: /path/to/original/Arctica_M_N1_SRF.xlsx -# Sentinel-2A-msi: -# path: /path/to/original/sentinel-2a/msi/data/S2-SRF_COPE-GSEG-EOPG-TN-15-0007_3.0.xlsx -# Sentinel-2B-msi: -# path: /path/to/original/sentinel-2b/msi/data/S2-SRF_COPE-GSEG-EOPG-TN-15-0007_3.0.xlsx +#Sentinel-2A-msi: +# path: /path/to/COPE-GSEG-EOPG-TN-15-0007-Sentinel-2_Spectral_Response_Functions_2024-4.0.xlsx +#Sentinel-2B-msi: +# path: /path/to/COPE-GSEG-EOPG-TN-15-0007-Sentinel-2_Spectral_Response_Functions_2024-4.0.xlsx +#Sentinel-2C-msi: +# path: /path/to/COPE-GSEG-EOPG-TN-15-0007-Sentinel-2_Spectral_Response_Functions_2024-4.0.xlsx # Himawari-8-ahi: # path: /path/to/original/ahi/data From 1273cd6adf5abc0a5e46eecfb66a24cb4d01415f Mon Sep 17 00:00:00 2001 From: Simon Proud Date: Fri, 6 Sep 2024 21:48:49 +0200 Subject: [PATCH 3/6] Add support for S2C. --- doc/platforms_supported.rst | 3 +++ pyspectral/utils.py | 1 + rsr_convert_scripts/README.rst | 4 ++-- 3 files changed, 6 insertions(+), 2 deletions(-) diff --git a/doc/platforms_supported.rst b/doc/platforms_supported.rst index c861ec54..e01b5e46 100644 --- a/doc/platforms_supported.rst +++ b/doc/platforms_supported.rst @@ -85,6 +85,9 @@ have been included in Pyspectral. * - Sentinel-2B msi - `rsr_msi_Sentinel-2B.h5` - ESA-Sentinel-MSI_ + * - Sentinel-2C msi + - `rsr_msi_Sentinel-2C.h5` + - ESA-Sentinel-MSI_ * - NOAA-20 viirs - `rsr_viirs_NOAA-20.h5` - NESDIS_ diff --git a/pyspectral/utils.py b/pyspectral/utils.py index 4b564eb7..c38e996d 100644 --- a/pyspectral/utils.py +++ b/pyspectral/utils.py @@ -80,6 +80,7 @@ 'EOS-Terra': 'modis', 'Sentinel-2A': 'msi', 'Sentinel-2B': 'msi', + 'Sentinel-2C': 'msi', 'Arctica-M-N1': 'msu-gsa', 'Electro-L-N2': 'msu-gs', 'Sentinel-3A': ['olci', 'slstr'], diff --git a/rsr_convert_scripts/README.rst b/rsr_convert_scripts/README.rst index d45bebfc..c822e4b4 100644 --- a/rsr_convert_scripts/README.rst +++ b/rsr_convert_scripts/README.rst @@ -95,8 +95,8 @@ Terra files have names like this: ``rsr.1.oobd.det`` %> python msi_reader.py -The original Sentinel-2 A&B MSI spectral responses. Filenames look like this -``S2-SRF_COPE-GSEG-EOPG-TN-15-0007_3.0.xlsx`` +The original Sentinel-2 A,B, and C MSI spectral responses. Filenames look like this +``COPE-GSEG-EOPG-TN-15-0007-Sentinel-2_Spectral_Response_Functions_2024-4.0.xlsx`` .. code:: From 8731e43245c1fc96aaa87f7a0f03e7fba8d3ed5d Mon Sep 17 00:00:00 2001 From: Simon Proud Date: Fri, 6 Sep 2024 21:52:48 +0200 Subject: [PATCH 4/6] Tidying S2/MSI RSR reader. --- rsr_convert_scripts/msi_reader.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/rsr_convert_scripts/msi_reader.py b/rsr_convert_scripts/msi_reader.py index 2d9116e1..57f3f30e 100644 --- a/rsr_convert_scripts/msi_reader.py +++ b/rsr_convert_scripts/msi_reader.py @@ -79,7 +79,6 @@ def __init__(self, bandname, platform_name): def _load(self, scale=0.001): """Load the Sentinel-2 MSI relative spectral responses.""" - bname = MSI_BAND_NAMES.get(self.bandname) df = pd.read_excel(self.path, engine='openpyxl', sheet_name=SHEET_HEADERS[self.short_plat]) wvl = np.array(df['SR_WL']) @@ -94,7 +93,6 @@ def _load(self, scale=0.001): self.rsr = {"wavelength": wvl / 1000., "response": resp} - if __name__ == "__main__": for plat_name in ["Sentinel-2A", "Sentinel-2B", "Sentinel-2C"]: From e33217e82508509df6f17b9698c772f59c358b7f Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 6 Sep 2024 19:53:37 +0000 Subject: [PATCH 5/6] [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci --- rsr_convert_scripts/msi_reader.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/rsr_convert_scripts/msi_reader.py b/rsr_convert_scripts/msi_reader.py index 57f3f30e..4ee1d605 100644 --- a/rsr_convert_scripts/msi_reader.py +++ b/rsr_convert_scripts/msi_reader.py @@ -28,8 +28,8 @@ import logging import os -import pandas as pd import numpy as np +import pandas as pd from pyspectral.raw_reader import InstrumentRSR from pyspectral.utils import convert2hdf5 as tohdf5 From 85fa0802a688763f0434f4937d86b4d47b3defc6 Mon Sep 17 00:00:00 2001 From: "Adam.Dybbroe" Date: Tue, 24 Sep 2024 15:42:53 +0200 Subject: [PATCH 6/6] Updated the link to the new data set on Zenodo Signed-off-by: Adam.Dybbroe --- pyspectral/utils.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/pyspectral/utils.py b/pyspectral/utils.py index c38e996d..08b521f5 100644 --- a/pyspectral/utils.py +++ b/pyspectral/utils.py @@ -106,10 +106,11 @@ 'avhrr-2': 'avhrr/2', 'avhrr-3': 'avhrr/3'} -HTTP_PYSPECTRAL_RSR = "https://zenodo.org/records/11110033/files/pyspectral_rsr_data.tgz" +HTTP_PYSPECTRAL_RSR = "https://zenodo.org/records/13833977/files/pyspectral_rsr_data.tgz" + RSR_DATA_VERSION_FILENAME = "PYSPECTRAL_RSR_VERSION" -RSR_DATA_VERSION = "v1.3.0" +RSR_DATA_VERSION = "v1.4.0" ATM_CORRECTION_LUT_VERSION = {} ATM_CORRECTION_LUT_VERSION['antarctic_aerosol'] = {'version': 'v1.0.1',