Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Jump to bottom

Add ability to load 2D flux arrays in Specviz #3229

Draft
wants to merge 8 commits into
base: main
Choose a base branch
from
Draft

Add ability to load 2D flux arrays in Specviz #3229

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
8 commits
Select commit Hold shift + click to select a range
d6745a5
Add ability to load 2D flux arrays in Specviz by splitting into separ…
rosteen Oct 18, 2024
ccc30fa
Changelog
rosteen Oct 18, 2024
9124201
Handle this in file load case
rosteen Oct 18, 2024
aa0cb56
Fix typo
rosteen Oct 18, 2024
180467a
Add test and debug
rosteen Oct 18, 2024
620c6ae
Codestyle again, whoops
rosteen Oct 18, 2024
4703b9d
Add keyword to force Specviz parser to use SpectrumList.read
rosteen Oct 18, 2024
fd5dc17
Add new keyword to helper load_data
rosteen Oct 18, 2024
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGES.rst
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@ Mosviz

Specviz
^^^^^^^
- Specviz parser will now split a spectrum with a 2D flux array into multiple spectra on load. [#3229]

Specviz2d
^^^^^^^^^
Expand Down
6 changes: 4 additions & 2 deletions jdaviz/configs/specviz/helper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -41,7 +41,8 @@ def __init__(self, *args, **kwargs):
handler=self._redshift_listener)

def load_data(self, data, data_label=None, format=None, show_in_viewer=True,
concat_by_file=False, cache=None, local_path=None, timeout=None):
concat_by_file=False, cache=None, local_path=None, timeout=None,
load_as_list=False):
"""
Load data into Specviz.

Expand Down Expand Up @@ -80,7 +81,8 @@ def load_data(self, data, data_label=None, format=None, show_in_viewer=True,
concat_by_file=concat_by_file,
cache=cache,
local_path=local_path,
timeout=timeout)
timeout=timeout,
load_as_list=load_as_list)

def get_spectra(self, data_label=None, spectral_subset=None, apply_slider_redshift="Warn"):
"""Returns the current data loaded into the main viewer
Expand Down
51 changes: 40 additions & 11 deletions jdaviz/configs/specviz/plugins/parsers.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -18,7 +18,8 @@

@data_parser_registry("specviz-spectrum1d-parser")
def specviz_spectrum1d_parser(app, data, data_label=None, format=None, show_in_viewer=True,
concat_by_file=False, cache=None, local_path=os.curdir, timeout=None):
concat_by_file=False, cache=None, local_path=os.curdir, timeout=None,
load_as_list=False):
"""
Loads a data file or `~specutils.Spectrum1D` object into Specviz.

Expand Down Expand Up @@ -46,6 +47,9 @@
remote requests in seconds (passed to
`~astropy.utils.data.download_file` or
`~astroquery.mast.Conf.timeout`).
load_as_list : bool, optional
Force the parser to load the input file with the `~specutils.SpectrumList` read function
instead of `~specutils.Spectrum1D`.
"""

spectrum_viewer_reference_name = app._jdaviz_helper._default_spectrum_viewer_reference_name
Expand All @@ -56,8 +60,12 @@
raise TypeError("SpectrumCollection detected."
" Please provide a Spectrum1D or SpectrumList")
elif isinstance(data, Spectrum1D):
data_label = [app.return_data_label(data_label, alt_name="specviz_data")]
data = [data]
# Handle the possibility of 2D spectra by splitting into separate spectra
if data.flux.ndim == 1:
data_label = [app.return_data_label(data_label, alt_name="specviz_data")]
data = [data]
elif data.flux.ndim == 2:
data, data_label = split_spectrum_with_2D_flux_array(data, data_label, app)
# No special processing is needed in this case, but we include it for completeness
elif isinstance(data, SpectrumList):
pass
Expand All @@ -77,19 +85,23 @@

path = pathlib.Path(data)

if path.is_file():
if path.is_dir() or load_as_list:
data = SpectrumList.read(str(path), format=format)
if data == []:
raise ValueError(f"`specutils.SpectrumList.read('{str(path)}')` "

Check warning on line 91 in jdaviz/configs/specviz/plugins/parsers.py

View check run for this annotation

Codecov / codecov/patch

jdaviz/configs/specviz/plugins/parsers.py#L91 

Added line #L91 was not covered by tests
"returned an empty list")
elif path.is_file():
try:
data = [Spectrum1D.read(str(path), format=format)]
data_label = [app.return_data_label(data_label, alt_name="specviz_data")]
data = Spectrum1D.read(str(path), format=format)
if data.flux.ndim == 2:
data, data_label = split_spectrum_with_2D_flux_array(data, data_label, app)

Check warning on line 97 in jdaviz/configs/specviz/plugins/parsers.py

View check run for this annotation

Codecov / codecov/patch

jdaviz/configs/specviz/plugins/parsers.py#L97 

Added line #L97 was not covered by tests
Comment on lines +93 to +97
Copy link
Collaborator

@havok2063 havok2063 Oct 18, 2024 •
edited
Loading

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

In our file case, there are 2 data extensions, with the 1st having a 1d flux array, and the 2nd one having the nd flux array. Without the load_as_list flag, this block only loads the first extension and never reaches line 97. With load_as_list set, which is the way I think we need to load these files, I get the error

File [~/Work/jdaviz/jdaviz/configs/specviz/plugins/parsers.py:129](http://localhost:8888/lab/workspaces/auto-b/tree/~/Work/jdaviz/jdaviz/configs/specviz/plugins/parsers.py#line=128), in specviz_spectrum1d_parser(app, data, data_label, format, show_in_viewer, concat_by_file, cache, local_path, timeout, load_as_list)
    127 for wlind in range(len(spec.spectral_axis)):
    128     wlallorig.append(spec.spectral_axis[wlind].value)
--> 129     fnuallorig.append(spec.flux[wlind].value)
    131     # because some spec in the list might have uncertainties and
    132     # others may not, if uncert is None, set to list of NaN. later,
    133     # if the concatenated list of uncertanties is all nan (meaning
    134     # they were all nan to begin with, or all None), it will be set
    135     # to None on the final Spectrum1D
    136     if spec.uncertainty[wlind] is not None:

File [~/anaconda3/envs/newsv/lib/python3.11/site-packages/astropy/units/quantity.py:1302](http://localhost:8888/lab/workspaces/auto-b/tree/~/anaconda3/envs/newsv/lib/python3.11/site-packages/astropy/units/quantity.py#line=1301), in Quantity.__getitem__(self, key)
   1297     return self._new_view(
   1298         self.view(np.ndarray)[key], self.unit[key], propagate_info=False
   1299     )
   1301 try:
-> 1302     out = super().__getitem__(key)
   1303 except IndexError:
   1304     # We want zero-dimensional Quantity objects to behave like scalars,
   1305     # so they should raise a TypeError rather than an IndexError.
   1306     if self.isscalar:

IndexError: index 1 is out of bounds for axis 0 with size 1

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

I think what I've done should fix this bug. It occurs when there's 2D spectra in a SpectrumList.

Copy link
Collaborator

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, with your PR it loads correctly, but your PR is basically unpacking the nd arrays into individual Spectrum1D objects, which this draft code is also doing. In principle, with the change here, we don't need to manually unpack the nd fluxes inside the loader.

This draft code should work loading nd-flux data more generally, outside of the specific mwm loader

Copy link

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes, I've already reverted that back to how it was initially, see specutils#1185, ca4d8c

else:
data = [data]
data_label = [app.return_data_label(data_label, alt_name="specviz_data")]

except IORegistryError:
# Multi-extension files may throw a registry error
data = SpectrumList.read(str(path), format=format)
elif path.is_dir():
data = SpectrumList.read(str(path), format=format)
if data == []:
raise ValueError(f"`specutils.SpectrumList.read('{str(path)}')` "
"returned an empty list")
else:
raise FileNotFoundError("No such file: " + str(path))

Expand Down Expand Up @@ -246,3 +258,20 @@
spec = Spectrum1D(flux=fnuall * flux_units, spectral_axis=wlall * wave_units,
uncertainty=unc)
return spec


def split_spectrum_with_2D_flux_array(data, data_label, app):
temp_data = []
temp_data_label = []
for i in range(data.flux.shape[0]):
unc = None
mask = None
if data.uncertainty is not None:
unc = data.uncertainty[i, :]

Check warning on line 270 in jdaviz/configs/specviz/plugins/parsers.py

View check run for this annotation

Codecov / codecov/patch

jdaviz/configs/specviz/plugins/parsers.py#L270 

Added line #L270 was not covered by tests
if mask is not None:
mask = data.mask[i, :]

Check warning on line 272 in jdaviz/configs/specviz/plugins/parsers.py

View check run for this annotation

Codecov / codecov/patch

jdaviz/configs/specviz/plugins/parsers.py#L272 

Added line #L272 was not covered by tests
temp_data.append(Spectrum1D(flux=data.flux[i, :], spectral_axis=data.spectral_axis,
uncertainty=unc))
temp_data_label.append(f'{app.return_data_label(data_label, alt_name="specviz_data")}[{i}]') # noqa

return temp_data, temp_data_label
9 changes: 9 additions & 0 deletions jdaviz/configs/specviz/tests/test_helper.py
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -405,6 +405,15 @@ def test_load_spectrum_list_directory_concat(tmpdir, specviz_helper):
assert len(specviz_helper.app.data_collection) == 41


def test_load_spectrum1d_2d_flux(specviz_helper):
spec = Spectrum1D(spectral_axis=np.linspace(4000, 6000, 100)*u.Angstrom,
flux=np.ones((4, 100))*u.Unit("1e-17 erg / (Angstrom cm2 s)"))
specviz_helper.load_data(spec, data_label="test")

assert len(specviz_helper.app.data_collection) == 4
assert specviz_helper.app.data_collection[0].label == "test[0]"


def test_plot_uncertainties(specviz_helper, spectrum1d):
specviz_helper.load_data(spectrum1d)

Expand Down
Loading