diff --git a/CHANGELOG.md b/CHANGELOG.md index 797b6381..798d984f 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -3,19 +3,29 @@ ## 0.1.7 (Upcoming) ### Bugs -* Use path relative to the current Zarr file in the definition of links and references to avoid breaking - links when moving Zarr files @oruebel [#46](https://github.com/hdmf-dev/hdmf-zarr/pull/46) -* Fix bugs in requirements defined in setup.py @oruebel [#46](https://github.com/hdmf-dev/hdmf-zarr/pull/46) -* Fix bug regarding Sphinx external links @mavaylon1 [#53](https://github.com/hdmf-dev/hdmf-zarr/pull/53) -* Updated gallery tests to use test_gallery.py and necessary package dependcies @mavaylon1 [#53](https://github.com/hdmf-dev/hdmf-zarr/pull/53) -* Update dateset used in conversion tutorial, which caused warnings @oruebel [#56](https://github.com/hdmf-dev/hdmf-zarr/pull/56) +* Updated the storage of links/references to use paths relative to the current Zarr file to avoid breaking + links/reference when moving Zarr files @oruebel [#46](https://github.com/hdmf-dev/hdmf-zarr/pull/46) +* Fixed bugs in requirements defined in setup.py @oruebel [#46](https://github.com/hdmf-dev/hdmf-zarr/pull/46) +* Fixed bug regarding Sphinx external links @mavaylon1 [#53](https://github.com/hdmf-dev/hdmf-zarr/pull/53) +* Updated gallery tests to use test_gallery.py and necessary package dependcies + @mavaylon1 [#53](https://github.com/hdmf-dev/hdmf-zarr/pull/53) +* Updated dateset used in conversion tutorial, which caused warnings + @oruebel [#56](https://github.com/hdmf-dev/hdmf-zarr/pull/56) ### Docs -* Add tutorial illustrating how to create a new NWB file with NWBZarrIO @oruebel [#46](https://github.com/hdmf-dev/hdmf-zarr/pull/46) -* Add docs for describing the mapping of HDMF schema to Zarr storage @oruebel [#48](https://github.com/hdmf-dev/hdmf-zarr/pull/48) +* Added tutorial illustrating how to create a new NWB file with NWBZarrIO + @oruebel [#46](https://github.com/hdmf-dev/hdmf-zarr/pull/46) +* Added docs for describing the mapping of HDMF schema to Zarr storage + @oruebel [#48](https://github.com/hdmf-dev/hdmf-zarr/pull/48) +* Added ``docs/gallery/resources`` for storing local files used by the tutorial galleries + @oruebel [#61](https://github.com/hdmf-dev/hdmf-zarr/pull/61) +* Removed dependency on ``dandi`` library for data download in the conversion tutorial by storing the NWB files as + local resources @oruebel [#61](https://github.com/hdmf-dev/hdmf-zarr/pull/61) ## 0.1.0 ### New features -- Created new optional Zarr-based I/O backend for writing files using Zarr's `zarr.store.DirectoryStore` backend, including support for iterative write, chunking, compression, simple and compound data types, links, object references, namespace and spec I/O. +* Created new optional Zarr-based I/O backend for writing files using Zarr's `zarr.store.DirectoryStore` backend, + including support for iterative write, chunking, compression, simple and compound data types, links, object + references, namespace and spec I/O. diff --git a/docs/gallery/plot_convert_nwb_hdf5.py b/docs/gallery/plot_convert_nwb_hdf5.py index bae2ade2..c26006f1 100644 --- a/docs/gallery/plot_convert_nwb_hdf5.py +++ b/docs/gallery/plot_convert_nwb_hdf5.py @@ -5,7 +5,8 @@ This tutorial illustrates how to convert data between HDF5 and Zarr using a Neurodata Without Borders (NWB) file from the DANDI data archive as an example. In this tutorial we will convert our example file from HDF5 to Zarr and then -back again to HDF5. +back again to HDF5. The NWB standard is defined using :hdmf-docs:`HDMF <>` and uses the +:py:class:`~ hdmf.backends.hdf5.h5tools.HDF5IO` HDF5 backend from HDMF for storage. """ @@ -13,29 +14,36 @@ # Setup # ----- # -# We first **download a small NWB file** from the DANDI neurophysiology data archive as an example. -# The NWB standard is defined using HDMF and uses the :py:class:`~ hdmf.backends.hdf5.h5tools.HDF5IO` -# HDF5 backend from HDMF for storage. +# Here we use a small NWB file from the DANDI neurophysiology data archive from +# `DANDIset 000009 `_ as an example. +# To download the file directly from DANDI we can use: +# +# .. code-block:: python +# :linenos: +# +# from dandi.dandiapi import DandiAPIClient +# dandiset_id = "000009" +# filepath = "sub-anm00239123/sub-anm00239123_ses-20170627T093549_ecephys+ogen.nwb" # ~0.5MB file +# with DandiAPIClient() as client: +# asset = client.get_dandiset(dandiset_id, 'draft').get_asset_by_path(filepath) +# s3_path = asset.get_content_url(follow_redirects=1, strip_query=True) +# filename = os.path.basename(asset.path) +# asset.download(filename) +# +# We here use a local copy of a small file from this DANDIset as an example: +# # sphinx_gallery_thumbnail_path = 'figures/gallery_thumbnail_plot_convert_nwb.png' import os import shutil -from dandi.dandiapi import DandiAPIClient - -dandiset_id = "000009" -filepath = "sub-anm00239123/sub-anm00239123_ses-20170627T093549_ecephys+ogen.nwb" # ~0.5MB file -with DandiAPIClient() as client: - asset = client.get_dandiset(dandiset_id, 'draft').get_asset_by_path(filepath) - s3_path = asset.get_content_url(follow_redirects=1, strip_query=True) - filename = os.path.basename(asset.path) -asset.download(filename) - -############################################################################### -# Next we define the names of the files to generate as part of this tutorial and clean up any -# data from previous executions of this tutorial. -zarr_filename = "test_zarr_" + filename + ".zarr" -hdf_filename = "test_hdf5_" + filename +# Input file to convert +basedir = "resources" +filename = os.path.join(basedir, "sub_anm00239123_ses_20170627T093549_ecephys_and_ogen.nwb") +# Zarr file to generate for converting from HDF5 to Zarr +zarr_filename = "test_zarr_" + os.path.basename(filename) + ".zarr" +# HDF5 file to generate for converting from Zarr to HDF5 +hdf_filename = "test_hdf5_" + os.path.basename(filename) # Delete our converted HDF5 and Zarr file from previous runs of this notebook for fname in [zarr_filename, hdf_filename]: @@ -60,8 +68,6 @@ with NWBHDF5IO(filename, 'r', load_namespaces=False) as read_io: # Create HDF5 IO object for read with NWBZarrIO(zarr_filename, mode='w') as export_io: # Create Zarr IO object for write export_io.export(src_io=read_io, write_args=dict(link_data=False)) # Export from HDF5 to Zarr - export_io.close() - read_io.close() ############################################################################### # .. note:: @@ -104,8 +110,6 @@ with NWBZarrIO(zarr_filename, mode='r') as read_io: # Create Zarr IO object for read with NWBHDF5IO(hdf_filename, 'w') as export_io: # Create HDF5 IO object for write export_io.export(src_io=read_io, write_args=dict(link_data=False)) # Export from Zarr to HDF5 - export_io.close() - read_io.close() ############################################################################### # Read the new HDF5 file back @@ -116,4 +120,3 @@ with NWBHDF5IO(hdf_filename, 'r') as hr: hf = hr.read() - hr.close() diff --git a/docs/gallery/plot_nwb_zarrio.py b/docs/gallery/plot_nwb_zarrio.py index 9edd1774..8065b262 100644 --- a/docs/gallery/plot_nwb_zarrio.py +++ b/docs/gallery/plot_nwb_zarrio.py @@ -134,7 +134,6 @@ absolute_path = os.path.abspath(path) with NWBZarrIO(path=path, mode="w") as io: io.write(nwbfile) - io.close() ############################################################################### # Test opening with the absolute path instead @@ -145,4 +144,3 @@ # relative ``path`` here instead is fine. with NWBZarrIO(path=absolute_path, mode="r") as io: infile = io.read() - io.close() diff --git a/docs/gallery/plot_zarr_dataset_io.py b/docs/gallery/plot_zarr_dataset_io.py index 89bb6cc2..753ac2f5 100644 --- a/docs/gallery/plot_zarr_dataset_io.py +++ b/docs/gallery/plot_zarr_dataset_io.py @@ -94,7 +94,6 @@ zarr_dir = "example_data.zarr" with ZarrIO(path=zarr_dir, manager=get_manager(), mode='w') as zarr_io: zarr_io.write(test_table) - zarr_io.close() ############################################################################### # reading the table from Zarr @@ -102,7 +101,6 @@ zarr_io = ZarrIO(path=zarr_dir, manager=get_manager(), mode='r') intable = zarr_io.read() intable.to_dataframe() -zarr_io.close() ############################################################################### # Check dataset settings used. @@ -112,3 +110,7 @@ (c.name, str(c.data.chunks), str(c.data.compressor))) + +############################################################################### +# +zarr_io.close() diff --git a/docs/gallery/plot_zarr_io.py b/docs/gallery/plot_zarr_io.py index bf103792..0d4965fe 100644 --- a/docs/gallery/plot_zarr_io.py +++ b/docs/gallery/plot_zarr_io.py @@ -78,7 +78,6 @@ zarr_dir = "example.zarr" with ZarrIO(path=zarr_dir, manager=get_manager(), mode='w') as zarr_io: zarr_io.write(users_table) - zarr_io.close() ############################################################################### # Reading the table from Zarr @@ -86,8 +85,10 @@ zarr_io = ZarrIO(path=zarr_dir, manager=get_manager(), mode='r') intable = zarr_io.read() intable.to_dataframe() -zarr_io.close() +############################################################################### +# +zarr_io.close() ############################################################################### # Converting to/from HDF5 using ``export`` @@ -105,8 +106,6 @@ with ZarrIO(path=zarr_dir, manager=get_manager(), mode='r') as zarr_read_io: with HDF5IO(path="example.h5", manager=get_manager(), mode='w') as hdf5_export_io: hdf5_export_io.export(src_io=zarr_read_io, write_args=dict(link_data=False)) # use export! - hdf5_export_io.close() - zarr_read_io.close() ############################################################################### # .. note:: @@ -121,7 +120,6 @@ intable_from_hdf5 = hdf5_read_io.read() intable_hdf5_df = intable_from_hdf5.to_dataframe() intable_hdf5_df # display the table in the gallery output -hdf5_read_io.close() ############################################################################### # Exporting the HDF5 file to Zarr @@ -134,8 +132,6 @@ with HDF5IO(path="example.h5", manager=get_manager(), mode='r') as hdf5_read_io: with ZarrIO(path="example_exp.zarr", manager=get_manager(), mode='w') as zarr_export_io: zarr_export_io.export(src_io=hdf5_read_io, write_args=dict(link_data=False)) # use export! - zarr_export_io.close() - hdf5_read_io.close() ############################################################################### # Check that the Zarr file is correct @@ -144,4 +140,3 @@ intable_from_zarr = zarr_read_io.read() intable_zarr_df = intable_from_zarr.to_dataframe() intable_zarr_df # display the table in the gallery output -zarr_read_io.close() diff --git a/docs/gallery/resources/README.rst b/docs/gallery/resources/README.rst new file mode 100644 index 00000000..1a762d3a --- /dev/null +++ b/docs/gallery/resources/README.rst @@ -0,0 +1,24 @@ +Resources +========= + +sub_anm00239123_ses_20170627T093549_ecephys_and_ogen.nwb +-------------------------------------------------------- + +This NWB file was downloaded from `DANDIset 000009 `_ +The file was modified to replace ``:`` characters used in the name of the ``ElectrodeGroup`` called ``ADunit: 32`` in +``'general/extracellular_ephys/`` to ``'ADunit_32'``. The dataset ``general/extracellular_ephys/electrodes/group_name`` +as part of the electrodes table was updated accordingly to list the appropriate group name. This is to avoid issues +on Windows file systems that do not support ``:`` as part of folder names. The asses can be downloaded from DANDI via: + +.. code-block:: python + :linenos: + + from dandi.dandiapi import DandiAPIClient + dandiset_id = "000009" + filepath = "sub-anm00239123/sub-anm00239123_ses-20170627T093549_ecephys+ogen.nwb" # ~0.5MB file + with DandiAPIClient() as client: + asset = client.get_dandiset(dandiset_id, 'draft').get_asset_by_path(filepath) + s3_path = asset.get_content_url(follow_redirects=1, strip_query=True) + filename = os.path.basename(asset.path) + asset.download(filename) + diff --git a/docs/gallery/resources/sub_anm00239123_ses_20170627T093549_ecephys_and_ogen.nwb b/docs/gallery/resources/sub_anm00239123_ses_20170627T093549_ecephys_and_ogen.nwb new file mode 100644 index 00000000..a7b4c87b Binary files /dev/null and b/docs/gallery/resources/sub_anm00239123_ses_20170627T093549_ecephys_and_ogen.nwb differ diff --git a/docs/source/conf.py b/docs/source/conf.py index 2dfedaef..de6ff8cc 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -77,8 +77,8 @@ intersphinx_mapping = { 'python': ('https://docs.python.org/3.10', None), 'numpy': ('https://numpy.org/doc/stable/', None), - 'scipy': ('https://docs.scipy.org/doc/scipy/reference', None), - 'matplotlib': ('https://matplotlib.org', None), + 'scipy': ('https://docs.scipy.org/doc/scipy/', None), + 'matplotlib': ('https://matplotlib.org/stable/', None), 'h5py': ('https://docs.h5py.org/en/latest/', None), 'pandas': ('https://pandas.pydata.org/pandas-docs/stable/', None), 'hdmf': ('https://hdmf.readthedocs.io/en/stable/', None), diff --git a/docs/source/overview.rst b/docs/source/overview.rst index eb58bdc0..d8ca65d7 100644 --- a/docs/source/overview.rst +++ b/docs/source/overview.rst @@ -30,3 +30,4 @@ Known Limitations - Currently the :py:class:`~hdmf_zarr.backend.ZarrIO` backend uses Zarr's :py:class:`~zarr.storage.DirectoryStore` only. Other `Zarr stores `_ could be added but will require proper treatment of links and references for those backends as links are not supported in Zarr (see `zarr-python issues #389 `_. - Exporting of HDF5 files with external links is not yet fully implemented/tested. (see `hdmf-zarr issue #49 `_. - Object references are currently always resolved on read (as are links) rather than being loaded lazily (see `hdmf-zarr issue #50 `_. +- Special characters (e.g., ``:``, ``<``, ``>``, ``"``, ``/``, ``\``, ``|``, ``?``, or ``*``) may not be supported by all file systems (e.g., on Windows) and as such should not be used as part of the names of Datasets or Groups as Zarr needs to create folders on the filesystem for these objects. diff --git a/requirements-doc.txt b/requirements-doc.txt index fcaf4996..32a790cf 100644 --- a/requirements-doc.txt +++ b/requirements-doc.txt @@ -4,4 +4,3 @@ sphinx>=4 # improved support for docutils>=0.17 sphinx_rtd_theme>=1 # <1 does not work with docutils>=0.17 sphinx-gallery sphinx-copybutton -dandi diff --git a/test.py b/test.py index 195ae477..a5a55603 100644 --- a/test.py +++ b/test.py @@ -1,7 +1,6 @@ #!/usr/bin/env python # NOTE this script is currently used in CI *only* to test the sphinx gallery examples using python test.py -e - import warnings import re import argparse @@ -67,6 +66,8 @@ def _import_from_file(script): def run_example_tests(): global TOTAL, FAILURES, ERRORS logging.info('running example tests') + + # get list of example scripts examples_scripts = list() for root, dirs, files in os.walk(os.path.join(os.path.dirname(__file__), "docs", "gallery")): for f in files: @@ -74,12 +75,18 @@ def run_example_tests(): examples_scripts.append(os.path.join(root, f)) TOTAL += len(examples_scripts) + curr_dir = os.getcwd() for script in examples_scripts: + os.chdir(curr_dir) # Reset the working directory + script_abs = os.path.abspath(script) # Determine the full path of the script + # Set the working dir to be relative to the script to allow the use of relative file paths in the scripts + os.chdir(os.path.dirname(script_abs)) try: logging.info("Executing %s" % script) ws = list() with warnings.catch_warnings(record=True) as tmp: - _import_from_file(script) + # Import/run the example gallery + _import_from_file(script_abs) for w in tmp: # ignore RunTimeWarnings about importing if isinstance(w.message, RuntimeWarning) and not warning_re.match(str(w.message)): ws.append(w) @@ -89,6 +96,8 @@ def run_example_tests(): print(traceback.format_exc()) FAILURES += 1 ERRORS += 1 + # Make sure to reset the working directory at the end + os.chdir(curr_dir) def main(): diff --git a/test_gallery.py b/test_gallery.py index 44e71a3b..731a9f0d 100644 --- a/test_gallery.py +++ b/test_gallery.py @@ -78,8 +78,13 @@ def run_gallery_tests(): warnings.simplefilter("error") TOTAL += len(gallery_file_names) + curr_dir = os.getcwd() for script in gallery_file_names: logging.info("Executing %s" % script) + os.chdir(curr_dir) # Reset the working directory + script_abs = os.path.abspath(script) # Determine the full path of the script + # Set the working dir to be relative to the script to allow the use of relative file paths in the scripts + os.chdir(os.path.dirname(script_abs)) try: with warnings.catch_warnings(record=True): warnings.filterwarnings( @@ -106,11 +111,13 @@ def run_gallery_tests(): # against a different version of numpy than the one installed "ignore", message=_numpy_warning_re, category=RuntimeWarning ) - _import_from_file(script) + _import_from_file(script_abs) except Exception: print(traceback.format_exc()) FAILURES += 1 ERRORS += 1 + # Make sure to reset the working directory at the end + os.chdir(curr_dir) def main():