Append a Dataset of References

CHANGELOG.md

@@ -7,6 +7,7 @@
 - Support appending to zarr arrays. @mavaylon1 [#1136](https://github.com/hdmf-dev/hdmf/pull/1136)
 - Support specifying "value" key in DatasetSpec. @rly [#1143](https://github.com/hdmf-dev/hdmf/pull/1143)
 - Add support for numpy 2. @rly [#1139](https://github.com/hdmf-dev/hdmf/pull/1139)
+- Add support for appending to a dataset of references. @mavaylon1 [#1135](https://github.com/hdmf-dev/hdmf/pull/1135)
 
 ### Bug fixes
 - Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128)

docs/source/install_developers.rst

@@ -73,7 +73,7 @@ environment by using the ``conda remove --name hdmf-venv --all`` command.
     For advanced users, we recommend using Mambaforge_, a faster version of the conda package manager
     that includes conda-forge as a default channel.
 
-.. _Anaconda: https://www.anaconda.com/products/distribution
+.. _Anaconda: https://www.anaconda.com
 .. _Mambaforge: https://github.com/conda-forge/miniforge
 
 Install from GitHub

docs/source/install_users.rst

@@ -29,4 +29,4 @@ You can also install HDMF using ``conda`` by running the following command in a
 
    conda install -c conda-forge hdmf
 
-.. _Anaconda Distribution: https://www.anaconda.com/products/distribution
+.. _Anaconda Distribution: https://www.anaconda.com/products

src/hdmf/build/objectmapper.py

@@ -11,6 +11,8 @@
                      ConstructError)
 from .manager import Proxy, BuildManager
 from .warnings import MissingRequiredBuildWarning, DtypeConversionWarning, IncorrectQuantityBuildWarning
+from hdmf.backends.hdf5.h5_utils import H5DataIO
+
 from ..container import AbstractContainer, Data, DataRegion
 from ..term_set import TermSetWrapper
 from ..data_utils import DataIO, AbstractDataChunkIterator
@@ -889,6 +891,8 @@ def __get_ref_builder(self, builder, dtype, shape, container, build_manager):
                 for d in container.data:
                     target_builder = self.__get_target_builder(d, build_manager, builder)
                     bldr_data.append(ReferenceBuilder(target_builder))
+                if isinstance(container.data, H5DataIO):
+                    bldr_data = H5DataIO(bldr_data, **container.data.get_io_params())
             else:
                 self.logger.debug("Setting %s '%s' data to reference builder"
                                   % (builder.__class__.__name__, builder.name))

src/hdmf/data_utils.py

@@ -18,7 +18,9 @@
 from .utils import docval, getargs, popargs, docval_macro, get_data_shape
 
 def append_data(data, arg):
-    if isinstance(data, (list, DataIO)):
+    from hdmf.backends.hdf5.h5_utils import HDMFDataset
+
+    if isinstance(data, (list, DataIO, HDMFDataset)):
         data.append(arg)
         return data
     elif type(data).__name__ == 'TermSetWrapper': # circular import

src/hdmf/query.py

@@ -4,6 +4,7 @@
 
 from .array import Array
 from .utils import ExtenderMeta, docval_macro, docval, getargs
+from .data_utils import append_data
 
 
 class Query(metaclass=ExtenderMeta):
@@ -163,6 +164,9 @@ def __next__(self):
     def next(self):
         return self.dataset.next()
 
+    def append(self, arg):
+        append_data(self.dataset, arg)
+
 
 class ReferenceResolver(metaclass=ABCMeta):
     """

tests/unit/test_io_hdf5_h5tools.py

@@ -2958,6 +2958,27 @@ def test_append_data(self):
             self.assertEqual(f['foofile_data'].file.filename, self.paths[1])
             self.assertIsInstance(f.attrs['foo_ref_attr'], h5py.Reference)
 
+    def test_append_dataset_of_references(self):
+        """Test that exporting a written container with a dataset of references works."""
+        bazs = []
+        num_bazs = 10
+        for i in range(num_bazs):
+            bazs.append(Baz(name='baz%d' % i))
+        array_bazs=np.array(bazs)
+        wrapped_bazs = H5DataIO(array_bazs, maxshape=(None,))
+        baz_data = BazData(name='baz_data1', data=wrapped_bazs)
+        bucket = BazBucket(name='bucket1', bazs=bazs.copy(), baz_data=baz_data)
+
+        with HDF5IO(self.paths[0], manager=get_baz_buildmanager(), mode='w') as write_io:
+            write_io.write(bucket)
+
+        with HDF5IO(self.paths[0], manager=get_baz_buildmanager(), mode='a') as read_io:
+            read_bucket1 = read_io.read()
+            DoR = read_bucket1.baz_data.data
+            ref = read_bucket1.baz_data.data.dataset[0]
+            DoR.append(ref)
+            self.assertEqual(len(read_bucket1.baz_data.data), 11)
+
     def test_append_external_link_data(self):
         """Test that exporting a written container after adding a link with link_data=True creates external links."""
         foo1 = Foo('foo1', [1, 2, 3, 4, 5], "I am foo1", 17, 3.14)