From 1f6aa5d47cd50e54b3aad85d2bac667f50f98915 Mon Sep 17 00:00:00 2001 From: Steph Prince <40640337+stephprince@users.noreply.github.com> Date: Thu, 25 Apr 2024 12:25:17 -0700 Subject: [PATCH 01/33] ignore sphinx cache warnings (#1105) --- docs/source/conf.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/docs/source/conf.py b/docs/source/conf.py index caff737e7..9781933f5 100644 --- a/docs/source/conf.py +++ b/docs/source/conf.py @@ -93,6 +93,8 @@ ('py:class', 'unittest.case.TestCase'), ] +suppress_warnings = ["config.cache"] + # Add any paths that contain templates here, relative to this directory. templates_path = ["_templates"] From 3146c97b68bf8149b7635f5505969fdfd01439af Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Fri, 26 Apr 2024 15:37:57 -0500 Subject: [PATCH 02/33] Workflows to support python 3.9 and 3.8 with Mac OS 13 (#1104) --- .github/workflows/run_all_tests.yml | 6 +++--- .github/workflows/run_tests.yml | 2 +- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml index d5f8afc7f..67a645037 100644 --- a/.github/workflows/run_all_tests.yml +++ b/.github/workflows/run_all_tests.yml @@ -41,8 +41,8 @@ jobs: - { name: windows-python3.12 , test-tox-env: pytest-py312-pinned , python-ver: "3.12", os: windows-latest } - { name: windows-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: windows-latest } - { name: windows-python3.12-prerelease , test-tox-env: pytest-py312-prerelease , python-ver: "3.12", os: windows-latest } - - { name: macos-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: macos-latest } - - { name: macos-python3.9 , test-tox-env: pytest-py39-pinned , python-ver: "3.9" , os: macos-latest } + - { name: macos-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: macos-13 } + - { name: macos-python3.9 , test-tox-env: pytest-py39-pinned , python-ver: "3.9" , os: macos-13 } - { name: macos-python3.10 , test-tox-env: pytest-py310-pinned , python-ver: "3.10", os: macos-latest } - { name: macos-python3.11 , test-tox-env: pytest-py311-pinned , python-ver: "3.11", os: macos-latest } - { name: macos-python3.11-optional , test-tox-env: pytest-py311-optional-pinned , python-ver: "3.11", os: macos-latest } @@ -105,7 +105,7 @@ jobs: - { name: windows-gallery-python3.11-optional , test-tox-env: gallery-py311-optional-pinned , python-ver: "3.11", os: windows-latest } - { name: windows-gallery-python3.12-upgraded , test-tox-env: gallery-py312-upgraded , python-ver: "3.12", os: windows-latest } - { name: windows-gallery-python3.12-prerelease, test-tox-env: gallery-py312-prerelease , python-ver: "3.12", os: windows-latest } - - { name: macos-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: macos-latest } + - { name: macos-gallery-python3.8-minimum , test-tox-env: gallery-py38-minimum , python-ver: "3.8" , os: macos-13 } - { name: macos-gallery-python3.11-optional , test-tox-env: gallery-py311-optional-pinned , python-ver: "3.11", os: macos-latest } - { name: macos-gallery-python3.12-upgraded , test-tox-env: gallery-py312-upgraded , python-ver: "3.12", os: macos-latest } - { name: macos-gallery-python3.12-prerelease , test-tox-env: gallery-py312-prerelease , python-ver: "3.12", os: macos-latest } diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 049cec2e5..d800d86f1 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -28,7 +28,7 @@ jobs: - { name: linux-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: ubuntu-latest , upload-wheels: true } - { name: windows-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: windows-latest } - { name: windows-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: windows-latest } - - { name: macos-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: macos-latest } + - { name: macos-python3.8-minimum , test-tox-env: pytest-py38-minimum , python-ver: "3.8" , os: macos-13 } - { name: macos-python3.12-upgraded , test-tox-env: pytest-py312-upgraded , python-ver: "3.12", os: macos-latest } steps: - name: Checkout repo with submodules From 126bdb100c6d5ce3e2dadd375de9d32524219404 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Wed, 1 May 2024 16:34:34 -0700 Subject: [PATCH 03/33] Don't install in editable/develop mode during testing (#1107) --- .github/workflows/run_all_tests.yml | 2 +- .github/workflows/run_coverage.yml | 8 ++++---- .github/workflows/run_hdmf_zarr_tests.yml | 3 +-- .github/workflows/run_pynwb_tests.yml | 3 +-- .github/workflows/run_tests.yml | 2 +- CHANGELOG.md | 1 + pyproject.toml | 11 +++++------ tox.ini | 1 - 8 files changed, 14 insertions(+), 17 deletions(-) diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml index 67a645037..def51537f 100644 --- a/.github/workflows/run_all_tests.yml +++ b/.github/workflows/run_all_tests.yml @@ -233,7 +233,7 @@ jobs: - name: Install run dependencies run: | - pip install -e . + pip install . pip list - name: Conda reporting diff --git a/.github/workflows/run_coverage.yml b/.github/workflows/run_coverage.yml index 7a18e5068..a72a05e73 100644 --- a/.github/workflows/run_coverage.yml +++ b/.github/workflows/run_coverage.yml @@ -55,18 +55,18 @@ jobs: - name: Install package run: | - python -m pip install -e . # must install in editable mode for coverage to find sources + python -m pip install . python -m pip list - name: Run tests and generate coverage report run: | - pytest --cov - python -m coverage xml # codecov uploader requires xml format - python -m coverage report -m + # coverage is configured in pyproject.toml + pytest --cov --cov-report=xml --cov-report=term # codecov uploader requires xml format - name: Upload coverage to Codecov uses: codecov/codecov-action@v4 with: fail_ci_if_error: true + file: ./coverage.xml env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/run_hdmf_zarr_tests.yml b/.github/workflows/run_hdmf_zarr_tests.yml index ecfdeaeeb..5e76711af 100644 --- a/.github/workflows/run_hdmf_zarr_tests.yml +++ b/.github/workflows/run_hdmf_zarr_tests.yml @@ -32,8 +32,7 @@ jobs: git clone https://github.com/hdmf-dev/hdmf-zarr.git --recurse-submodules cd hdmf-zarr python -m pip install -r requirements-dev.txt # do not install the pinned install requirements - # must install in editable mode for coverage to find sources - python -m pip install -e . # this will install a different version of hdmf from the current one + python -m pip install . # this will install a different version of hdmf from the current one cd .. python -m pip uninstall -y hdmf # uninstall the other version of hdmf python -m pip install . # reinstall current branch of hdmf diff --git a/.github/workflows/run_pynwb_tests.yml b/.github/workflows/run_pynwb_tests.yml index bf3f32343..1a714ed9f 100644 --- a/.github/workflows/run_pynwb_tests.yml +++ b/.github/workflows/run_pynwb_tests.yml @@ -32,8 +32,7 @@ jobs: git clone https://github.com/NeurodataWithoutBorders/pynwb.git --recurse-submodules cd pynwb python -m pip install -r requirements-dev.txt # do not install the pinned install requirements - # must install in editable mode for coverage to find sources - python -m pip install -e . # this will install a different version of hdmf from the current one + python -m pip install . # this will install a different version of hdmf from the current one cd .. python -m pip uninstall -y hdmf # uninstall the other version of hdmf python -m pip install . # reinstall current branch of hdmf diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index d800d86f1..2723e03d0 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -243,7 +243,7 @@ jobs: - name: Install run dependencies run: | - pip install -e . + pip install . pip list - name: Conda reporting diff --git a/CHANGELOG.md b/CHANGELOG.md index 00eeeb5dd..5d5a2cc62 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -6,6 +6,7 @@ - Updated `_field_config` to take `type_map` as an argument for APIs. @mavaylon1 [#1094](https://github.com/hdmf-dev/hdmf/pull/1094) - Added `TypeConfigurator` to automatically wrap fields with `TermSetWrapper` according to a configuration file. @mavaylon1 [#1016](https://github.com/hdmf-dev/hdmf/pull/1016) - Updated `TermSetWrapper` to support validating a single field within a compound array. @mavaylon1 [#1061](https://github.com/hdmf-dev/hdmf/pull/1061) +- Updated testing to not install in editable mode and not run `coverage` by default. @rly [#1107](https://github.com/hdmf-dev/hdmf/pull/1107) ## HDMF 3.13.0 (March 20, 2024) diff --git a/pyproject.toml b/pyproject.toml index b60ae6943..e5584b581 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -77,7 +77,6 @@ packages = ["src/hdmf"] # verbose = 1 [tool.pytest.ini_options] -addopts = "--cov --cov-report html" norecursedirs = "tests/unit/helpers" [tool.codespell] @@ -86,17 +85,17 @@ ignore-words-list = "datas" [tool.coverage.run] branch = true -source = ["src/"] -omit = [ - "src/hdmf/_due.py", - "src/hdmf/testing/*", -] +source = ["hdmf"] [tool.coverage.report] exclude_lines = [ "pragma: no cover", "@abstract" ] +omit = [ + "*/hdmf/_due.py", + "*/hdmf/testing/*", +] # [tool.black] # line-length = 120 diff --git a/tox.ini b/tox.ini index aeb743c45..75b011aa0 100644 --- a/tox.ini +++ b/tox.ini @@ -8,7 +8,6 @@ requires = pip >= 22.0 [testenv] download = True -usedevelop = True setenv = PYTHONDONTWRITEBYTECODE = 1 VIRTUALENV_PIP = 23.3.1 From 6377b43ae39d16015c940845dc5aae4a6874e6e6 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Tue, 14 May 2024 17:51:48 -0700 Subject: [PATCH 04/33] Post init option for class generator (#1089) --- CHANGELOG.md | 1 + src/hdmf/build/classgenerator.py | 32 ++++++- src/hdmf/build/manager.py | 15 ++- src/hdmf/common/__init__.py | 7 +- tests/unit/build_tests/test_classgenerator.py | 91 +++++++++++++++++-- 5 files changed, 133 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5d5a2cc62..909ef5253 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -7,6 +7,7 @@ - Added `TypeConfigurator` to automatically wrap fields with `TermSetWrapper` according to a configuration file. @mavaylon1 [#1016](https://github.com/hdmf-dev/hdmf/pull/1016) - Updated `TermSetWrapper` to support validating a single field within a compound array. @mavaylon1 [#1061](https://github.com/hdmf-dev/hdmf/pull/1061) - Updated testing to not install in editable mode and not run `coverage` by default. @rly [#1107](https://github.com/hdmf-dev/hdmf/pull/1107) +- Add `post_init_method` parameter when generating classes to perform post-init functionality, i.e., validation. @mavaylon1 [#1089](https://github.com/hdmf-dev/hdmf/pull/1089) ## HDMF 3.13.0 (March 20, 2024) diff --git a/src/hdmf/build/classgenerator.py b/src/hdmf/build/classgenerator.py index d2e7d4fc0..a3336b98e 100644 --- a/src/hdmf/build/classgenerator.py +++ b/src/hdmf/build/classgenerator.py @@ -1,5 +1,6 @@ from copy import deepcopy from datetime import datetime, date +from collections.abc import Callable import numpy as np @@ -35,6 +36,8 @@ def register_generator(self, **kwargs): {'name': 'spec', 'type': BaseStorageSpec, 'doc': ''}, {'name': 'parent_cls', 'type': type, 'doc': ''}, {'name': 'attr_names', 'type': dict, 'doc': ''}, + {'name': 'post_init_method', 'type': Callable, 'default': None, + 'doc': 'The function used as a post_init method to validate the class generation.'}, {'name': 'type_map', 'type': 'hdmf.build.manager.TypeMap', 'doc': ''}, returns='the class for the given namespace and data_type', rtype=type) def generate_class(self, **kwargs): @@ -42,8 +45,10 @@ def generate_class(self, **kwargs): If no class has been associated with the ``data_type`` from ``namespace``, a class will be dynamically created and returned. """ - data_type, spec, parent_cls, attr_names, type_map = getargs('data_type', 'spec', 'parent_cls', 'attr_names', - 'type_map', kwargs) + data_type, spec, parent_cls, attr_names, type_map, post_init_method = getargs('data_type', 'spec', + 'parent_cls', 'attr_names', + 'type_map', + 'post_init_method', kwargs) not_inherited_fields = dict() for k, field_spec in attr_names.items(): @@ -82,6 +87,8 @@ def generate_class(self, **kwargs): + str(e) + " Please define that type before defining '%s'." % name) cls = ExtenderMeta(data_type, tuple(bases), classdict) + cls.post_init_method = post_init_method + return cls @@ -316,8 +323,19 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): elif attr_name not in attrs_not_to_set: attrs_to_set.append(attr_name) - @docval(*docval_args, allow_positional=AllowPositional.WARNING) + # We want to use the skip_post_init of the current class and not the parent class + for item in docval_args: + if item['name'] == 'skip_post_init': + docval_args.remove(item) + + @docval(*docval_args, + {'name': 'skip_post_init', 'type': bool, 'default': False, + 'doc': 'bool to skip post_init'}, + allow_positional=AllowPositional.WARNING) def __init__(self, **kwargs): + skip_post_init = popargs('skip_post_init', kwargs) + + original_kwargs = dict(kwargs) if name is not None: # force container name to be the fixed name in the spec kwargs.update(name=name) @@ -343,6 +361,9 @@ def __init__(self, **kwargs): for f in fixed_value_attrs_to_set: self.fields[f] = getattr(not_inherited_fields[f], 'value') + if self.post_init_method is not None and not skip_post_init: + self.post_init_method(**original_kwargs) + classdict['__init__'] = __init__ @@ -417,6 +438,7 @@ def set_init(cls, classdict, bases, docval_args, not_inherited_fields, name): def __init__(self, **kwargs): # store the values passed to init for each MCI attribute so that they can be added # after calling __init__ + original_kwargs = dict(kwargs) new_kwargs = list() for field_clsconf in classdict['__clsconf__']: attr_name = field_clsconf['attr'] @@ -437,6 +459,7 @@ def __init__(self, **kwargs): kwargs[attr_name] = list() # call the parent class init without the MCI attribute + kwargs['skip_post_init'] = True previous_init(self, **kwargs) # call the add method for each MCI attribute @@ -444,5 +467,8 @@ def __init__(self, **kwargs): add_method = getattr(self, new_kwarg['add_method_name']) add_method(new_kwarg['value']) + if self.post_init_method is not None: + self.post_init_method(**original_kwargs) + # override __init__ classdict['__init__'] = __init__ diff --git a/src/hdmf/build/manager.py b/src/hdmf/build/manager.py index a26de3279..25b9b81bd 100644 --- a/src/hdmf/build/manager.py +++ b/src/hdmf/build/manager.py @@ -1,6 +1,7 @@ import logging from collections import OrderedDict, deque from copy import copy +from collections.abc import Callable from .builders import DatasetBuilder, GroupBuilder, LinkBuilder, Builder, BaseBuilder from .classgenerator import ClassGenerator, CustomClassGenerator, MCIClassGenerator @@ -498,11 +499,14 @@ def get_container_cls(self, **kwargs): created and returned. """ # NOTE: this internally used function get_container_cls will be removed in favor of get_dt_container_cls + # Deprecated: Will be removed by HDMF 4.0 namespace, data_type, autogen = getargs('namespace', 'data_type', 'autogen', kwargs) return self.get_dt_container_cls(data_type, namespace, autogen) @docval({"name": "data_type", "type": str, "doc": "the data type to create a AbstractContainer class for"}, {"name": "namespace", "type": str, "doc": "the namespace containing the data_type", "default": None}, + {'name': 'post_init_method', 'type': Callable, 'default': None, + 'doc': 'The function used as a post_init method to validate the class generation.'}, {"name": "autogen", "type": bool, "doc": "autogenerate class if one does not exist", "default": True}, returns='the class for the given namespace and data_type', rtype=type) def get_dt_container_cls(self, **kwargs): @@ -513,7 +517,8 @@ def get_dt_container_cls(self, **kwargs): Replaces get_container_cls but namespace is optional. If namespace is unknown, it will be looked up from all namespaces. """ - namespace, data_type, autogen = getargs('namespace', 'data_type', 'autogen', kwargs) + namespace, data_type, post_init_method, autogen = getargs('namespace', 'data_type', + 'post_init_method','autogen', kwargs) # namespace is unknown, so look it up if namespace is None: @@ -527,12 +532,18 @@ def get_dt_container_cls(self, **kwargs): raise ValueError("Namespace could not be resolved.") cls = self.__get_container_cls(namespace, data_type) + if cls is None and autogen: # dynamically generate a class spec = self.__ns_catalog.get_spec(namespace, data_type) self.__check_dependent_types(spec, namespace) parent_cls = self.__get_parent_cls(namespace, data_type, spec) attr_names = self.__default_mapper_cls.get_attr_names(spec) - cls = self.__class_generator.generate_class(data_type, spec, parent_cls, attr_names, self) + cls = self.__class_generator.generate_class(data_type=data_type, + spec=spec, + parent_cls=parent_cls, + attr_names=attr_names, + post_init_method=post_init_method, + type_map=self) self.register_container_type(namespace, data_type, cls) return cls diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 248ca1095..4d724d1d1 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -3,6 +3,7 @@ ''' import os.path from copy import deepcopy +from collections.abc import Callable CORE_NAMESPACE = 'hdmf-common' EXP_NAMESPACE = 'hdmf-experimental' @@ -136,12 +137,14 @@ def available_namespaces(): @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the Container class for'}, {'name': 'namespace', 'type': str, 'doc': 'the namespace the data_type is defined in'}, + {'name': 'post_init_method', 'type': Callable, 'default': None, + 'doc': 'The function used as a post_init method to validate the class generation.'}, is_method=False) def get_class(**kwargs): """Get the class object of the Container subclass corresponding to a given neurdata_type. """ - data_type, namespace = getargs('data_type', 'namespace', kwargs) - return __TYPE_MAP.get_dt_container_cls(data_type, namespace) + data_type, namespace, post_init_method = getargs('data_type', 'namespace', 'post_init_method', kwargs) + return __TYPE_MAP.get_dt_container_cls(data_type, namespace, post_init_method) @docval({'name': 'extensions', 'type': (str, TypeMap, list), diff --git a/tests/unit/build_tests/test_classgenerator.py b/tests/unit/build_tests/test_classgenerator.py index 0c117820b..52fdc4839 100644 --- a/tests/unit/build_tests/test_classgenerator.py +++ b/tests/unit/build_tests/test_classgenerator.py @@ -2,6 +2,7 @@ import os import shutil import tempfile +from warnings import warn from hdmf.build import TypeMap, CustomClassGenerator from hdmf.build.classgenerator import ClassGenerator, MCIClassGenerator @@ -82,6 +83,79 @@ def test_no_generators(self): self.assertTrue(hasattr(cls, '__init__')) +class TestPostInitGetClass(TestCase): + def setUp(self): + def post_init_method(self, **kwargs): + attr1 = kwargs['attr1'] + if attr1<10: + msg = "attr1 should be >=10" + warn(msg) + self.post_init=post_init_method + + def test_post_init(self): + spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Baz', + attributes=[ + AttributeSpec(name='attr1', doc='a int attribute', dtype='int') + ] + ) + + spec_catalog = SpecCatalog() + spec_catalog.register_spec(spec, 'test.yaml') + namespace = SpecNamespace( + doc='a test namespace', + name=CORE_NAMESPACE, + schema=[{'source': 'test.yaml'}], + version='0.1.0', + catalog=spec_catalog + ) + namespace_catalog = NamespaceCatalog() + namespace_catalog.add_namespace(CORE_NAMESPACE, namespace) + type_map = TypeMap(namespace_catalog) + + cls = type_map.get_dt_container_cls('Baz', CORE_NAMESPACE, self.post_init) + + with self.assertWarns(Warning): + cls(name='instance', attr1=9) + + def test_multi_container_post_init(self): + bar_spec = GroupSpec( + doc='A test group specification with a data type', + data_type_def='Bar', + datasets=[ + DatasetSpec( + doc='a dataset', + dtype='int', + name='data', + attributes=[AttributeSpec(name='attr2', doc='an integer attribute', dtype='int')] + ) + ], + attributes=[AttributeSpec(name='attr1', doc='a string attribute', dtype='text')]) + + multi_spec = GroupSpec(doc='A test extension that contains a multi', + data_type_def='Multi', + groups=[GroupSpec(data_type_inc=bar_spec, doc='test multi', quantity='*')], + attributes=[AttributeSpec(name='attr1', doc='a float attribute', dtype='float')]) + + spec_catalog = SpecCatalog() + spec_catalog.register_spec(bar_spec, 'test.yaml') + spec_catalog.register_spec(multi_spec, 'test.yaml') + namespace = SpecNamespace( + doc='a test namespace', + name=CORE_NAMESPACE, + schema=[{'source': 'test.yaml'}], + version='0.1.0', + catalog=spec_catalog + ) + namespace_catalog = NamespaceCatalog() + namespace_catalog.add_namespace(CORE_NAMESPACE, namespace) + type_map = TypeMap(namespace_catalog) + Multi = type_map.get_dt_container_cls('Multi', CORE_NAMESPACE, self.post_init) + + with self.assertWarns(Warning): + Multi(name='instance', attr1=9.1) + class TestDynamicContainer(TestCase): def setUp(self): @@ -109,13 +183,15 @@ def test_dynamic_container_creation(self): AttributeSpec('attr4', 'another float attribute', 'float')]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) - expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4'} + expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'skip_post_init'} received_args = set() + for x in get_docval(cls.__init__): if x['name'] != 'foo': received_args.add(x['name']) with self.subTest(name=x['name']): - self.assertNotIn('default', x) + if x['name'] != 'skip_post_init': + self.assertNotIn('default', x) self.assertSetEqual(expected_args, received_args) self.assertEqual(cls.__name__, 'Baz') self.assertTrue(issubclass(cls, Bar)) @@ -135,7 +211,7 @@ def test_dynamic_container_creation_defaults(self): AttributeSpec('attr4', 'another float attribute', 'float')]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) - expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'foo'} + expected_args = {'name', 'data', 'attr1', 'attr2', 'attr3', 'attr4', 'foo', 'skip_post_init'} received_args = set(map(lambda x: x['name'], get_docval(cls.__init__))) self.assertSetEqual(expected_args, received_args) self.assertEqual(cls.__name__, 'Baz') @@ -285,13 +361,14 @@ def __init__(self, **kwargs): AttributeSpec('attr4', 'another float attribute', 'float')]) self.spec_catalog.register_spec(baz_spec, 'extension.yaml') cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) - expected_args = {'name', 'data', 'attr2', 'attr3', 'attr4'} + expected_args = {'name', 'data', 'attr2', 'attr3', 'attr4', 'skip_post_init'} received_args = set() for x in get_docval(cls.__init__): if x['name'] != 'foo': received_args.add(x['name']) with self.subTest(name=x['name']): - self.assertNotIn('default', x) + if x['name'] != 'skip_post_init': + self.assertNotIn('default', x) self.assertSetEqual(expected_args, received_args) self.assertTrue(issubclass(cls, FixedAttrBar)) inst = cls(name="My Baz", data=[1, 2, 3, 4], attr2=1000, attr3=98.6, attr4=1.0) @@ -445,7 +522,7 @@ def setUp(self): def test_init_docval(self): cls = self.type_map.get_dt_container_cls('Baz', CORE_NAMESPACE) # generate the class - expected_args = {'name'} # 'attr1' should not be included + expected_args = {'name', 'skip_post_init'} # 'attr1' should not be included received_args = set() for x in get_docval(cls.__init__): received_args.add(x['name']) @@ -518,6 +595,8 @@ def test_gen_parent_class(self): {'name': 'my_baz1', 'doc': 'A composition inside with a fixed name', 'type': baz1_cls}, {'name': 'my_baz2', 'doc': 'A composition inside with a fixed name', 'type': baz2_cls}, {'name': 'my_baz1_link', 'doc': 'A composition inside without a fixed name', 'type': baz1_cls}, + {'name': 'skip_post_init', 'type': bool, 'default': False, + 'doc': 'bool to skip post_init'} )) def test_init_fields(self): From d390c14b370ce231f58eea22101a7d5208b1a0d6 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Wed, 15 May 2024 12:37:06 -0700 Subject: [PATCH 05/33] Update pyproject.toml (#1115) --- pyproject.toml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pyproject.toml b/pyproject.toml index e5584b581..67b13350b 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ dependencies = [ "h5py>=2.10", "jsonschema>=2.6.0", - "numpy>=1.18", + 'numpy>=1.18, <2.0', # pin below 2.0 until HDMF supports numpy 2.0 "pandas>=1.0.5", "ruamel.yaml>=0.16", "scipy>=1.4", From a497da35ca8d6c747f10b0d2b51d690ca345e035 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Fri, 17 May 2024 11:56:55 -0700 Subject: [PATCH 06/33] Fix warning (#1116) * Fix warning * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * breakpoint * Delete docs/gallery/expanded_example_dynamic_term_set.yaml * Delete docs/gallery/schemasheets/nwb_static_enums.yaml * Update CHANGELOG.md * Update .gitignore --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .gitignore | 2 + CHANGELOG.md | 3 + .../expanded_example_dynamic_term_set.yaml | 2073 ----------------- .../schemasheets/nwb_static_enums.yaml | 52 - src/hdmf/container.py | 12 +- 5 files changed, 11 insertions(+), 2131 deletions(-) delete mode 100644 docs/gallery/expanded_example_dynamic_term_set.yaml delete mode 100644 docs/gallery/schemasheets/nwb_static_enums.yaml diff --git a/.gitignore b/.gitignore index 8257bc927..d75abc985 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,8 @@ /docs/source/hdmf*.rst /docs/gallery/*.hdf5 /docs/gallery/*.sqlite +/docs/gallery/expanded_example_dynamic_term_set.yaml +/docs/gallery/schemasheets/nwb_static_enums.yaml # Auto-generated files after running tutorials mylab.*.yaml diff --git a/CHANGELOG.md b/CHANGELOG.md index 909ef5253..47cb8c8d6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,9 @@ - Updated testing to not install in editable mode and not run `coverage` by default. @rly [#1107](https://github.com/hdmf-dev/hdmf/pull/1107) - Add `post_init_method` parameter when generating classes to perform post-init functionality, i.e., validation. @mavaylon1 [#1089](https://github.com/hdmf-dev/hdmf/pull/1089) +### Bug Fixes +- Fixed `TermSetWrapper` warning raised during the setters. @mavaylon1 [#1116](https://github.com/hdmf-dev/hdmf/pull/1116) + ## HDMF 3.13.0 (March 20, 2024) ### Enhancements diff --git a/docs/gallery/expanded_example_dynamic_term_set.yaml b/docs/gallery/expanded_example_dynamic_term_set.yaml deleted file mode 100644 index a2631696a..000000000 --- a/docs/gallery/expanded_example_dynamic_term_set.yaml +++ /dev/null @@ -1,2073 +0,0 @@ -id: https://w3id.org/linkml/examples/nwb_dynamic_enums -title: dynamic enums example -name: nwb_dynamic_enums -description: this schema demonstrates the use of dynamic enums - -prefixes: - linkml: https://w3id.org/linkml/ - CL: http://purl.obolibrary.org/obo/CL_ - -imports: -- linkml:types - -default_range: string - -# ======================== # -# CLASSES # -# ======================== # -classes: - BrainSample: - slots: - - cell_type - -# ======================== # -# SLOTS # -# ======================== # -slots: - cell_type: - required: true - range: NeuronTypeEnum - -# ======================== # -# ENUMS # -# ======================== # -enums: - NeuronTypeEnum: - reachable_from: - source_ontology: obo:cl - source_nodes: - - CL:0000540 ## neuron - include_self: false - relationship_types: - - rdfs:subClassOf - permissible_values: - CL:0000705: - text: CL:0000705 - description: R6 photoreceptor cell - meaning: CL:0000705 - CL:4023108: - text: CL:4023108 - description: oxytocin-secreting magnocellular cell - meaning: CL:4023108 - CL:0004240: - text: CL:0004240 - description: WF1 amacrine cell - meaning: CL:0004240 - CL:0004242: - text: CL:0004242 - description: WF3-1 amacrine cell - meaning: CL:0004242 - CL:1000380: - text: CL:1000380 - description: type 1 vestibular sensory cell of epithelium of macula of saccule - of membranous labyrinth - meaning: CL:1000380 - CL:4023128: - text: CL:4023128 - description: rostral periventricular region of the third ventricle KNDy neuron - meaning: CL:4023128 - CL:0003020: - text: CL:0003020 - description: retinal ganglion cell C outer - meaning: CL:0003020 - CL:4023094: - text: CL:4023094 - description: tufted pyramidal neuron - meaning: CL:4023094 - CL:4023057: - text: CL:4023057 - description: cerebellar inhibitory GABAergic interneuron - meaning: CL:4023057 - CL:2000049: - text: CL:2000049 - description: primary motor cortex pyramidal cell - meaning: CL:2000049 - CL:0000119: - text: CL:0000119 - description: cerebellar Golgi cell - meaning: CL:0000119 - CL:0004227: - text: CL:0004227 - description: flat bistratified amacrine cell - meaning: CL:0004227 - CL:1000606: - text: CL:1000606 - description: kidney nerve cell - meaning: CL:1000606 - CL:1001582: - text: CL:1001582 - description: lateral ventricle neuron - meaning: CL:1001582 - CL:0000165: - text: CL:0000165 - description: neuroendocrine cell - meaning: CL:0000165 - CL:0000555: - text: CL:0000555 - description: neuronal brush cell - meaning: CL:0000555 - CL:0004231: - text: CL:0004231 - description: recurving diffuse amacrine cell - meaning: CL:0004231 - CL:0000687: - text: CL:0000687 - description: R1 photoreceptor cell - meaning: CL:0000687 - CL:0001031: - text: CL:0001031 - description: cerebellar granule cell - meaning: CL:0001031 - CL:0003026: - text: CL:0003026 - description: retinal ganglion cell D1 - meaning: CL:0003026 - CL:4033035: - text: CL:4033035 - description: giant bipolar cell - meaning: CL:4033035 - CL:4023009: - text: CL:4023009 - description: extratelencephalic-projecting glutamatergic cortical neuron - meaning: CL:4023009 - CL:0010022: - text: CL:0010022 - description: cardiac neuron - meaning: CL:0010022 - CL:0000287: - text: CL:0000287 - description: eye photoreceptor cell - meaning: CL:0000287 - CL:0000488: - text: CL:0000488 - description: visible light photoreceptor cell - meaning: CL:0000488 - CL:0003046: - text: CL:0003046 - description: M13 retinal ganglion cell - meaning: CL:0003046 - CL:4023169: - text: CL:4023169 - description: trigeminal neuron - meaning: CL:4023169 - CL:0005007: - text: CL:0005007 - description: Kolmer-Agduhr neuron - meaning: CL:0005007 - CL:0005008: - text: CL:0005008 - description: macular hair cell - meaning: CL:0005008 - CL:4023027: - text: CL:4023027 - description: L5 T-Martinotti sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023027 - CL:4033032: - text: CL:4033032 - description: diffuse bipolar 6 cell - meaning: CL:4033032 - CL:0008021: - text: CL:0008021 - description: anterior lateral line ganglion neuron - meaning: CL:0008021 - CL:4023028: - text: CL:4023028 - description: L5 non-Martinotti sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023028 - CL:4023063: - text: CL:4023063 - description: medial ganglionic eminence derived interneuron - meaning: CL:4023063 - CL:4023032: - text: CL:4023032 - description: ON retinal ganglion cell - meaning: CL:4023032 - CL:0003039: - text: CL:0003039 - description: M8 retinal ganglion cell - meaning: CL:0003039 - CL:0000757: - text: CL:0000757 - description: type 5 cone bipolar cell (sensu Mus) - meaning: CL:0000757 - CL:0000609: - text: CL:0000609 - description: vestibular hair cell - meaning: CL:0000609 - CL:0004219: - text: CL:0004219 - description: A2 amacrine cell - meaning: CL:0004219 - CL:4030028: - text: CL:4030028 - description: glycinergic amacrine cell - meaning: CL:4030028 - CL:0002450: - text: CL:0002450 - description: tether cell - meaning: CL:0002450 - CL:0002374: - text: CL:0002374 - description: ear hair cell - meaning: CL:0002374 - CL:0004124: - text: CL:0004124 - description: retinal ganglion cell C1 - meaning: CL:0004124 - CL:0004115: - text: CL:0004115 - description: retinal ganglion cell B - meaning: CL:0004115 - CL:1000384: - text: CL:1000384 - description: type 2 vestibular sensory cell of epithelium of macula of saccule - of membranous labyrinth - meaning: CL:1000384 - CL:2000037: - text: CL:2000037 - description: posterior lateral line neuromast hair cell - meaning: CL:2000037 - CL:0000673: - text: CL:0000673 - description: Kenyon cell - meaning: CL:0000673 - CL:4023052: - text: CL:4023052 - description: Betz upper motor neuron - meaning: CL:4023052 - CL:0004243: - text: CL:0004243 - description: WF3-2 amacrine cell - meaning: CL:0004243 - CL:1000222: - text: CL:1000222 - description: stomach neuroendocrine cell - meaning: CL:1000222 - CL:0002310: - text: CL:0002310 - description: mammosomatotroph - meaning: CL:0002310 - CL:4023066: - text: CL:4023066 - description: horizontal pyramidal neuron - meaning: CL:4023066 - CL:0000379: - text: CL:0000379 - description: sensory processing neuron - meaning: CL:0000379 - CL:0011006: - text: CL:0011006 - description: Lugaro cell - meaning: CL:0011006 - CL:0004216: - text: CL:0004216 - description: type 5b cone bipolar cell - meaning: CL:0004216 - CL:0004126: - text: CL:0004126 - description: retinal ganglion cell C2 outer - meaning: CL:0004126 - CL:0000108: - text: CL:0000108 - description: cholinergic neuron - meaning: CL:0000108 - CL:0011103: - text: CL:0011103 - description: sympathetic neuron - meaning: CL:0011103 - CL:4023107: - text: CL:4023107 - description: reticulospinal neuron - meaning: CL:4023107 - CL:4023002: - text: CL:4023002 - description: dynamic beta motor neuron - meaning: CL:4023002 - CL:4030048: - text: CL:4030048 - description: striosomal D1 medium spiny neuron - meaning: CL:4030048 - CL:4023163: - text: CL:4023163 - description: spherical bushy cell - meaning: CL:4023163 - CL:4023061: - text: CL:4023061 - description: hippocampal CA4 neuron - meaning: CL:4023061 - CL:0000532: - text: CL:0000532 - description: CAP motoneuron - meaning: CL:0000532 - CL:0000526: - text: CL:0000526 - description: afferent neuron - meaning: CL:0000526 - CL:0003003: - text: CL:0003003 - description: G2 retinal ganglion cell - meaning: CL:0003003 - CL:0000530: - text: CL:0000530 - description: primary neuron (sensu Teleostei) - meaning: CL:0000530 - CL:4023045: - text: CL:4023045 - description: medulla-projecting glutamatergic neuron of the primary motor - cortex - meaning: CL:4023045 - CL:3000004: - text: CL:3000004 - description: peripheral sensory neuron - meaning: CL:3000004 - CL:0000544: - text: CL:0000544 - description: slowly adapting mechanoreceptor cell - meaning: CL:0000544 - CL:4030047: - text: CL:4030047 - description: matrix D2 medium spiny neuron - meaning: CL:4030047 - CL:0004220: - text: CL:0004220 - description: flag amacrine cell - meaning: CL:0004220 - CL:4023125: - text: CL:4023125 - description: KNDy neuron - meaning: CL:4023125 - CL:0004228: - text: CL:0004228 - description: broad diffuse amacrine cell - meaning: CL:0004228 - CL:4023122: - text: CL:4023122 - description: oxytocin receptor sst GABAergic cortical interneuron - meaning: CL:4023122 - CL:1000379: - text: CL:1000379 - description: type 1 vestibular sensory cell of epithelium of macula of utricle - of membranous labyrinth - meaning: CL:1000379 - CL:0011111: - text: CL:0011111 - description: gonadotropin-releasing hormone neuron - meaning: CL:0011111 - CL:0003042: - text: CL:0003042 - description: M9-OFF retinal ganglion cell - meaning: CL:0003042 - CL:0003030: - text: CL:0003030 - description: M3 retinal ganglion cell - meaning: CL:0003030 - CL:0003011: - text: CL:0003011 - description: G8 retinal ganglion cell - meaning: CL:0003011 - CL:0000202: - text: CL:0000202 - description: auditory hair cell - meaning: CL:0000202 - CL:0002271: - text: CL:0002271 - description: type EC1 enteroendocrine cell - meaning: CL:0002271 - CL:4023013: - text: CL:4023013 - description: corticothalamic-projecting glutamatergic cortical neuron - meaning: CL:4023013 - CL:4023114: - text: CL:4023114 - description: calyx vestibular afferent neuron - meaning: CL:4023114 - CL:0003045: - text: CL:0003045 - description: M12 retinal ganglion cell - meaning: CL:0003045 - CL:0002487: - text: CL:0002487 - description: cutaneous/subcutaneous mechanoreceptor cell - meaning: CL:0002487 - CL:4030053: - text: CL:4030053 - description: Island of Calleja granule cell - meaning: CL:4030053 - CL:0000490: - text: CL:0000490 - description: photopic photoreceptor cell - meaning: CL:0000490 - CL:2000023: - text: CL:2000023 - description: spinal cord ventral column interneuron - meaning: CL:2000023 - CL:1000381: - text: CL:1000381 - description: type 1 vestibular sensory cell of epithelium of crista of ampulla - of semicircular duct of membranous labyrinth - meaning: CL:1000381 - CL:0003013: - text: CL:0003013 - description: G10 retinal ganglion cell - meaning: CL:0003013 - CL:0000602: - text: CL:0000602 - description: pressoreceptor cell - meaning: CL:0000602 - CL:4023039: - text: CL:4023039 - description: amygdala excitatory neuron - meaning: CL:4023039 - CL:4030043: - text: CL:4030043 - description: matrix D1 medium spiny neuron - meaning: CL:4030043 - CL:0000105: - text: CL:0000105 - description: pseudounipolar neuron - meaning: CL:0000105 - CL:0004137: - text: CL:0004137 - description: retinal ganglion cell A2 inner - meaning: CL:0004137 - CL:1001436: - text: CL:1001436 - description: hair-tylotrich neuron - meaning: CL:1001436 - CL:1001503: - text: CL:1001503 - description: olfactory bulb tufted cell - meaning: CL:1001503 - CL:0000406: - text: CL:0000406 - description: CNS short range interneuron - meaning: CL:0000406 - CL:2000087: - text: CL:2000087 - description: dentate gyrus of hippocampal formation basket cell - meaning: CL:2000087 - CL:0000534: - text: CL:0000534 - description: primary interneuron (sensu Teleostei) - meaning: CL:0000534 - CL:0000246: - text: CL:0000246 - description: Mauthner neuron - meaning: CL:0000246 - CL:0003027: - text: CL:0003027 - description: retinal ganglion cell D2 - meaning: CL:0003027 - CL:0000752: - text: CL:0000752 - description: cone retinal bipolar cell - meaning: CL:0000752 - CL:0000410: - text: CL:0000410 - description: CNS long range interneuron - meaning: CL:0000410 - CL:0009000: - text: CL:0009000 - description: sensory neuron of spinal nerve - meaning: CL:0009000 - CL:0000754: - text: CL:0000754 - description: type 2 cone bipolar cell (sensu Mus) - meaning: CL:0000754 - CL:0002309: - text: CL:0002309 - description: corticotroph - meaning: CL:0002309 - CL:0010009: - text: CL:0010009 - description: camera-type eye photoreceptor cell - meaning: CL:0010009 - CL:4023069: - text: CL:4023069 - description: medial ganglionic eminence derived GABAergic cortical interneuron - meaning: CL:4023069 - CL:0000102: - text: CL:0000102 - description: polymodal neuron - meaning: CL:0000102 - CL:0000694: - text: CL:0000694 - description: R3 photoreceptor cell - meaning: CL:0000694 - CL:0004183: - text: CL:0004183 - description: retinal ganglion cell B3 - meaning: CL:0004183 - CL:0000693: - text: CL:0000693 - description: neurogliaform cell - meaning: CL:0000693 - CL:0000760: - text: CL:0000760 - description: type 8 cone bipolar cell (sensu Mus) - meaning: CL:0000760 - CL:4023001: - text: CL:4023001 - description: static beta motor neuron - meaning: CL:4023001 - CL:1000424: - text: CL:1000424 - description: chromaffin cell of paraaortic body - meaning: CL:1000424 - CL:0000120: - text: CL:0000120 - description: granule cell - meaning: CL:0000120 - CL:0002312: - text: CL:0002312 - description: somatotroph - meaning: CL:0002312 - CL:0000107: - text: CL:0000107 - description: autonomic neuron - meaning: CL:0000107 - CL:2000047: - text: CL:2000047 - description: brainstem motor neuron - meaning: CL:2000047 - CL:4023080: - text: CL:4023080 - description: stellate L6 intratelencephalic projecting glutamatergic neuron - of the primary motor cortex (Mmus) - meaning: CL:4023080 - CL:0000848: - text: CL:0000848 - description: microvillous olfactory receptor neuron - meaning: CL:0000848 - CL:0004213: - text: CL:0004213 - description: type 3a cone bipolar cell - meaning: CL:0004213 - CL:0000116: - text: CL:0000116 - description: pioneer neuron - meaning: CL:0000116 - CL:4023187: - text: CL:4023187 - description: koniocellular cell - meaning: CL:4023187 - CL:4023116: - text: CL:4023116 - description: type 2 spiral ganglion neuron - meaning: CL:4023116 - CL:0008015: - text: CL:0008015 - description: inhibitory motor neuron - meaning: CL:0008015 - CL:0003048: - text: CL:0003048 - description: L cone cell - meaning: CL:0003048 - CL:1000082: - text: CL:1000082 - description: stretch receptor cell - meaning: CL:1000082 - CL:0003031: - text: CL:0003031 - description: M3-ON retinal ganglion cell - meaning: CL:0003031 - CL:1001474: - text: CL:1001474 - description: medium spiny neuron - meaning: CL:1001474 - CL:0000745: - text: CL:0000745 - description: retina horizontal cell - meaning: CL:0000745 - CL:0002515: - text: CL:0002515 - description: interrenal norepinephrine type cell - meaning: CL:0002515 - CL:2000027: - text: CL:2000027 - description: cerebellum basket cell - meaning: CL:2000027 - CL:0004225: - text: CL:0004225 - description: spider amacrine cell - meaning: CL:0004225 - CL:4023031: - text: CL:4023031 - description: L4 sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023031 - CL:0008038: - text: CL:0008038 - description: alpha motor neuron - meaning: CL:0008038 - CL:4033030: - text: CL:4033030 - description: diffuse bipolar 3b cell - meaning: CL:4033030 - CL:0000336: - text: CL:0000336 - description: adrenal medulla chromaffin cell - meaning: CL:0000336 - CL:0000751: - text: CL:0000751 - description: rod bipolar cell - meaning: CL:0000751 - CL:0008037: - text: CL:0008037 - description: gamma motor neuron - meaning: CL:0008037 - CL:0003028: - text: CL:0003028 - description: M1 retinal ganglion cell - meaning: CL:0003028 - CL:0003016: - text: CL:0003016 - description: G11-OFF retinal ganglion cell - meaning: CL:0003016 - CL:0004239: - text: CL:0004239 - description: wavy bistratified amacrine cell - meaning: CL:0004239 - CL:4023168: - text: CL:4023168 - description: somatosensory neuron - meaning: CL:4023168 - CL:4023018: - text: CL:4023018 - description: pvalb GABAergic cortical interneuron - meaning: CL:4023018 - CL:0004138: - text: CL:0004138 - description: retinal ganglion cell A2 - meaning: CL:0004138 - CL:0000750: - text: CL:0000750 - description: OFF-bipolar cell - meaning: CL:0000750 - CL:0000709: - text: CL:0000709 - description: R8 photoreceptor cell - meaning: CL:0000709 - CL:0004214: - text: CL:0004214 - description: type 3b cone bipolar cell - meaning: CL:0004214 - CL:0003047: - text: CL:0003047 - description: M14 retinal ganglion cell - meaning: CL:0003047 - CL:0015000: - text: CL:0015000 - description: cranial motor neuron - meaning: CL:0015000 - CL:0003036: - text: CL:0003036 - description: M7 retinal ganglion cell - meaning: CL:0003036 - CL:0000397: - text: CL:0000397 - description: ganglion interneuron - meaning: CL:0000397 - CL:1001509: - text: CL:1001509 - description: glycinergic neuron - meaning: CL:1001509 - CL:4023038: - text: CL:4023038 - description: L6b glutamatergic cortical neuron - meaning: CL:4023038 - CL:0000112: - text: CL:0000112 - description: columnar neuron - meaning: CL:0000112 - CL:0002517: - text: CL:0002517 - description: interrenal epinephrin secreting cell - meaning: CL:0002517 - CL:1000383: - text: CL:1000383 - description: type 2 vestibular sensory cell of epithelium of macula of utricle - of membranous labyrinth - meaning: CL:1000383 - CL:0004116: - text: CL:0004116 - description: retinal ganglion cell C - meaning: CL:0004116 - CL:4023113: - text: CL:4023113 - description: bouton vestibular afferent neuron - meaning: CL:4023113 - CL:0003034: - text: CL:0003034 - description: M5 retinal ganglion cell - meaning: CL:0003034 - CL:0011005: - text: CL:0011005 - description: GABAergic interneuron - meaning: CL:0011005 - CL:0011105: - text: CL:0011105 - description: dopamanergic interplexiform cell - meaning: CL:0011105 - CL:0000749: - text: CL:0000749 - description: ON-bipolar cell - meaning: CL:0000749 - CL:0000498: - text: CL:0000498 - description: inhibitory interneuron - meaning: CL:0000498 - CL:4023071: - text: CL:4023071 - description: L5/6 cck cortical GABAergic interneuron (Mmus) - meaning: CL:4023071 - CL:1000245: - text: CL:1000245 - description: posterior lateral line ganglion neuron - meaning: CL:1000245 - CL:0004139: - text: CL:0004139 - description: retinal ganglion cell A2 outer - meaning: CL:0004139 - CL:0000531: - text: CL:0000531 - description: primary sensory neuron (sensu Teleostei) - meaning: CL:0000531 - CL:0004125: - text: CL:0004125 - description: retinal ganglion cell C2 inner - meaning: CL:0004125 - CL:4023064: - text: CL:4023064 - description: caudal ganglionic eminence derived interneuron - meaning: CL:4023064 - CL:4030049: - text: CL:4030049 - description: striosomal D2 medium spiny neuron - meaning: CL:4030049 - CL:0017002: - text: CL:0017002 - description: prostate neuroendocrine cell - meaning: CL:0017002 - CL:0000756: - text: CL:0000756 - description: type 4 cone bipolar cell (sensu Mus) - meaning: CL:0000756 - CL:0000707: - text: CL:0000707 - description: R7 photoreceptor cell - meaning: CL:0000707 - CL:0000700: - text: CL:0000700 - description: dopaminergic neuron - meaning: CL:0000700 - CL:0003002: - text: CL:0003002 - description: G1 retinal ganglion cell - meaning: CL:0003002 - CL:1000001: - text: CL:1000001 - description: retrotrapezoid nucleus neuron - meaning: CL:1000001 - CL:4023007: - text: CL:4023007 - description: L2/3 bipolar vip GABAergic cortical interneuron (Mmus) - meaning: CL:4023007 - CL:0000528: - text: CL:0000528 - description: nitrergic neuron - meaning: CL:0000528 - CL:0000639: - text: CL:0000639 - description: basophil cell of pars distalis of adenohypophysis - meaning: CL:0000639 - CL:0000849: - text: CL:0000849 - description: crypt olfactory receptor neuron - meaning: CL:0000849 - CL:0011110: - text: CL:0011110 - description: histaminergic neuron - meaning: CL:0011110 - CL:0005025: - text: CL:0005025 - description: visceromotor neuron - meaning: CL:0005025 - CL:0003001: - text: CL:0003001 - description: bistratified retinal ganglion cell - meaning: CL:0003001 - CL:0004241: - text: CL:0004241 - description: WF2 amacrine cell - meaning: CL:0004241 - CL:4023019: - text: CL:4023019 - description: L5/6 cck, vip cortical GABAergic interneuron (Mmus) - meaning: CL:4023019 - CL:4023040: - text: CL:4023040 - description: L2/3-6 intratelencephalic projecting glutamatergic cortical neuron - meaning: CL:4023040 - CL:1001435: - text: CL:1001435 - description: periglomerular cell - meaning: CL:1001435 - CL:4023127: - text: CL:4023127 - description: arcuate nucleus of hypothalamus KNDy neuron - meaning: CL:4023127 - CL:0003007: - text: CL:0003007 - description: G4-OFF retinal ganglion cell - meaning: CL:0003007 - CL:0000101: - text: CL:0000101 - description: sensory neuron - meaning: CL:0000101 - CL:2000097: - text: CL:2000097 - description: midbrain dopaminergic neuron - meaning: CL:2000097 - CL:4023095: - text: CL:4023095 - description: untufted pyramidal neuron - meaning: CL:4023095 - CL:0003004: - text: CL:0003004 - description: G3 retinal ganglion cell - meaning: CL:0003004 - CL:0000527: - text: CL:0000527 - description: efferent neuron - meaning: CL:0000527 - CL:1000382: - text: CL:1000382 - description: type 2 vestibular sensory cell of stato-acoustic epithelium - meaning: CL:1000382 - CL:4033019: - text: CL:4033019 - description: ON-blue cone bipolar cell - meaning: CL:4033019 - CL:0000589: - text: CL:0000589 - description: cochlear inner hair cell - meaning: CL:0000589 - CL:4023160: - text: CL:4023160 - description: cartwheel cell - meaning: CL:4023160 - CL:1001437: - text: CL:1001437 - description: hair-down neuron - meaning: CL:1001437 - CL:0011102: - text: CL:0011102 - description: parasympathetic neuron - meaning: CL:0011102 - CL:2000029: - text: CL:2000029 - description: central nervous system neuron - meaning: CL:2000029 - CL:4023115: - text: CL:4023115 - description: type 1 spiral ganglion neuron - meaning: CL:4023115 - CL:0002311: - text: CL:0002311 - description: mammotroph - meaning: CL:0002311 - CL:0003025: - text: CL:0003025 - description: retinal ganglion cell C3 - meaning: CL:0003025 - CL:4030050: - text: CL:4030050 - description: D1/D2-hybrid medium spiny neuron - meaning: CL:4030050 - CL:4023118: - text: CL:4023118 - description: L5/6 non-Martinotti sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023118 - CL:4023110: - text: CL:4023110 - description: amygdala pyramidal neuron - meaning: CL:4023110 - CL:0002273: - text: CL:0002273 - description: type ECL enteroendocrine cell - meaning: CL:0002273 - CL:0003050: - text: CL:0003050 - description: S cone cell - meaning: CL:0003050 - CL:4023121: - text: CL:4023121 - description: sst chodl GABAergic cortical interneuron - meaning: CL:4023121 - CL:4023020: - text: CL:4023020 - description: dynamic gamma motor neuron - meaning: CL:4023020 - CL:0004246: - text: CL:0004246 - description: monostratified cell - meaning: CL:0004246 - CL:0000495: - text: CL:0000495 - description: blue sensitive photoreceptor cell - meaning: CL:0000495 - CL:0000029: - text: CL:0000029 - description: neural crest derived neuron - meaning: CL:0000029 - CL:0004001: - text: CL:0004001 - description: local interneuron - meaning: CL:0004001 - CL:0000551: - text: CL:0000551 - description: unimodal nocireceptor - meaning: CL:0000551 - CL:0003006: - text: CL:0003006 - description: G4-ON retinal ganglion cell - meaning: CL:0003006 - CL:4023011: - text: CL:4023011 - description: lamp5 GABAergic cortical interneuron - meaning: CL:4023011 - CL:4023109: - text: CL:4023109 - description: vasopressin-secreting magnocellular cell - meaning: CL:4023109 - CL:0000121: - text: CL:0000121 - description: Purkinje cell - meaning: CL:0000121 - CL:0000678: - text: CL:0000678 - description: commissural neuron - meaning: CL:0000678 - CL:0004252: - text: CL:0004252 - description: medium field retinal amacrine cell - meaning: CL:0004252 - CL:0000103: - text: CL:0000103 - description: bipolar neuron - meaning: CL:0000103 - CL:4033036: - text: CL:4033036 - description: OFFx cell - meaning: CL:4033036 - CL:4023014: - text: CL:4023014 - description: L5 vip cortical GABAergic interneuron (Mmus) - meaning: CL:4023014 - CL:0008031: - text: CL:0008031 - description: cortical interneuron - meaning: CL:0008031 - CL:0008010: - text: CL:0008010 - description: cranial somatomotor neuron - meaning: CL:0008010 - CL:0000637: - text: CL:0000637 - description: chromophil cell of anterior pituitary gland - meaning: CL:0000637 - CL:0003014: - text: CL:0003014 - description: G11 retinal ganglion cell - meaning: CL:0003014 - CL:4033029: - text: CL:4033029 - description: diffuse bipolar 3a cell - meaning: CL:4033029 - CL:0002611: - text: CL:0002611 - description: neuron of the dorsal spinal cord - meaning: CL:0002611 - CL:0010010: - text: CL:0010010 - description: cerebellar stellate cell - meaning: CL:0010010 - CL:1000465: - text: CL:1000465 - description: chromaffin cell of ovary - meaning: CL:1000465 - CL:0000761: - text: CL:0000761 - description: type 9 cone bipolar cell (sensu Mus) - meaning: CL:0000761 - CL:0004226: - text: CL:0004226 - description: monostratified amacrine cell - meaning: CL:0004226 - CL:0004253: - text: CL:0004253 - description: wide field retinal amacrine cell - meaning: CL:0004253 - CL:4023075: - text: CL:4023075 - description: L6 tyrosine hydroxylase sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023075 - CL:4023068: - text: CL:4023068 - description: thalamic excitatory neuron - meaning: CL:4023068 - CL:1000377: - text: CL:1000377 - description: dense-core granulated cell of epithelium of trachea - meaning: CL:1000377 - CL:4023089: - text: CL:4023089 - description: nest basket cell - meaning: CL:4023089 - CL:4023189: - text: CL:4023189 - description: parasol ganglion cell of retina - meaning: CL:4023189 - CL:0000856: - text: CL:0000856 - description: neuromast hair cell - meaning: CL:0000856 - CL:4023025: - text: CL:4023025 - description: long-range projecting sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023025 - CL:0003043: - text: CL:0003043 - description: M10 retinal ganglion cell - meaning: CL:0003043 - CL:4023000: - text: CL:4023000 - description: beta motor neuron - meaning: CL:4023000 - CL:4023048: - text: CL:4023048 - description: L4/5 intratelencephalic projecting glutamatergic neuron of the - primary motor cortex - meaning: CL:4023048 - CL:0000855: - text: CL:0000855 - description: sensory hair cell - meaning: CL:0000855 - CL:4023070: - text: CL:4023070 - description: caudal ganglionic eminence derived GABAergic cortical interneuron - meaning: CL:4023070 - CL:0002070: - text: CL:0002070 - description: type I vestibular sensory cell - meaning: CL:0002070 - CL:2000028: - text: CL:2000028 - description: cerebellum glutamatergic neuron - meaning: CL:2000028 - CL:0000533: - text: CL:0000533 - description: primary motor neuron (sensu Teleostei) - meaning: CL:0000533 - CL:4023083: - text: CL:4023083 - description: chandelier cell - meaning: CL:4023083 - CL:2000034: - text: CL:2000034 - description: anterior lateral line neuromast hair cell - meaning: CL:2000034 - CL:0003015: - text: CL:0003015 - description: G11-ON retinal ganglion cell - meaning: CL:0003015 - CL:0000204: - text: CL:0000204 - description: acceleration receptive cell - meaning: CL:0000204 - CL:4033031: - text: CL:4033031 - description: diffuse bipolar 4 cell - meaning: CL:4033031 - CL:0003024: - text: CL:0003024 - description: retinal ganglion cell C inner - meaning: CL:0003024 - CL:4023074: - text: CL:4023074 - description: mammillary body neuron - meaning: CL:4023074 - CL:2000089: - text: CL:2000089 - description: dentate gyrus granule cell - meaning: CL:2000089 - CL:4033028: - text: CL:4033028 - description: diffuse bipolar 2 cell - meaning: CL:4033028 - CL:0000110: - text: CL:0000110 - description: peptidergic neuron - meaning: CL:0000110 - CL:4033002: - text: CL:4033002 - description: neuroendocrine cell of epithelium of crypt of Lieberkuhn - meaning: CL:4033002 - CL:4033027: - text: CL:4033027 - description: diffuse bipolar 1 cell - meaning: CL:4033027 - CL:3000003: - text: CL:3000003 - description: sympathetic cholinergic neuron - meaning: CL:3000003 - CL:4023158: - text: CL:4023158 - description: octopus cell of the mammalian cochlear nucleus - meaning: CL:4023158 - CL:0000118: - text: CL:0000118 - description: basket cell - meaning: CL:0000118 - CL:0004223: - text: CL:0004223 - description: AB diffuse-1 amacrine cell - meaning: CL:0004223 - CL:4030054: - text: CL:4030054 - description: RXFP1-positive interface island D1-medium spiny neuron - meaning: CL:4030054 - CL:0002610: - text: CL:0002610 - description: raphe nuclei neuron - meaning: CL:0002610 - CL:4023026: - text: CL:4023026 - description: direct pathway medium spiny neuron - meaning: CL:4023026 - CL:4023016: - text: CL:4023016 - description: vip GABAergic cortical interneuron - meaning: CL:4023016 - CL:0004237: - text: CL:0004237 - description: fountain amacrine cell - meaning: CL:0004237 - CL:0003035: - text: CL:0003035 - description: M6 retinal ganglion cell - meaning: CL:0003035 - CL:1001611: - text: CL:1001611 - description: cerebellar neuron - meaning: CL:1001611 - CL:0000591: - text: CL:0000591 - description: warmth sensing thermoreceptor cell - meaning: CL:0000591 - CL:0002613: - text: CL:0002613 - description: striatum neuron - meaning: CL:0002613 - CL:0000496: - text: CL:0000496 - description: green sensitive photoreceptor cell - meaning: CL:0000496 - CL:0007011: - text: CL:0007011 - description: enteric neuron - meaning: CL:0007011 - CL:2000056: - text: CL:2000056 - description: Meynert cell - meaning: CL:2000056 - CL:0003040: - text: CL:0003040 - description: M9 retinal ganglion cell - meaning: CL:0003040 - CL:0004250: - text: CL:0004250 - description: bistratified retinal amacrine cell - meaning: CL:0004250 - CL:0003029: - text: CL:0003029 - description: M2 retinal ganglion cell - meaning: CL:0003029 - CL:4023017: - text: CL:4023017 - description: sst GABAergic cortical interneuron - meaning: CL:4023017 - CL:0008028: - text: CL:0008028 - description: visual system neuron - meaning: CL:0008028 - CL:0008039: - text: CL:0008039 - description: lower motor neuron - meaning: CL:0008039 - CL:2000086: - text: CL:2000086 - description: neocortex basket cell - meaning: CL:2000086 - CL:4023023: - text: CL:4023023 - description: L5,6 neurogliaform lamp5 GABAergic cortical interneuron (Mmus) - meaning: CL:4023023 - CL:0000697: - text: CL:0000697 - description: R4 photoreceptor cell - meaning: CL:0000697 - CL:2000088: - text: CL:2000088 - description: Ammon's horn basket cell - meaning: CL:2000088 - CL:0004232: - text: CL:0004232 - description: starburst amacrine cell - meaning: CL:0004232 - CL:4023041: - text: CL:4023041 - description: L5 extratelencephalic projecting glutamatergic cortical neuron - meaning: CL:4023041 - CL:0004121: - text: CL:0004121 - description: retinal ganglion cell B2 - meaning: CL:0004121 - CL:0000748: - text: CL:0000748 - description: retinal bipolar neuron - meaning: CL:0000748 - CL:4023164: - text: CL:4023164 - description: globular bushy cell - meaning: CL:4023164 - CL:0000536: - text: CL:0000536 - description: secondary motor neuron (sensu Teleostei) - meaning: CL:0000536 - CL:1000466: - text: CL:1000466 - description: chromaffin cell of right ovary - meaning: CL:1000466 - CL:0011001: - text: CL:0011001 - description: spinal cord motor neuron - meaning: CL:0011001 - CL:0000755: - text: CL:0000755 - description: type 3 cone bipolar cell (sensu Mus) - meaning: CL:0000755 - CL:0004238: - text: CL:0004238 - description: asymmetric bistratified amacrine cell - meaning: CL:0004238 - CL:0004161: - text: CL:0004161 - description: 510 nm-cone - meaning: CL:0004161 - CL:0000198: - text: CL:0000198 - description: pain receptor cell - meaning: CL:0000198 - CL:0003038: - text: CL:0003038 - description: M7-OFF retinal ganglion cell - meaning: CL:0003038 - CL:0003033: - text: CL:0003033 - description: M4 retinal ganglion cell - meaning: CL:0003033 - CL:0012001: - text: CL:0012001 - description: neuron of the forebrain - meaning: CL:0012001 - CL:0011104: - text: CL:0011104 - description: interplexiform cell - meaning: CL:0011104 - CL:0003049: - text: CL:0003049 - description: M cone cell - meaning: CL:0003049 - CL:2000032: - text: CL:2000032 - description: peripheral nervous system neuron - meaning: CL:2000032 - CL:0011100: - text: CL:0011100 - description: galanergic neuron - meaning: CL:0011100 - CL:0008025: - text: CL:0008025 - description: noradrenergic neuron - meaning: CL:0008025 - CL:0000122: - text: CL:0000122 - description: stellate neuron - meaning: CL:0000122 - CL:0003005: - text: CL:0003005 - description: G4 retinal ganglion cell - meaning: CL:0003005 - CL:0000699: - text: CL:0000699 - description: paraganglial type 1 cell - meaning: CL:0000699 - CL:4033050: - text: CL:4033050 - description: catecholaminergic neuron - meaning: CL:4033050 - CL:1001502: - text: CL:1001502 - description: mitral cell - meaning: CL:1001502 - CL:0002069: - text: CL:0002069 - description: type II vestibular sensory cell - meaning: CL:0002069 - CL:4023065: - text: CL:4023065 - description: meis2 expressing cortical GABAergic cell - meaning: CL:4023065 - CL:4023077: - text: CL:4023077 - description: bitufted neuron - meaning: CL:4023077 - CL:0000847: - text: CL:0000847 - description: ciliated olfactory receptor neuron - meaning: CL:0000847 - CL:4023188: - text: CL:4023188 - description: midget ganglion cell of retina - meaning: CL:4023188 - CL:2000090: - text: CL:2000090 - description: dentate gyrus of hippocampal formation stellate cell - meaning: CL:2000090 - CL:0000568: - text: CL:0000568 - description: amine precursor uptake and decarboxylation cell - meaning: CL:0000568 - CL:1000426: - text: CL:1000426 - description: chromaffin cell of adrenal gland - meaning: CL:1000426 - CL:0000100: - text: CL:0000100 - description: motor neuron - meaning: CL:0000100 - CL:0011109: - text: CL:0011109 - description: hypocretin-secreting neuron - meaning: CL:0011109 - CL:4023171: - text: CL:4023171 - description: trigeminal motor neuron - meaning: CL:4023171 - CL:1001434: - text: CL:1001434 - description: olfactory bulb interneuron - meaning: CL:1001434 - CL:0000494: - text: CL:0000494 - description: UV sensitive photoreceptor cell - meaning: CL:0000494 - CL:0004117: - text: CL:0004117 - description: retinal ganglion cell A - meaning: CL:0004117 - CL:0000205: - text: CL:0000205 - description: thermoreceptor cell - meaning: CL:0000205 - CL:0004217: - text: CL:0004217 - description: H1 horizontal cell - meaning: CL:0004217 - CL:0000200: - text: CL:0000200 - description: touch receptor cell - meaning: CL:0000200 - CL:4023111: - text: CL:4023111 - description: cerebral cortex pyramidal neuron - meaning: CL:4023111 - CL:4032001: - text: CL:4032001 - description: reelin GABAergic cortical interneuron - meaning: CL:4032001 - CL:4023076: - text: CL:4023076 - description: Martinotti neuron - meaning: CL:4023076 - CL:0000753: - text: CL:0000753 - description: type 1 cone bipolar cell (sensu Mus) - meaning: CL:0000753 - CL:1001451: - text: CL:1001451 - description: sensory neuron of dorsal root ganglion - meaning: CL:1001451 - CL:4023021: - text: CL:4023021 - description: static gamma motor neuron - meaning: CL:4023021 - CL:0002066: - text: CL:0002066 - description: Feyrter cell - meaning: CL:0002066 - CL:0000598: - text: CL:0000598 - description: pyramidal neuron - meaning: CL:0000598 - CL:0000702: - text: CL:0000702 - description: R5 photoreceptor cell - meaning: CL:0000702 - CL:0008049: - text: CL:0008049 - description: Betz cell - meaning: CL:0008049 - CL:0001033: - text: CL:0001033 - description: hippocampal granule cell - meaning: CL:0001033 - CL:0000587: - text: CL:0000587 - description: cold sensing thermoreceptor cell - meaning: CL:0000587 - CL:4023161: - text: CL:4023161 - description: unipolar brush cell - meaning: CL:4023161 - CL:2000031: - text: CL:2000031 - description: lateral line ganglion neuron - meaning: CL:2000031 - CL:4023119: - text: CL:4023119 - description: displaced amacrine cell - meaning: CL:4023119 - CL:1001569: - text: CL:1001569 - description: hippocampal interneuron - meaning: CL:1001569 - CL:4023130: - text: CL:4023130 - description: kisspeptin neuron - meaning: CL:4023130 - CL:4023090: - text: CL:4023090 - description: small basket cell - meaning: CL:4023090 - CL:4023033: - text: CL:4023033 - description: OFF retinal ganglion cell - meaning: CL:4023033 - CL:4023112: - text: CL:4023112 - description: vestibular afferent neuron - meaning: CL:4023112 - CL:0004234: - text: CL:0004234 - description: diffuse multistratified amacrine cell - meaning: CL:0004234 - CL:0002082: - text: CL:0002082 - description: type II cell of adrenal medulla - meaning: CL:0002082 - CL:0010011: - text: CL:0010011 - description: cerebral cortex GABAergic interneuron - meaning: CL:0010011 - CL:4030052: - text: CL:4030052 - description: nucleus accumbens shell and olfactory tubercle D2 medium spiny - neuron - meaning: CL:4030052 - CL:0000604: - text: CL:0000604 - description: retinal rod cell - meaning: CL:0000604 - CL:4030027: - text: CL:4030027 - description: GABAergic amacrine cell - meaning: CL:4030027 - CL:1001561: - text: CL:1001561 - description: vomeronasal sensory neuron - meaning: CL:1001561 - CL:0000210: - text: CL:0000210 - description: photoreceptor cell - meaning: CL:0000210 - CL:4023012: - text: CL:4023012 - description: near-projecting glutamatergic cortical neuron - meaning: CL:4023012 - CL:4023087: - text: CL:4023087 - description: fan Martinotti neuron - meaning: CL:4023087 - CL:0000028: - text: CL:0000028 - description: CNS neuron (sensu Nematoda and Protostomia) - meaning: CL:0000028 - CL:0000006: - text: CL:0000006 - description: neuronal receptor cell - meaning: CL:0000006 - CL:0004247: - text: CL:0004247 - description: bistratified cell - meaning: CL:0004247 - CL:0010012: - text: CL:0010012 - description: cerebral cortex neuron - meaning: CL:0010012 - CL:0004245: - text: CL:0004245 - description: indoleamine-accumulating amacrine cell - meaning: CL:0004245 - CL:0004224: - text: CL:0004224 - description: AB diffuse-2 amacrine cell - meaning: CL:0004224 - CL:0003009: - text: CL:0003009 - description: G6 retinal ganglion cell - meaning: CL:0003009 - CL:0000679: - text: CL:0000679 - description: glutamatergic neuron - meaning: CL:0000679 - CL:0000166: - text: CL:0000166 - description: chromaffin cell - meaning: CL:0000166 - CL:4023088: - text: CL:4023088 - description: large basket cell - meaning: CL:4023088 - CL:4030057: - text: CL:4030057 - description: eccentric medium spiny neuron - meaning: CL:4030057 - CL:4023024: - text: CL:4023024 - description: neurogliaform lamp5 GABAergic cortical interneuron (Mmus) - meaning: CL:4023024 - CL:0005024: - text: CL:0005024 - description: somatomotor neuron - meaning: CL:0005024 - CL:4023049: - text: CL:4023049 - description: L5 intratelencephalic projecting glutamatergic neuron of the - primary motor cortex - meaning: CL:4023049 - CL:0000573: - text: CL:0000573 - description: retinal cone cell - meaning: CL:0000573 - CL:4023123: - text: CL:4023123 - description: hypothalamus kisspeptin neuron - meaning: CL:4023123 - CL:0000376: - text: CL:0000376 - description: humidity receptor cell - meaning: CL:0000376 - CL:0004235: - text: CL:0004235 - description: AB broad diffuse-1 amacrine cell - meaning: CL:0004235 - CL:0000106: - text: CL:0000106 - description: unipolar neuron - meaning: CL:0000106 - CL:0001032: - text: CL:0001032 - description: cortical granule cell - meaning: CL:0001032 - CL:0000561: - text: CL:0000561 - description: amacrine cell - meaning: CL:0000561 - CL:4023093: - text: CL:4023093 - description: stellate pyramidal neuron - meaning: CL:4023093 - CL:0000247: - text: CL:0000247 - description: Rohon-Beard neuron - meaning: CL:0000247 - CL:0003008: - text: CL:0003008 - description: G5 retinal ganglion cell - meaning: CL:0003008 - CL:0000203: - text: CL:0000203 - description: gravity sensitive cell - meaning: CL:0000203 - CL:0003037: - text: CL:0003037 - description: M7-ON retinal ganglion cell - meaning: CL:0003037 - CL:0004221: - text: CL:0004221 - description: flag A amacrine cell - meaning: CL:0004221 - CL:0000638: - text: CL:0000638 - description: acidophil cell of pars distalis of adenohypophysis - meaning: CL:0000638 - CL:0004229: - text: CL:0004229 - description: A2-like amacrine cell - meaning: CL:0004229 - CL:4023120: - text: CL:4023120 - description: cochlea auditory hair cell - meaning: CL:4023120 - CL:0008032: - text: CL:0008032 - description: rosehip neuron - meaning: CL:0008032 - CL:0008027: - text: CL:0008027 - description: rod bipolar cell (sensu Mus) - meaning: CL:0008027 - CL:0000497: - text: CL:0000497 - description: red sensitive photoreceptor cell - meaning: CL:0000497 - CL:4023062: - text: CL:4023062 - description: dentate gyrus neuron - meaning: CL:4023062 - CL:0002516: - text: CL:0002516 - description: interrenal chromaffin cell - meaning: CL:0002516 - CL:0004119: - text: CL:0004119 - description: retinal ganglion cell B1 - meaning: CL:0004119 - CL:4030039: - text: CL:4030039 - description: von Economo neuron - meaning: CL:4030039 - CL:4023036: - text: CL:4023036 - description: chandelier pvalb GABAergic cortical interneuron - meaning: CL:4023036 - CL:0000117: - text: CL:0000117 - description: CNS neuron (sensu Vertebrata) - meaning: CL:0000117 - CL:4023015: - text: CL:4023015 - description: sncg GABAergic cortical interneuron - meaning: CL:4023015 - CL:4033033: - text: CL:4033033 - description: flat midget bipolar cell - meaning: CL:4033033 - CL:0000626: - text: CL:0000626 - description: olfactory granule cell - meaning: CL:0000626 - CL:0004218: - text: CL:0004218 - description: H2 horizontal cell - meaning: CL:0004218 - CL:0004233: - text: CL:0004233 - description: DAPI-3 amacrine cell - meaning: CL:0004233 - CL:0003021: - text: CL:0003021 - description: retinal ganglion cell C4 - meaning: CL:0003021 - CL:0000489: - text: CL:0000489 - description: scotopic photoreceptor cell - meaning: CL:0000489 - CL:4023159: - text: CL:4023159 - description: double bouquet cell - meaning: CL:4023159 - CL:0002612: - text: CL:0002612 - description: neuron of the ventral spinal cord - meaning: CL:0002612 - CL:0000476: - text: CL:0000476 - description: thyrotroph - meaning: CL:0000476 - CL:4033034: - text: CL:4033034 - description: invaginating midget bipolar cell - meaning: CL:4033034 - CL:4023029: - text: CL:4023029 - description: indirect pathway medium spiny neuron - meaning: CL:4023029 - CL:0004236: - text: CL:0004236 - description: AB broad diffuse-2 amacrine cell - meaning: CL:0004236 - CL:0003017: - text: CL:0003017 - description: retinal ganglion cell B3 outer - meaning: CL:0003017 - CL:0000759: - text: CL:0000759 - description: type 7 cone bipolar cell (sensu Mus) - meaning: CL:0000759 - CL:0000740: - text: CL:0000740 - description: retinal ganglion cell - meaning: CL:0000740 - CL:0004120: - text: CL:0004120 - description: retinal ganglion cell A1 - meaning: CL:0004120 - CL:3000002: - text: CL:3000002 - description: sympathetic noradrenergic neuron - meaning: CL:3000002 - CL:0003023: - text: CL:0003023 - description: retinal ganglion cell C6 - meaning: CL:0003023 - CL:0000690: - text: CL:0000690 - description: R2 photoreceptor cell - meaning: CL:0000690 - CL:4023047: - text: CL:4023047 - description: L2/3 intratelencephalic projecting glutamatergic neuron of the - primary motor cortex - meaning: CL:4023047 - CL:4023022: - text: CL:4023022 - description: canopy lamp5 GABAergic cortical interneuron (Mmus) - meaning: CL:4023022 - CL:4023060: - text: CL:4023060 - description: hippocampal CA1-3 neuron - meaning: CL:4023060 - CL:0000758: - text: CL:0000758 - description: type 6 cone bipolar cell (sensu Mus) - meaning: CL:0000758 - CL:0000535: - text: CL:0000535 - description: secondary neuron (sensu Teleostei) - meaning: CL:0000535 - CL:4023055: - text: CL:4023055 - description: corticothalamic VAL/VM projecting glutamatergic neuron of the - primary motor cortex - meaning: CL:4023055 - CL:1000467: - text: CL:1000467 - description: chromaffin cell of left ovary - meaning: CL:1000467 - CL:0011002: - text: CL:0011002 - description: lateral motor column neuron - meaning: CL:0011002 - CL:0004244: - text: CL:0004244 - description: WF4 amacrine cell - meaning: CL:0004244 - CL:1000223: - text: CL:1000223 - description: lung neuroendocrine cell - meaning: CL:1000223 - CL:1000385: - text: CL:1000385 - description: type 2 vestibular sensory cell of epithelium of crista of ampulla - of semicircular duct of membranous labyrinth - meaning: CL:1000385 - CL:0000691: - text: CL:0000691 - description: stellate interneuron - meaning: CL:0000691 - CL:4023008: - text: CL:4023008 - description: intratelencephalic-projecting glutamatergic cortical neuron - meaning: CL:4023008 - CL:4023044: - text: CL:4023044 - description: non-medulla, extratelencephalic-projecting glutamatergic neuron - of the primary motor cortex - meaning: CL:4023044 - CL:0000850: - text: CL:0000850 - description: serotonergic neuron - meaning: CL:0000850 - CL:0000695: - text: CL:0000695 - description: Cajal-Retzius cell - meaning: CL:0000695 - CL:0003051: - text: CL:0003051 - description: UV cone cell - meaning: CL:0003051 - CL:0000402: - text: CL:0000402 - description: CNS interneuron - meaning: CL:0000402 - CL:0005023: - text: CL:0005023 - description: branchiomotor neuron - meaning: CL:0005023 - CL:4023043: - text: CL:4023043 - description: L5/6 near-projecting glutamatergic neuron of the primary motor - cortex - meaning: CL:4023043 - CL:0004162: - text: CL:0004162 - description: 360 nm-cone - meaning: CL:0004162 - CL:0011003: - text: CL:0011003 - description: magnocellular neurosecretory cell - meaning: CL:0011003 - CL:0004230: - text: CL:0004230 - description: diffuse bistratified amacrine cell - meaning: CL:0004230 - CL:1001505: - text: CL:1001505 - description: parvocellular neurosecretory cell - meaning: CL:1001505 - CL:0011106: - text: CL:0011106 - description: GABAnergic interplexiform cell - meaning: CL:0011106 - CL:0000437: - text: CL:0000437 - description: gonadtroph - meaning: CL:0000437 - CL:4023010: - text: CL:4023010 - description: alpha7 GABAergic cortical interneuron (Mmus) - meaning: CL:4023010 - CL:4023046: - text: CL:4023046 - description: L6b subplate glutamatergic neuron of the primary motor cortex - meaning: CL:4023046 - CL:0000109: - text: CL:0000109 - description: adrenergic neuron - meaning: CL:0000109 - CL:0011000: - text: CL:0011000 - description: dorsal horn interneuron - meaning: CL:0011000 - CL:0000251: - text: CL:0000251 - description: extramedullary cell - meaning: CL:0000251 - CL:0003044: - text: CL:0003044 - description: M11 retinal ganglion cell - meaning: CL:0003044 - CL:4023053: - text: CL:4023053 - description: spinal interneuron synapsing Betz cell - meaning: CL:4023053 - CL:1000378: - text: CL:1000378 - description: type 1 vestibular sensory cell of stato-acoustic epithelium - meaning: CL:1000378 - CL:4023124: - text: CL:4023124 - description: dentate gyrus kisspeptin neuron - meaning: CL:4023124 - CL:1000427: - text: CL:1000427 - description: adrenal cortex chromaffin cell - meaning: CL:1000427 - CL:0000207: - text: CL:0000207 - description: olfactory receptor cell - meaning: CL:0000207 - CL:4023162: - text: CL:4023162 - description: bushy cell - meaning: CL:4023162 - CL:2000019: - text: CL:2000019 - description: compound eye photoreceptor cell - meaning: CL:2000019 - CL:4023086: - text: CL:4023086 - description: T Martinotti neuron - meaning: CL:4023086 - CL:0003012: - text: CL:0003012 - description: G9 retinal ganglion cell - meaning: CL:0003012 - CL:0002270: - text: CL:0002270 - description: type EC2 enteroendocrine cell - meaning: CL:0002270 - CL:2000024: - text: CL:2000024 - description: spinal cord medial motor column neuron - meaning: CL:2000024 - CL:0003022: - text: CL:0003022 - description: retinal ganglion cell C5 - meaning: CL:0003022 - CL:0000104: - text: CL:0000104 - description: multipolar neuron - meaning: CL:0000104 - CL:4023050: - text: CL:4023050 - description: L6 intratelencephalic projecting glutamatergic neuron of the - primary motor cortex - meaning: CL:4023050 - CL:4023030: - text: CL:4023030 - description: L2/3/5 fan Martinotti sst GABAergic cortical interneuron (Mmus) - meaning: CL:4023030 - CL:0000741: - text: CL:0000741 - description: spinal accessory motor neuron - meaning: CL:0000741 - CL:4033010: - text: CL:4033010 - description: neuroendocrine cell of epithelium of lobar bronchus - meaning: CL:4033010 - CL:1000425: - text: CL:1000425 - description: chromaffin cell of paraganglion - meaning: CL:1000425 - CL:4030051: - text: CL:4030051 - description: nucleus accumbens shell and olfactory tubercle D1 medium spiny - neuron - meaning: CL:4030051 - CL:0000567: - text: CL:0000567 - description: polymodal nocireceptor - meaning: CL:0000567 - CL:0004215: - text: CL:0004215 - description: type 5a cone bipolar cell - meaning: CL:0004215 - CL:0003032: - text: CL:0003032 - description: M3-OFF retinal ganglion cell - meaning: CL:0003032 - CL:4023079: - text: CL:4023079 - description: midbrain-derived inhibitory neuron - meaning: CL:4023079 - CL:0000099: - text: CL:0000099 - description: interneuron - meaning: CL:0000099 - CL:0000253: - text: CL:0000253 - description: eurydendroid cell - meaning: CL:0000253 - CL:0008013: - text: CL:0008013 - description: cranial visceromotor neuron - meaning: CL:0008013 - CL:0005000: - text: CL:0005000 - description: spinal cord interneuron - meaning: CL:0005000 - CL:0004222: - text: CL:0004222 - description: flag B amacrine cell - meaning: CL:0004222 - CL:0000617: - text: CL:0000617 - description: GABAergic neuron - meaning: CL:0000617 - CL:0003010: - text: CL:0003010 - description: G7 retinal ganglion cell - meaning: CL:0003010 - CL:0000577: - text: CL:0000577 - description: type EC enteroendocrine cell - meaning: CL:0000577 - CL:0003018: - text: CL:0003018 - description: retinal ganglion cell B3 inner - meaning: CL:0003018 - CL:0002083: - text: CL:0002083 - description: type I cell of adrenal medulla - meaning: CL:0002083 - CL:4023081: - text: CL:4023081 - description: inverted L6 intratelencephalic projecting glutamatergic neuron - of the primary motor cortex (Mmus) - meaning: CL:4023081 - CL:0004251: - text: CL:0004251 - description: narrow field retinal amacrine cell - meaning: CL:0004251 - CL:4023092: - text: CL:4023092 - description: inverted pyramidal neuron - meaning: CL:4023092 - CL:0002608: - text: CL:0002608 - description: hippocampal neuron - meaning: CL:0002608 - CL:0008048: - text: CL:0008048 - description: upper motor neuron - meaning: CL:0008048 - CL:0011113: - text: CL:0011113 - description: spiral ganglion neuron - meaning: CL:0011113 - CL:0000601: - text: CL:0000601 - description: cochlear outer hair cell - meaning: CL:0000601 - CL:0003041: - text: CL:0003041 - description: M9-ON retinal ganglion cell - meaning: CL:0003041 - CL:4023042: - text: CL:4023042 - description: L6 corticothalamic-projecting glutamatergic cortical neuron - meaning: CL:4023042 - CL:0000199: - text: CL:0000199 - description: mechanoreceptor cell - meaning: CL:0000199 - CL:1001571: - text: CL:1001571 - description: hippocampal pyramidal neuron - meaning: CL:1001571 - CL:2000048: - text: CL:2000048 - description: anterior horn motor neuron - meaning: CL:2000048 - CL:4023170: - text: CL:4023170 - description: trigeminal sensory neuron - meaning: CL:4023170 - CL:0002614: - text: CL:0002614 - description: neuron of the substantia nigra - meaning: CL:0002614 diff --git a/docs/gallery/schemasheets/nwb_static_enums.yaml b/docs/gallery/schemasheets/nwb_static_enums.yaml deleted file mode 100644 index 222205959..000000000 --- a/docs/gallery/schemasheets/nwb_static_enums.yaml +++ /dev/null @@ -1,52 +0,0 @@ -classes: - BrainSample: - slot_usage: - cell_type: {} - slots: - - cell_type -default_prefix: TEMP -default_range: string -description: this schema demonstrates the use of static enums -enums: - NeuronOrGlialCellTypeEnum: - description: Enumeration to capture various cell types found in the brain. - permissible_values: - ASTROCYTE: - description: Characteristic star-shaped glial cells in the brain and spinal - cord. - meaning: CL:0000127 - INTERNEURON: - description: Neurons whose axons (and dendrites) are limited to a single brain - area. - meaning: CL:0000099 - MICROGLIAL_CELL: - description: Microglia are the resident immune cells of the brain and constantly - patrol the cerebral microenvironment to respond to pathogens and damage. - meaning: CL:0000129 - MOTOR_NEURON: - description: Neurons whose cell body is located in the motor cortex, brainstem - or the spinal cord, and whose axon (fiber) projects to the spinal cord or - outside of the spinal cord to directly or indirectly control effector organs, - mainly muscles and glands. - meaning: CL:0000100 - OLIGODENDROCYTE: - description: Type of neuroglia whose main functions are to provide support - and insulation to axons within the central nervous system (CNS) of jawed - vertebrates. - meaning: CL:0000128 - PYRAMIDAL_NEURON: - description: Neurons with a pyramidal shaped cell body (soma) and two distinct - dendritic trees. - meaning: CL:0000598 -id: https://w3id.org/linkml/examples/nwb_static_enums -imports: -- linkml:types -name: nwb_static_enums -prefixes: - CL: http://purl.obolibrary.org/obo/CL_ - TEMP: https://example.org/TEMP/ - linkml: https://w3id.org/linkml/ -slots: - cell_type: - required: true -title: static enums example diff --git a/src/hdmf/container.py b/src/hdmf/container.py index ca2c5252b..287809406 100644 --- a/src/hdmf/container.py +++ b/src/hdmf/container.py @@ -112,12 +112,6 @@ def _field_config(self, arg_name, val, type_map): itself is only one file. When a user loads custom configs, the config is appended/modified. The modifications are not written to file, avoiding permanent modifications. """ - # If the val has been manually wrapped then skip checking the config for the attr - if isinstance(val, TermSetWrapper): - msg = "Field value already wrapped with TermSetWrapper." - warn(msg) - return val - configurator = type_map.type_config if len(configurator.path)>0: @@ -127,6 +121,12 @@ def _field_config(self, arg_name, val, type_map): else: return val + # If the val has been manually wrapped then skip checking the config for the attr + if isinstance(val, TermSetWrapper): + msg = "Field value already wrapped with TermSetWrapper." + warn(msg) + return val + # check to see that the namespace for the container is in the config if self.namespace not in termset_config['namespaces']: msg = "%s not found within loaded configuration." % self.namespace From 8a2658f223ea08830333e5abeb3d823952972d1b Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Mon, 20 May 2024 15:40:18 -0700 Subject: [PATCH 07/33] Config Typemap generalize (#1117) * Config Typemap generalize * Update CHANGELOG.md --- CHANGELOG.md | 1 + src/hdmf/common/__init__.py | 23 +++++++++++++++++------ 2 files changed, 18 insertions(+), 6 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 47cb8c8d6..f14ce7c8d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Updated `TermSetWrapper` to support validating a single field within a compound array. @mavaylon1 [#1061](https://github.com/hdmf-dev/hdmf/pull/1061) - Updated testing to not install in editable mode and not run `coverage` by default. @rly [#1107](https://github.com/hdmf-dev/hdmf/pull/1107) - Add `post_init_method` parameter when generating classes to perform post-init functionality, i.e., validation. @mavaylon1 [#1089](https://github.com/hdmf-dev/hdmf/pull/1089) +- Updated loading, unloading, and getting the `TypeConfigurator` to support a `TypeMap` parameter. @mavaylon1 [#1117](https://github.com/hdmf-dev/hdmf/pull/1117) ### Bug Fixes - Fixed `TermSetWrapper` warning raised during the setters. @mavaylon1 [#1116](https://github.com/hdmf-dev/hdmf/pull/1116) diff --git a/src/hdmf/common/__init__.py b/src/hdmf/common/__init__.py index 4d724d1d1..5c9d9a3b7 100644 --- a/src/hdmf/common/__init__.py +++ b/src/hdmf/common/__init__.py @@ -22,6 +22,7 @@ global __TYPE_MAP @docval({'name': 'config_path', 'type': str, 'doc': 'Path to the configuration file.'}, + {'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None}, is_method=False) def load_type_config(**kwargs): """ @@ -29,23 +30,33 @@ def load_type_config(**kwargs): NOTE: This config is global and shared across all type maps. """ config_path = kwargs['config_path'] - __TYPE_MAP.type_config.load_type_config(config_path) + type_map = kwargs['type_map'] or get_type_map() -def get_loaded_type_config(): + type_map.type_config.load_type_config(config_path) + +@docval({'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None}, + is_method=False) +def get_loaded_type_config(**kwargs): """ This method returns the entire config file. """ - if __TYPE_MAP.type_config.config is None: + type_map = kwargs['type_map'] or get_type_map() + + if type_map.type_config.config is None: msg = "No configuration is loaded." raise ValueError(msg) else: - return __TYPE_MAP.type_config.config + return type_map.type_config.config -def unload_type_config(): +@docval({'name': 'type_map', 'type': TypeMap, 'doc': 'The TypeMap.', 'default': None}, + is_method=False) +def unload_type_config(**kwargs): """ Unload the configuration file. """ - return __TYPE_MAP.type_config.unload_type_config() + type_map = kwargs['type_map'] or get_type_map() + + return type_map.type_config.unload_type_config() # a function to register a container classes with the global map @docval({'name': 'data_type', 'type': str, 'doc': 'the data_type to get the spec for'}, From e6e6c5bbd418270aa19c7d510a1123c973cec167 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 20 May 2024 19:03:38 -0400 Subject: [PATCH 08/33] Expose progress bar class control (#1110) * expose progress bar class control * update types * grab progress bar class from kwargs * fix * swap back to callable but from typing * swap from typing to collections * Update CHANGELOG.md * add test * [pre-commit.ci] auto fixes from pre-commit.com hooks for more information, see https://pre-commit.ci * fix --------- Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- CHANGELOG.md | 1 + src/hdmf/data_utils.py | 35 +++++++++++++++---- .../test_core_GenericDataChunkIterator.py | 29 ++++++++++++++- 3 files changed, 58 insertions(+), 7 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f14ce7c8d..bf2134756 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Updated `TermSetWrapper` to support validating a single field within a compound array. @mavaylon1 [#1061](https://github.com/hdmf-dev/hdmf/pull/1061) - Updated testing to not install in editable mode and not run `coverage` by default. @rly [#1107](https://github.com/hdmf-dev/hdmf/pull/1107) - Add `post_init_method` parameter when generating classes to perform post-init functionality, i.e., validation. @mavaylon1 [#1089](https://github.com/hdmf-dev/hdmf/pull/1089) +- Exposed `progress_bar_class` to the `GenericDataChunkIterator` for more custom control over display of progress while iterating. @codycbakerphd [#1110](https://github.com/hdmf-dev/hdmf/pull/1110) - Updated loading, unloading, and getting the `TypeConfigurator` to support a `TypeMap` parameter. @mavaylon1 [#1117](https://github.com/hdmf-dev/hdmf/pull/1117) ### Bug Fixes diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index 23f0b4019..0e83bde2d 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -1,9 +1,9 @@ import copy import math from abc import ABCMeta, abstractmethod -from collections.abc import Iterable +from collections.abc import Iterable, Callable from warnings import warn -from typing import Tuple, Callable +from typing import Tuple from itertools import product, chain import h5py @@ -179,9 +179,15 @@ class GenericDataChunkIterator(AbstractDataChunkIterator): doc="Display a progress bar with iteration rate and estimated completion time.", default=False, ), + dict( + name="progress_bar_class", + type=Callable, + doc="The progress bar class to use. Defaults to tqdm.tqdm if the TQDM package is installed.", + default=None, + ), dict( name="progress_bar_options", - type=None, + type=dict, doc="Dictionary of keyword arguments to be passed directly to tqdm.", default=None, ), @@ -199,8 +205,23 @@ def __init__(self, **kwargs): HDF5 recommends chunk size in the range of 2 to 16 MB for optimal cloud performance. https://youtu.be/rcS5vt-mKok?t=621 """ - buffer_gb, buffer_shape, chunk_mb, chunk_shape, self.display_progress, progress_bar_options = getargs( - "buffer_gb", "buffer_shape", "chunk_mb", "chunk_shape", "display_progress", "progress_bar_options", kwargs + ( + buffer_gb, + buffer_shape, + chunk_mb, + chunk_shape, + self.display_progress, + progress_bar_class, + progress_bar_options, + ) = getargs( + "buffer_gb", + "buffer_shape", + "chunk_mb", + "chunk_shape", + "display_progress", + "progress_bar_class", + "progress_bar_options", + kwargs, ) self.progress_bar_options = progress_bar_options or dict() @@ -277,11 +298,13 @@ def __init__(self, **kwargs): try: from tqdm import tqdm + progress_bar_class = progress_bar_class or tqdm + if "total" in self.progress_bar_options: warn("Option 'total' in 'progress_bar_options' is not allowed to be over-written! Ignoring.") self.progress_bar_options.pop("total") - self.progress_bar = tqdm(total=self.num_buffers, **self.progress_bar_options) + self.progress_bar = progress_bar_class(total=self.num_buffers, **self.progress_bar_options) except ImportError: warn( "You must install tqdm to use the progress bar feature (pip install tqdm)! " diff --git a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py index debac9cab..2117eb6d0 100644 --- a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py +++ b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py @@ -4,7 +4,7 @@ from pathlib import Path from tempfile import mkdtemp from shutil import rmtree -from typing import Tuple, Iterable, Callable +from typing import Tuple, Iterable, Callable, Union from sys import version_info import h5py @@ -408,6 +408,33 @@ def test_progress_bar(self): first_line = file.read() self.assertIn(member=desc, container=first_line) + @unittest.skipIf(not TQDM_INSTALLED, "optional tqdm module is not installed") + def test_progress_bar_class(self): + import tqdm + + class MyCustomProgressBar(tqdm.tqdm): + def update(self, n: int = 1) -> Union[bool, None]: + displayed = super().update(n) + print(f"Custom injection on step {n}") # noqa: T201 + + return displayed + + out_text_file = self.test_dir / "test_progress_bar_class.txt" + desc = "Testing progress bar..." + with open(file=out_text_file, mode="w") as file: + iterator = self.TestNumpyArrayDataChunkIterator( + array=self.test_array, + display_progress=True, + progress_bar_class=MyCustomProgressBar, + progress_bar_options=dict(file=file, desc=desc), + ) + j = 0 + for buffer in iterator: + j += 1 # dummy operation; must be silent for proper updating of bar + with open(file=out_text_file, mode="r") as file: + first_line = file.read() + self.assertIn(member=desc, container=first_line) + @unittest.skipIf(not TQDM_INSTALLED, "optional tqdm module is installed") def test_progress_bar_no_options(self): dci = self.TestNumpyArrayDataChunkIterator(array=self.test_array, display_progress=True) From 77319108a6074a4f3b6317b92fa935dce21dc1f3 Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 23:09:18 +0000 Subject: [PATCH 09/33] Bump tqdm from 4.41.0 to 4.66.3 (#1109) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan Ly --- requirements-opt.txt | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/requirements-opt.txt b/requirements-opt.txt index 53fd11e3a..c1d34220b 100644 --- a/requirements-opt.txt +++ b/requirements-opt.txt @@ -1,5 +1,5 @@ # pinned dependencies that are optional. used to reproduce an entire development environment to use HDMF -tqdm==4.66.2 +tqdm==4.66.3 zarr==2.17.1 linkml-runtime==1.7.4; python_version >= "3.9" schemasheets==0.2.1; python_version >= "3.9" From 6a0f9d8184387f73c6abed65d919d2aafc03f832 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 20 May 2024 23:24:09 +0000 Subject: [PATCH 10/33] [pre-commit.ci] pre-commit autoupdate (#1082) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ryan Ly --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 786a3e4b7..3f80639c7 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -1,7 +1,7 @@ # NOTE: run `pre-commit autoupdate` to update hooks to latest version repos: - repo: https://github.com/pre-commit/pre-commit-hooks - rev: v4.5.0 + rev: v4.6.0 hooks: - id: check-yaml - id: end-of-file-fixer @@ -18,7 +18,7 @@ repos: # hooks: # - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.3.3 + rev: v0.4.4 hooks: - id: ruff # - repo: https://github.com/econchick/interrogate From c18d1b3d108ee8798c977a1c994fc528b4fa05b0 Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Mon, 20 May 2024 22:09:54 -0400 Subject: [PATCH 11/33] Expose AWS Region to HDF5IO (#1040) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ryan Ly --- .github/workflows/run_all_tests.yml | 8 +-- .github/workflows/run_coverage.yml | 55 ++++++++++++++++++ .github/workflows/run_tests.yml | 4 +- CHANGELOG.md | 1 + src/hdmf/backends/hdf5/h5tools.py | 38 ++++++++++--- tests/unit/test_io_hdf5_streaming.py | 56 +++++++++++++++++++ tests/unit/utils_test/test_core_DataIO.py | 13 ++++- .../test_core_GenericDataChunkIterator.py | 1 - 8 files changed, 159 insertions(+), 17 deletions(-) diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml index def51537f..8df190d55 100644 --- a/.github/workflows/run_all_tests.yml +++ b/.github/workflows/run_all_tests.yml @@ -197,7 +197,7 @@ jobs: run: | tox -e wheelinstall --installpkg dist/*.tar.gz - run-gallery-ros3-tests: + run-ros3-tests: name: ${{ matrix.name }} runs-on: ${{ matrix.os }} defaults: @@ -210,9 +210,9 @@ jobs: fail-fast: false matrix: include: - - { name: linux-gallery-python3.12-ros3 , python-ver: "3.12", os: ubuntu-latest } - - { name: windows-gallery-python3.12-ros3 , python-ver: "3.12", os: windows-latest } - - { name: macos-gallery-python3.12-ros3 , python-ver: "3.12", os: macos-latest } + - { name: linux-python3.12-ros3 , python-ver: "3.12", os: ubuntu-latest } + - { name: windows-python3.12-ros3 , python-ver: "3.12", os: windows-latest } + - { name: macos-python3.12-ros3 , python-ver: "3.12", os: macos-latest } steps: - name: Checkout repo with submodules uses: actions/checkout@v4 diff --git a/.github/workflows/run_coverage.yml b/.github/workflows/run_coverage.yml index a72a05e73..bd2eeb921 100644 --- a/.github/workflows/run_coverage.yml +++ b/.github/workflows/run_coverage.yml @@ -70,3 +70,58 @@ jobs: file: ./coverage.xml env: CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} + + run-ros3-coverage: + name: ${{ matrix.name }} + runs-on: ${{ matrix.os }} + defaults: + run: + shell: bash -l {0} # necessary for conda + concurrency: + group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.name }} + cancel-in-progress: true + strategy: + fail-fast: false + matrix: + include: + - { name: linux-python3.12-ros3 , python-ver: "3.12", os: ubuntu-latest } + steps: + - name: Checkout repo with submodules + uses: actions/checkout@v4 + with: + submodules: 'recursive' + fetch-depth: 0 # tags are required to determine the version + + - name: Set up Conda + uses: conda-incubator/setup-miniconda@v3 + with: + auto-update-conda: true + activate-environment: ros3 + environment-file: environment-ros3.yml + python-version: ${{ matrix.python-ver }} + channels: conda-forge + auto-activate-base: false + mamba-version: "*" + + - name: Install run dependencies + run: | + pip install . + pip list + + - name: Conda reporting + run: | + conda info + conda config --show-sources + conda list --show-channel-urls + + - name: Run ros3 tests # TODO include gallery tests after they are written + run: | + pytest --cov --cov-report=xml --cov-report=term tests/unit/test_io_hdf5_streaming.py + + - name: Upload coverage to Codecov + uses: codecov/codecov-action@v4 + with: + fail_ci_if_error: true + file: ./coverage.xml + env: + CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }} diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml index 2723e03d0..5e0b3bff2 100644 --- a/.github/workflows/run_tests.yml +++ b/.github/workflows/run_tests.yml @@ -209,7 +209,7 @@ jobs: --token ${{ secrets.BOT_GITHUB_TOKEN }} \ --re-upload - run-gallery-ros3-tests: + run-ros3-tests: name: ${{ matrix.name }} runs-on: ${{ matrix.os }} defaults: @@ -222,7 +222,7 @@ jobs: fail-fast: false matrix: include: - - { name: linux-gallery-python3.12-ros3 , python-ver: "3.12", os: ubuntu-latest } + - { name: linux-python3.12-ros3 , python-ver: "3.12", os: ubuntu-latest } steps: - name: Checkout repo with submodules uses: actions/checkout@v4 diff --git a/CHANGELOG.md b/CHANGELOG.md index bf2134756..8a1d0b2c5 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -8,6 +8,7 @@ - Updated `TermSetWrapper` to support validating a single field within a compound array. @mavaylon1 [#1061](https://github.com/hdmf-dev/hdmf/pull/1061) - Updated testing to not install in editable mode and not run `coverage` by default. @rly [#1107](https://github.com/hdmf-dev/hdmf/pull/1107) - Add `post_init_method` parameter when generating classes to perform post-init functionality, i.e., validation. @mavaylon1 [#1089](https://github.com/hdmf-dev/hdmf/pull/1089) +- Exposed `aws_region` to `HDF5IO` and downstream passes to `h5py.File`. @codycbakerphd [#1040](https://github.com/hdmf-dev/hdmf/pull/1040) - Exposed `progress_bar_class` to the `GenericDataChunkIterator` for more custom control over display of progress while iterating. @codycbakerphd [#1110](https://github.com/hdmf-dev/hdmf/pull/1110) - Updated loading, unloading, and getting the `TypeConfigurator` to support a `TypeMap` parameter. @mavaylon1 [#1117](https://github.com/hdmf-dev/hdmf/pull/1117) diff --git a/src/hdmf/backends/hdf5/h5tools.py b/src/hdmf/backends/hdf5/h5tools.py index 05ce36e13..0604881bb 100644 --- a/src/hdmf/backends/hdf5/h5tools.py +++ b/src/hdmf/backends/hdf5/h5tools.py @@ -62,15 +62,21 @@ def can_read(path): {'name': 'file', 'type': [File, "S3File", "RemFile"], 'doc': 'a pre-existing h5py.File, S3File, or RemFile object', 'default': None}, {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None}, + { + 'name': 'aws_region', + 'type': str, + 'doc': 'If driver is ros3, then specify the aws region of the url.', + 'default': None + }, {'name': 'herd_path', 'type': str, 'doc': 'The path to read/write the HERD file', 'default': None},) def __init__(self, **kwargs): """Open an HDF5 file for IO. """ self.logger = logging.getLogger('%s.%s' % (self.__class__.__module__, self.__class__.__qualname__)) - path, manager, mode, comm, file_obj, driver, herd_path = popargs('path', 'manager', 'mode', + path, manager, mode, comm, file_obj, driver, aws_region, herd_path = popargs('path', 'manager', 'mode', 'comm', 'file', 'driver', - 'herd_path', + 'aws_region', 'herd_path', kwargs) self.__open_links = [] # keep track of other files opened from links in this file @@ -91,6 +97,7 @@ def __init__(self, **kwargs): elif isinstance(manager, TypeMap): manager = BuildManager(manager) self.__driver = driver + self.__aws_region = aws_region self.__comm = comm self.__mode = mode self.__file = file_obj @@ -116,6 +123,10 @@ def _file(self): def driver(self): return self.__driver + @property + def aws_region(self): + return self.__aws_region + @classmethod def __check_path_file_obj(cls, path, file_obj): if isinstance(path, Path): @@ -133,13 +144,17 @@ def __check_path_file_obj(cls, path, file_obj): return path @classmethod - def __resolve_file_obj(cls, path, file_obj, driver): + def __resolve_file_obj(cls, path, file_obj, driver, aws_region=None): + """Helper function to return a File when loading or getting namespaces from a file.""" path = cls.__check_path_file_obj(path, file_obj) if file_obj is None: file_kwargs = dict() if driver is not None: file_kwargs.update(driver=driver) + + if aws_region is not None: + file_kwargs.update(aws_region=bytes(aws_region, "ascii")) file_obj = File(path, 'r', **file_kwargs) return file_obj @@ -150,6 +165,8 @@ def __resolve_file_obj(cls, path, file_obj, driver): {'name': 'namespaces', 'type': list, 'doc': 'the namespaces to load', 'default': None}, {'name': 'file', 'type': File, 'doc': 'a pre-existing h5py.File object', 'default': None}, {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None}, + {'name': 'aws_region', 'type': str, 'doc': 'If driver is ros3, then specify the aws region of the url.', + 'default': None}, returns=("dict mapping the names of the loaded namespaces to a dict mapping included namespace names and " "the included data types"), rtype=dict) @@ -162,10 +179,10 @@ def load_namespaces(cls, **kwargs): :raises ValueError: if both `path` and `file` are supplied but `path` is not the same as the path of `file`. """ - namespace_catalog, path, namespaces, file_obj, driver = popargs( - 'namespace_catalog', 'path', 'namespaces', 'file', 'driver', kwargs) + namespace_catalog, path, namespaces, file_obj, driver, aws_region = popargs( + 'namespace_catalog', 'path', 'namespaces', 'file', 'driver', 'aws_region', kwargs) - open_file_obj = cls.__resolve_file_obj(path, file_obj, driver) + open_file_obj = cls.__resolve_file_obj(path, file_obj, driver, aws_region=aws_region) if file_obj is None: # need to close the file object that we just opened with open_file_obj: return cls.__load_namespaces(namespace_catalog, namespaces, open_file_obj) @@ -214,6 +231,8 @@ def __check_specloc(cls, file_obj): @docval({'name': 'path', 'type': (str, Path), 'doc': 'the path to the HDF5 file', 'default': None}, {'name': 'file', 'type': File, 'doc': 'a pre-existing h5py.File object', 'default': None}, {'name': 'driver', 'type': str, 'doc': 'driver for h5py to use when opening HDF5 file', 'default': None}, + {'name': 'aws_region', 'type': str, 'doc': 'If driver is ros3, then specify the aws region of the url.', + 'default': None}, returns="dict mapping names to versions of the namespaces in the file", rtype=dict) def get_namespaces(cls, **kwargs): """Get the names and versions of the cached namespaces from a file. @@ -227,9 +246,9 @@ def get_namespaces(cls, **kwargs): :raises ValueError: if both `path` and `file` are supplied but `path` is not the same as the path of `file`. """ - path, file_obj, driver = popargs('path', 'file', 'driver', kwargs) + path, file_obj, driver, aws_region = popargs('path', 'file', 'driver', 'aws_region', kwargs) - open_file_obj = cls.__resolve_file_obj(path, file_obj, driver) + open_file_obj = cls.__resolve_file_obj(path, file_obj, driver, aws_region=aws_region) if file_obj is None: # need to close the file object that we just opened with open_file_obj: return cls.__get_namespaces(open_file_obj) @@ -756,6 +775,9 @@ def open(self): if self.driver is not None: kwargs.update(driver=self.driver) + if self.driver == "ros3" and self.aws_region is not None: + kwargs.update(aws_region=bytes(self.aws_region, "ascii")) + self.__file = File(self.source, open_flag, **kwargs) def close(self, close_links=True): diff --git a/tests/unit/test_io_hdf5_streaming.py b/tests/unit/test_io_hdf5_streaming.py index d1c9d1ab3..d82c9c5c3 100644 --- a/tests/unit/test_io_hdf5_streaming.py +++ b/tests/unit/test_io_hdf5_streaming.py @@ -2,7 +2,9 @@ import os import urllib.request import h5py +import warnings +from hdmf.backends.hdf5.h5tools import HDF5IO from hdmf.build import TypeMap, BuildManager from hdmf.common import get_hdf5io, get_type_map from hdmf.spec import GroupSpec, DatasetSpec, SpecNamespace, NamespaceBuilder, NamespaceCatalog @@ -10,6 +12,7 @@ from hdmf.utils import docval, get_docval + class TestRos3(TestCase): """Test reading an HDMF file using HDF5 ROS3 streaming. @@ -77,6 +80,8 @@ def setUp(self): self.manager = BuildManager(type_map) + warnings.filterwarnings(action="ignore", message="Ignoring cached namespace .*") + def tearDown(self): if os.path.exists(self.ns_filename): os.remove(self.ns_filename) @@ -89,6 +94,57 @@ def test_basic_read(self): with get_hdf5io(s3_path, "r", manager=self.manager, driver="ros3") as io: io.read() + def test_basic_read_with_aws_region(self): + s3_path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + with get_hdf5io(s3_path, "r", manager=self.manager, driver="ros3", aws_region="us-east-2") as io: + io.read() + + def test_basic_read_s3_with_aws_region(self): + # NOTE: if an s3 path is used with ros3 driver, aws_region must be specified + s3_path = "s3://dandiarchive/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + with get_hdf5io(s3_path, "r", manager=self.manager, driver="ros3", aws_region="us-east-2") as io: + io.read() + assert io.aws_region == "us-east-2" + + def test_get_namespaces(self): + s3_path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + namespaces = HDF5IO.get_namespaces(s3_path, driver="ros3") + self.assertEqual(namespaces, {'core': '2.3.0', 'hdmf-common': '1.5.0', 'hdmf-experimental': '0.1.0'}) + + def test_get_namespaces_with_aws_region(self): + s3_path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + namespaces = HDF5IO.get_namespaces(s3_path, driver="ros3", aws_region="us-east-2") + self.assertEqual(namespaces, {'core': '2.3.0', 'hdmf-common': '1.5.0', 'hdmf-experimental': '0.1.0'}) + + def test_get_namespaces_s3_with_aws_region(self): + s3_path = "s3://dandiarchive/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + namespaces = HDF5IO.get_namespaces(s3_path, driver="ros3", aws_region="us-east-2") + self.assertEqual(namespaces, {'core': '2.3.0', 'hdmf-common': '1.5.0', 'hdmf-experimental': '0.1.0'}) + + def test_load_namespaces(self): + s3_path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + HDF5IO.load_namespaces(self.manager.namespace_catalog, path=s3_path, driver="ros3") + assert set(self.manager.namespace_catalog.namespaces) == set(["core", "hdmf-common", "hdmf-experimental"]) + + def test_load_namespaces_with_aws_region(self): + s3_path = "https://dandiarchive.s3.amazonaws.com/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + HDF5IO.load_namespaces(self.manager.namespace_catalog, path=s3_path, driver="ros3", aws_region="us-east-2") + assert set(self.manager.namespace_catalog.namespaces) == set(["core", "hdmf-common", "hdmf-experimental"]) + + def test_load_namespaces_s3_with_aws_region(self): + s3_path = "s3://dandiarchive/blobs/11e/c89/11ec8933-1456-4942-922b-94e5878bb991" + + HDF5IO.load_namespaces(self.manager.namespace_catalog, path=s3_path, driver="ros3", aws_region="us-east-2") + assert set(self.manager.namespace_catalog.namespaces) == set(["core", "hdmf-common", "hdmf-experimental"]) + + # Util functions and classes to enable loading of the NWB namespace -- see pynwb/src/pynwb/spec.py diff --git a/tests/unit/utils_test/test_core_DataIO.py b/tests/unit/utils_test/test_core_DataIO.py index 778dd2617..4c2ffac15 100644 --- a/tests/unit/utils_test/test_core_DataIO.py +++ b/tests/unit/utils_test/test_core_DataIO.py @@ -4,6 +4,7 @@ from hdmf.container import Data from hdmf.data_utils import DataIO from hdmf.testing import TestCase +import warnings class DataIOTests(TestCase): @@ -36,7 +37,9 @@ def test_set_dataio(self): dataio = DataIO() data = np.arange(30).reshape(5, 2, 3) container = Data('wrapped_data', data) - container.set_dataio(dataio) + msg = "Data.set_dataio() is deprecated. Please use Data.set_data_io() instead." + with self.assertWarnsWith(DeprecationWarning, msg): + container.set_dataio(dataio) self.assertIs(dataio.data, data) self.assertIs(dataio, container.data) @@ -48,7 +51,13 @@ def test_set_dataio_data_already_set(self): data = np.arange(30).reshape(5, 2, 3) container = Data('wrapped_data', data) with self.assertRaisesWith(ValueError, "cannot overwrite 'data' on DataIO"): - container.set_dataio(dataio) + with warnings.catch_warnings(record=True): + warnings.filterwarnings( + action='ignore', + category=DeprecationWarning, + message="Data.set_dataio() is deprecated. Please use Data.set_data_io() instead.", + ) + container.set_dataio(dataio) def test_dataio_options(self): """ diff --git a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py index 2117eb6d0..cb1a727a4 100644 --- a/tests/unit/utils_test/test_core_GenericDataChunkIterator.py +++ b/tests/unit/utils_test/test_core_GenericDataChunkIterator.py @@ -410,7 +410,6 @@ def test_progress_bar(self): @unittest.skipIf(not TQDM_INSTALLED, "optional tqdm module is not installed") def test_progress_bar_class(self): - import tqdm class MyCustomProgressBar(tqdm.tqdm): def update(self, n: int = 1) -> Union[bool, None]: From 9387e85058fb44ea1702ad888e9df547c41fc324 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Mon, 20 May 2024 20:34:43 -0700 Subject: [PATCH 12/33] Release 3.14 (#1118) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 8a1d0b2c5..89c937e80 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # HDMF Changelog -## HDMF 3.14.0 (Upcoming) +## HDMF 3.14.0 (May 20, 2024) ### Enhancements - Updated `_field_config` to take `type_map` as an argument for APIs. @mavaylon1 [#1094](https://github.com/hdmf-dev/hdmf/pull/1094) From 2505a0e3b310a7c096dd9e23f262255f410e7551 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Tue, 4 Jun 2024 22:19:13 -0700 Subject: [PATCH 13/33] Fix issue with resolving attribute specs with same name (#1122) * Fix issue with resolving attribute specs with same name * fix codespell --- CHANGELOG.md | 9 +- pyproject.toml | 2 +- src/hdmf/data_utils.py | 2 +- src/hdmf/spec/spec.py | 143 ++++++++++++----------- tests/unit/spec_tests/test_group_spec.py | 121 ++++++++++++++++--- 5 files changed, 186 insertions(+), 91 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 89c937e80..5ed9984db 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # HDMF Changelog +## HDMF 3.14.1 (June 3, 2024) + +### Bug Fixes +- Fixed issue with resolving attribute specs that have the same name at different levels of a spec hierarchy. + @rly [#1122](https://github.com/hdmf-dev/hdmf/pull/1122) + + ## HDMF 3.14.0 (May 20, 2024) ### Enhancements @@ -548,7 +555,7 @@ the fields (i.e., when the constructor sets some fields to fixed values). @rly Each sub-table is itself a DynamicTable that is aligned with the main table by row index. Each subtable defines a sub-category in the main table effectively creating a table with sub-headings to organize columns. @oruebel (#551) -- Add tutoral for new `AlignedDynamicTable` type. @oruebel (#571) +- Add tutorial for new `AlignedDynamicTable` type. @oruebel (#571) - Equality check for `DynamicTable` now also checks that the name and description of the table are the same. @rly (#566) ### Internal improvements diff --git a/pyproject.toml b/pyproject.toml index 67b13350b..f0a0b7fb7 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -81,7 +81,7 @@ norecursedirs = "tests/unit/helpers" [tool.codespell] skip = "htmlcov,.git,.mypy_cache,.pytest_cache,.coverage,*.pdf,*.svg,venvs,.tox,hdmf-common-schema,./docs/_build/*,*.ipynb" -ignore-words-list = "datas" +ignore-words-list = "datas,assertIn" [tool.coverage.run] branch = true diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index 0e83bde2d..c03665caa 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -938,7 +938,7 @@ class ShapeValidatorResult: {'name': 'message', 'type': str, 'doc': 'Message describing the result of the shape validation', 'default': None}, {'name': 'ignored', 'type': tuple, - 'doc': 'Axes that have been ignored in the validaton process', 'default': tuple(), 'shape': (None,)}, + 'doc': 'Axes that have been ignored in the validation process', 'default': tuple(), 'shape': (None,)}, {'name': 'unmatched', 'type': tuple, 'doc': 'List of axes that did not match during shape validation', 'default': tuple(), 'shape': (None,)}, {'name': 'error', 'type': str, 'doc': 'Error that may have occurred. One of ERROR_TYPE', 'default': None}, diff --git a/src/hdmf/spec/spec.py b/src/hdmf/spec/spec.py index 585fc6494..1a6e8d987 100644 --- a/src/hdmf/spec/spec.py +++ b/src/hdmf/spec/spec.py @@ -385,7 +385,7 @@ def resolve_spec(self, **kwargs): self.set_attribute(attribute) self.__resolved = True - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to check'}) + @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) def is_inherited_spec(self, **kwargs): ''' Return True if this spec was inherited from the parent type, False otherwise. @@ -393,13 +393,11 @@ def is_inherited_spec(self, **kwargs): Returns False if the spec is not found. ''' spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - spec = spec.name - if spec in self.__attributes: - return self.is_inherited_attribute(spec) + if spec.parent is self and spec.name in self.__attributes: + return self.is_inherited_attribute(spec.name) return False - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to check'}) + @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) def is_overridden_spec(self, **kwargs): ''' Return True if this spec overrides a specification from the parent type, False otherwise. @@ -407,10 +405,8 @@ def is_overridden_spec(self, **kwargs): Returns False if the spec is not found. ''' spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - spec = spec.name - if spec in self.__attributes: - return self.is_overridden_attribute(spec) + if spec.parent is self and spec.name in self.__attributes: + return self.is_overridden_attribute(spec.name) return False @docval({'name': 'name', 'type': str, 'doc': 'the name of the attribute to check'}) @@ -1011,85 +1007,92 @@ def is_overridden_link(self, **kwargs): raise ValueError("Link '%s' not found in spec" % name) return name in self.__overridden_links - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to check'}) + @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) def is_inherited_spec(self, **kwargs): ''' Returns 'True' if specification was inherited from a parent type ''' spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - name = spec.name - if name is None and hasattr(spec, 'data_type_def'): - name = spec.data_type_def - if name is None: # NOTE: this will return the target type for LinkSpecs - name = spec.data_type_inc - if name is None: # pragma: no cover - # this should not be possible - raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') - spec = name + spec_name = spec.name + if spec_name is None and hasattr(spec, 'data_type_def'): + spec_name = spec.data_type_def + if spec_name is None: # NOTE: this will return the target type for LinkSpecs + spec_name = spec.data_type_inc + if spec_name is None: # pragma: no cover + # this should not be possible + raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') # if the spec has a name, it will be found in __links/__groups/__datasets before __data_types/__target_types - if spec in self.__links: - return self.is_inherited_link(spec) - elif spec in self.__groups: - return self.is_inherited_group(spec) - elif spec in self.__datasets: - return self.is_inherited_dataset(spec) - elif spec in self.__data_types: + if spec_name in self.__links: + return self.is_inherited_link(spec_name) + elif spec_name in self.__groups: + return self.is_inherited_group(spec_name) + elif spec_name in self.__datasets: + return self.is_inherited_dataset(spec_name) + elif spec_name in self.__data_types: # NOTE: the same data type can be both an unnamed data type and an unnamed target type - return self.is_inherited_type(spec) - elif spec in self.__target_types: - return self.is_inherited_target_type(spec) + return self.is_inherited_type(spec_name) + elif spec_name in self.__target_types: + return self.is_inherited_target_type(spec_name) else: + # attribute spec if super().is_inherited_spec(spec): return True else: - for s in self.__datasets: - if self.is_inherited_dataset(s): - if self.__datasets[s].get_attribute(spec) is not None: - return True - for s in self.__groups: - if self.is_inherited_group(s): - if self.__groups[s].get_attribute(spec) is not None: - return True + parent_name = spec.parent.name + if parent_name is None: + parent_name = spec.parent.data_type + if isinstance(spec.parent, DatasetSpec): + if parent_name in self.__datasets: + if self.is_inherited_dataset(parent_name): + if self.__datasets[parent_name].get_attribute(spec_name) is not None: + return True + else: + if parent_name in self.__groups: + if self.is_inherited_group(parent_name): + if self.__groups[parent_name].get_attribute(spec_name) is not None: + return True return False - @docval({'name': 'spec', 'type': (Spec, str), 'doc': 'the specification to check'}) + @docval({'name': 'spec', 'type': Spec, 'doc': 'the specification to check'}) def is_overridden_spec(self, **kwargs): # noqa: C901 ''' Returns 'True' if specification overrides a specification from the parent type ''' spec = getargs('spec', kwargs) - if isinstance(spec, Spec): - name = spec.name - if name is None: - if isinstance(spec, LinkSpec): # unnamed LinkSpec cannot be overridden - return False - if spec.is_many(): # this is a wildcard spec, so it cannot be overridden - return False - name = spec.data_type_def - if name is None: # NOTE: this will return the target type for LinkSpecs - name = spec.data_type_inc - if name is None: # pragma: no cover - # this should not happen - raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') - spec = name + spec_name = spec.name + if spec_name is None: + if isinstance(spec, LinkSpec): # unnamed LinkSpec cannot be overridden + return False + if spec.is_many(): # this is a wildcard spec, so it cannot be overridden + return False + spec_name = spec.data_type_def + if spec_name is None: # NOTE: this will return the target type for LinkSpecs + spec_name = spec.data_type_inc + if spec_name is None: # pragma: no cover + # this should not happen + raise ValueError('received Spec with wildcard name but no data_type_inc or data_type_def') # if the spec has a name, it will be found in __links/__groups/__datasets before __data_types/__target_types - if spec in self.__links: - return self.is_overridden_link(spec) - elif spec in self.__groups: - return self.is_overridden_group(spec) - elif spec in self.__datasets: - return self.is_overridden_dataset(spec) - elif spec in self.__data_types: - return self.is_overridden_type(spec) + if spec_name in self.__links: + return self.is_overridden_link(spec_name) + elif spec_name in self.__groups: + return self.is_overridden_group(spec_name) + elif spec_name in self.__datasets: + return self.is_overridden_dataset(spec_name) + elif spec_name in self.__data_types: + return self.is_overridden_type(spec_name) else: if super().is_overridden_spec(spec): # check if overridden attribute return True else: - for s in self.__datasets: - if self.is_overridden_dataset(s): - if self.__datasets[s].is_overridden_spec(spec): - return True - for s in self.__groups: - if self.is_overridden_group(s): - if self.__groups[s].is_overridden_spec(spec): - return True + parent_name = spec.parent.name + if parent_name is None: + parent_name = spec.parent.data_type + if isinstance(spec.parent, DatasetSpec): + if parent_name in self.__datasets: + if self.is_overridden_dataset(parent_name): + if self.__datasets[parent_name].is_overridden_spec(spec): + return True + else: + if parent_name in self.__groups: + if self.is_overridden_group(parent_name): + if self.__groups[parent_name].is_overridden_spec(spec): + return True return False @docval({'name': 'spec', 'type': (BaseStorageSpec, str), 'doc': 'the specification to check'}) diff --git a/tests/unit/spec_tests/test_group_spec.py b/tests/unit/spec_tests/test_group_spec.py index 9c117fa1f..00a937538 100644 --- a/tests/unit/spec_tests/test_group_spec.py +++ b/tests/unit/spec_tests/test_group_spec.py @@ -365,26 +365,22 @@ def test_resolved(self): self.assertTrue(self.inc_group_spec.resolved) def test_is_inherited_spec(self): - self.assertFalse(self.def_group_spec.is_inherited_spec('attribute1')) - self.assertFalse(self.def_group_spec.is_inherited_spec('attribute2')) - self.assertTrue(self.inc_group_spec.is_inherited_spec( - AttributeSpec('attribute1', 'my first attribute', 'text') - )) - self.assertTrue(self.inc_group_spec.is_inherited_spec('attribute1')) - self.assertTrue(self.inc_group_spec.is_inherited_spec('attribute2')) - self.assertFalse(self.inc_group_spec.is_inherited_spec('attribute3')) - self.assertFalse(self.inc_group_spec.is_inherited_spec('attribute4')) + self.assertFalse(self.def_group_spec.is_inherited_spec(self.def_group_spec.attributes[0])) + self.assertFalse(self.def_group_spec.is_inherited_spec(self.def_group_spec.attributes[1])) + + attr_spec_map = {attr.name: attr for attr in self.inc_group_spec.attributes} + self.assertTrue(self.inc_group_spec.is_inherited_spec(attr_spec_map["attribute1"])) + self.assertTrue(self.inc_group_spec.is_inherited_spec(attr_spec_map["attribute2"])) + self.assertFalse(self.inc_group_spec.is_inherited_spec(attr_spec_map["attribute3"])) def test_is_overridden_spec(self): - self.assertFalse(self.def_group_spec.is_overridden_spec('attribute1')) - self.assertFalse(self.def_group_spec.is_overridden_spec('attribute2')) - self.assertFalse(self.inc_group_spec.is_overridden_spec( - AttributeSpec('attribute1', 'my first attribute', 'text') - )) - self.assertFalse(self.inc_group_spec.is_overridden_spec('attribute1')) - self.assertTrue(self.inc_group_spec.is_overridden_spec('attribute2')) - self.assertFalse(self.inc_group_spec.is_overridden_spec('attribute3')) - self.assertFalse(self.inc_group_spec.is_overridden_spec('attribute4')) + self.assertFalse(self.def_group_spec.is_overridden_spec(self.def_group_spec.attributes[0])) + self.assertFalse(self.def_group_spec.is_overridden_spec(self.def_group_spec.attributes[0])) + + attr_spec_map = {attr.name: attr for attr in self.inc_group_spec.attributes} + self.assertFalse(self.inc_group_spec.is_overridden_spec(attr_spec_map["attribute1"])) + self.assertTrue(self.inc_group_spec.is_overridden_spec(attr_spec_map["attribute2"])) + self.assertFalse(self.inc_group_spec.is_overridden_spec(attr_spec_map["attribute3"])) def test_is_inherited_attribute(self): self.assertFalse(self.def_group_spec.is_inherited_attribute('attribute1')) @@ -405,6 +401,95 @@ def test_is_overridden_attribute(self): self.inc_group_spec.is_overridden_attribute('attribute4') +class TestResolveGroupSameAttributeName(TestCase): + # https://github.com/hdmf-dev/hdmf/issues/1121 + + def test_is_inherited_two_different_datasets(self): + self.def_group_spec = GroupSpec( + doc='A test group', + data_type_def='MyGroup', + datasets=[ + DatasetSpec( + name='dset1', + doc="dset1", + dtype='int', + attributes=[AttributeSpec('attr1', 'MyGroup.dset1.attr1', 'text')] + ), + ] + ) + self.inc_group_spec = GroupSpec( + doc='A test subgroup', + data_type_def='SubGroup', + data_type_inc='MyGroup', + datasets=[ + DatasetSpec( + name='dset2', + doc="dset2", + dtype='int', + attributes=[AttributeSpec('attr1', 'SubGroup.dset2.attr1', 'text')] + ), + ] + ) + self.inc_group_spec.resolve_spec(self.def_group_spec) + + self.assertFalse(self.def_group_spec.is_inherited_spec(self.def_group_spec.datasets[0].attributes[0])) + + dset_spec_map = {dset.name: dset for dset in self.inc_group_spec.datasets} + self.assertFalse(self.inc_group_spec.is_inherited_spec(dset_spec_map["dset2"].attributes[0])) + self.assertTrue(self.inc_group_spec.is_inherited_spec(dset_spec_map["dset1"].attributes[0])) + + def test_is_inherited_different_groups_and_datasets(self): + self.def_group_spec = GroupSpec( + doc='A test group', + data_type_def='MyGroup', + attributes=[AttributeSpec('attr1', 'MyGroup.attr1', 'text')], # <-- added from above + datasets=[ + DatasetSpec( + name='dset1', + doc="dset1", + dtype='int', + attributes=[AttributeSpec('attr1', 'MyGroup.dset1.attr1', 'text')] + ), + ] + ) + self.inc_group_spec = GroupSpec( + doc='A test subgroup', + data_type_def='SubGroup', + data_type_inc='MyGroup', + attributes=[AttributeSpec('attr1', 'SubGroup.attr1', 'text')], # <-- added from above + datasets=[ + DatasetSpec( + name='dset2', + doc="dset2", + dtype='int', + attributes=[AttributeSpec('attr1', 'SubGroup.dset2.attr1', 'text')] + ), + ] + ) + self.inc_group_spec.resolve_spec(self.def_group_spec) + + self.assertFalse(self.def_group_spec.is_inherited_spec(self.def_group_spec.datasets[0].attributes[0])) + + dset_spec_map = {dset.name: dset for dset in self.inc_group_spec.datasets} + self.assertFalse(self.inc_group_spec.is_inherited_spec(dset_spec_map["dset2"].attributes[0])) + self.assertTrue(self.inc_group_spec.is_inherited_spec(dset_spec_map["dset1"].attributes[0])) + self.assertTrue(self.inc_group_spec.is_inherited_spec(self.inc_group_spec.attributes[0])) + + self.inc_group_spec2 = GroupSpec( + doc='A test subsubgroup', + data_type_def='SubSubGroup', + data_type_inc='SubGroup', + ) + self.inc_group_spec2.resolve_spec(self.inc_group_spec) + + dset_spec_map = {dset.name: dset for dset in self.inc_group_spec2.datasets} + self.assertTrue(self.inc_group_spec2.is_inherited_spec(dset_spec_map["dset1"].attributes[0])) + self.assertTrue(self.inc_group_spec2.is_inherited_spec(dset_spec_map["dset2"].attributes[0])) + self.assertTrue(self.inc_group_spec2.is_inherited_spec(self.inc_group_spec2.attributes[0])) + + + + class GroupSpecWithLinksTest(TestCase): def test_constructor(self): From 5ad0beb327310e1fc44a2539bae12f8e632890b4 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Wed, 5 Jun 2024 14:28:17 -0700 Subject: [PATCH 14/33] Delete MANIFEST.in and exclude artifacts from sdist and wheel (#1119) * Delete MANIFEST.in * Exclude files from sdist and wheel * Update changelog --- CHANGELOG.md | 6 +++--- MANIFEST.in | 5 ----- pyproject.toml | 15 ++++++++++++++- 3 files changed, 17 insertions(+), 9 deletions(-) delete mode 100644 MANIFEST.in diff --git a/CHANGELOG.md b/CHANGELOG.md index 5ed9984db..4f11fa143 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,12 +1,12 @@ # HDMF Changelog -## HDMF 3.14.1 (June 3, 2024) +## HDMF 3.14.1 (Upcoming) -### Bug Fixes +### Bug fixes +- Excluded unnecessary artifacts from sdist and wheel. @rly [#1119](https://github.com/hdmf-dev/hdmf/pull/1119) - Fixed issue with resolving attribute specs that have the same name at different levels of a spec hierarchy. @rly [#1122](https://github.com/hdmf-dev/hdmf/pull/1122) - ## HDMF 3.14.0 (May 20, 2024) ### Enhancements diff --git a/MANIFEST.in b/MANIFEST.in deleted file mode 100644 index 9b77b2ac8..000000000 --- a/MANIFEST.in +++ /dev/null @@ -1,5 +0,0 @@ -include license.txt Legal.txt src/hdmf/_due.py -include requirements.txt requirements-dev.txt requirements-doc.txt requirements-min.txt requirements-opt.txt -include test_gallery.py tox.ini -graft tests -global-exclude *.py[cod] diff --git a/pyproject.toml b/pyproject.toml index f0a0b7fb7..3a0034087 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -64,10 +64,23 @@ source = "vcs" version-file = "src/hdmf/_version.py" [tool.hatch.build.targets.sdist] -exclude = [".git_archival.txt"] +exclude = [ + ".git*", + ".codecov.yml", + ".readthedocs.yaml", + ".mailmap", + ".pre-commit-config.yaml", +] [tool.hatch.build.targets.wheel] packages = ["src/hdmf"] +exclude = [ + ".git*", + ".codecov.yml", + ".readthedocs.yaml", + ".mailmap", + ".pre-commit-config.yaml", +] # [tool.mypy] # no_incremental = true # needed b/c mypy and ruamel.yaml do not play nice. https://github.com/python/mypy/issues/12664 From 4d131a0d93f26f483484ae776a33a6d018e0d391 Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Wed, 5 Jun 2024 16:11:31 -0700 Subject: [PATCH 15/33] [pre-commit.ci] pre-commit autoupdate (#1120) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Ryan Ly --- .pre-commit-config.yaml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f80639c7..3f4d13e28 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: # hooks: # - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.4 + rev: v0.4.7 hooks: - id: ruff # - repo: https://github.com/econchick/interrogate @@ -26,7 +26,7 @@ repos: # hooks: # - id: interrogate - repo: https://github.com/codespell-project/codespell - rev: v2.2.6 + rev: v2.3.0 hooks: - id: codespell additional_dependencies: From 543935f164d5e8beb77bb4d854b1f1a8b1b4b18b Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Thu, 6 Jun 2024 08:13:47 -0700 Subject: [PATCH 16/33] Update CHANGELOG.md (#1124) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4f11fa143..f022ef0e7 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # HDMF Changelog -## HDMF 3.14.1 (Upcoming) +## HDMF 3.14.1 (June 6, 2024) ### Bug fixes - Excluded unnecessary artifacts from sdist and wheel. @rly [#1119](https://github.com/hdmf-dev/hdmf/pull/1119) From ece2c27d9c828f4ecb5e2ee3b66dd49e547f7b4c Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 10 Jun 2024 22:26:04 -0700 Subject: [PATCH 17/33] [pre-commit.ci] pre-commit autoupdate (#1127) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 3f4d13e28..7684975ab 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: # hooks: # - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.7 + rev: v0.4.8 hooks: - id: ruff # - repo: https://github.com/econchick/interrogate From d756abeb98ef89fde37a71ed64ec84de627f8bfb Mon Sep 17 00:00:00 2001 From: Cody Baker <51133164+CodyCBakerPhD@users.noreply.github.com> Date: Wed, 12 Jun 2024 12:41:51 -0400 Subject: [PATCH 18/33] Fix iterator increment (#1128) --- CHANGELOG.md | 7 +++++++ src/hdmf/data_utils.py | 10 +++++++--- 2 files changed, 14 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index f022ef0e7..ae753b98b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # HDMF Changelog +## HDMF 3.14.2 (???) + +### Bug fixes +- Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128) + + + ## HDMF 3.14.1 (June 6, 2024) ### Bug fixes diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index c03665caa..f4ac6541f 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -386,14 +386,18 @@ def __next__(self): :returns: DataChunk object with the data and selection of the current buffer. :rtype: DataChunk """ - if self.display_progress: - self.progress_bar.update(n=1) try: buffer_selection = next(self.buffer_selection_generator) + + # Only update after successful iteration + if self.display_progress: + self.progress_bar.update(n=1) + return DataChunk(data=self._get_data(selection=buffer_selection), selection=buffer_selection) except StopIteration: + # Allow text to be written to new lines after completion if self.display_progress: - self.progress_bar.write("\n") # Allows text to be written to new lines after completion + self.progress_bar.write("\n") raise StopIteration def __reduce__(self) -> Tuple[Callable, Iterable]: From 7426275cacc769a10ffca89836765df1355ba9db Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 17 Jun 2024 23:40:49 -0400 Subject: [PATCH 19/33] [pre-commit.ci] pre-commit autoupdate (#1131) MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit updates: - [github.com/astral-sh/ruff-pre-commit: v0.4.8 → v0.4.9](https://github.com/astral-sh/ruff-pre-commit/compare/v0.4.8...v0.4.9) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 7684975ab..0f486273b 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: # hooks: # - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.8 + rev: v0.4.9 hooks: - id: ruff # - repo: https://github.com/econchick/interrogate From 539ecf47ad1ad70e23666f7a7d750d2d84535632 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Fri, 28 Jun 2024 16:23:06 -0700 Subject: [PATCH 20/33] Zarr append for datasets (non-reference) (#1136) * Zarr append * Update CHANGELOG.md * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update pyproject.toml * Update validator.py * Update testcase.py * Update objectmapper.py * Update h5tools.py * Update h5tools.py * Update h5tools.py * Update src/hdmf/validate/validator.py Co-authored-by: Steph Prince <40640337+stephprince@users.noreply.github.com> --------- Co-authored-by: Steph Prince <40640337+stephprince@users.noreply.github.com> --- CHANGELOG.md | 5 +++-- pyproject.toml | 8 ++++---- src/hdmf/backends/hdf5/h5tools.py | 2 +- src/hdmf/build/objectmapper.py | 2 +- src/hdmf/data_utils.py | 5 ++++- src/hdmf/testing/testcase.py | 2 +- src/hdmf/validate/validator.py | 2 +- tests/unit/utils_test/test_data_utils.py | 14 ++++++++++++++ 8 files changed, 29 insertions(+), 11 deletions(-) create mode 100644 tests/unit/utils_test/test_data_utils.py diff --git a/CHANGELOG.md b/CHANGELOG.md index ae753b98b..5f5db4918 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,11 +1,12 @@ # HDMF Changelog -## HDMF 3.14.2 (???) +## HDMF 3.14.2 (Upcoming) ### Bug fixes - Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128) - +### Enhancements +- Support appending to zarr arrays. @mavaylon1 [#1136](https://github.com/hdmf-dev/hdmf/pull/1136) ## HDMF 3.14.1 (June 6, 2024) diff --git a/pyproject.toml b/pyproject.toml index 3a0034087..a089113c0 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,12 +36,12 @@ dependencies = [ "pandas>=1.0.5", "ruamel.yaml>=0.16", "scipy>=1.4", + "zarr >= 2.12.0", "importlib-resources; python_version < '3.9'", # TODO: remove when minimum python version is 3.9 ] dynamic = ["version"] [project.optional-dependencies] -zarr = ["zarr>=2.12.0"] tqdm = ["tqdm>=4.41.0"] termset = ["linkml-runtime>=1.5.5; python_version >= '3.9'", "schemasheets>=0.1.23; python_version >= '3.9'", @@ -117,7 +117,7 @@ omit = [ # force-exclude = "src/hdmf/common/hdmf-common-schema|docs/gallery" [tool.ruff] -select = ["E", "F", "T100", "T201", "T203"] +lint.select = ["E", "F", "T100", "T201", "T203"] exclude = [ ".git", ".tox", @@ -132,11 +132,11 @@ exclude = [ ] line-length = 120 -[tool.ruff.per-file-ignores] +[tool.ruff.lint.per-file-ignores] "docs/gallery/*" = ["E402", "T201"] "src/*/__init__.py" = ["F401"] "setup.py" = ["T201"] "test_gallery.py" = ["T201"] -[tool.ruff.mccabe] +[tool.ruff.lint.mccabe] max-complexity = 17 diff --git a/src/hdmf/backends/hdf5/h5tools.py b/src/hdmf/backends/hdf5/h5tools.py index 0604881bb..8135d75e7 100644 --- a/src/hdmf/backends/hdf5/h5tools.py +++ b/src/hdmf/backends/hdf5/h5tools.py @@ -728,7 +728,7 @@ def __read_dataset(self, h5obj, name=None): def _check_str_dtype(self, h5obj): dtype = h5obj.dtype if dtype.kind == 'O': - if dtype.metadata.get('vlen') == str and H5PY_3: + if dtype.metadata.get('vlen') is str and H5PY_3: return StrDataset(h5obj, None) return h5obj diff --git a/src/hdmf/build/objectmapper.py b/src/hdmf/build/objectmapper.py index fed678d41..52fbfec49 100644 --- a/src/hdmf/build/objectmapper.py +++ b/src/hdmf/build/objectmapper.py @@ -1164,7 +1164,7 @@ def __get_subspec_values(self, builder, spec, manager): if not isinstance(builder, DatasetBuilder): # pragma: no cover raise ValueError("__get_subspec_values - must pass DatasetBuilder with DatasetSpec") if (spec.shape is None and getattr(builder.data, 'shape', None) == (1,) and - type(builder.data[0]) != np.void): + type(builder.data[0]) is not np.void): # if a scalar dataset is expected and a 1-element non-compound dataset is given, then read the dataset builder['data'] = builder.data[0] # use dictionary reference instead of .data to bypass error ret[spec] = self.__check_ref_resolver(builder.data) diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index f4ac6541f..71f5bdf6d 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -5,17 +5,20 @@ from warnings import warn from typing import Tuple from itertools import product, chain +from zarr import Array as ZarrArray import h5py import numpy as np from .utils import docval, getargs, popargs, docval_macro, get_data_shape - def append_data(data, arg): if isinstance(data, (list, DataIO)): data.append(arg) return data + elif isinstance(data, ZarrArray): + data.append([arg], axis=0) + return data elif type(data).__name__ == 'TermSetWrapper': # circular import data.append(arg) return data diff --git a/src/hdmf/testing/testcase.py b/src/hdmf/testing/testcase.py index 798df6fe4..1be4bcecd 100644 --- a/src/hdmf/testing/testcase.py +++ b/src/hdmf/testing/testcase.py @@ -174,7 +174,7 @@ def _assert_array_equal(self, :param message: custom additional message to show when assertions as part of this assert are failing """ array_data_types = tuple([i for i in get_docval_macro('array_data') - if (i != list and i != tuple and i != AbstractDataChunkIterator)]) + if (i is not list and i is not tuple and i is not AbstractDataChunkIterator)]) # We construct array_data_types this way to avoid explicit dependency on h5py, Zarr and other # I/O backends. Only list and tuple do not support [()] slicing, and AbstractDataChunkIterator # should never occur here. The effective value of array_data_types is then: diff --git a/src/hdmf/validate/validator.py b/src/hdmf/validate/validator.py index bdfc15f8f..e39011d9f 100644 --- a/src/hdmf/validate/validator.py +++ b/src/hdmf/validate/validator.py @@ -164,7 +164,7 @@ def get_type(data, builder_dtype=None): # Empty array else: # Empty string array - if data.dtype.metadata["vlen"] == str: + if data.dtype.metadata["vlen"] is str: return "utf", None # Undetermined variable length data type. else: # pragma: no cover diff --git a/tests/unit/utils_test/test_data_utils.py b/tests/unit/utils_test/test_data_utils.py new file mode 100644 index 000000000..2e0df7ba8 --- /dev/null +++ b/tests/unit/utils_test/test_data_utils.py @@ -0,0 +1,14 @@ +from hdmf.data_utils import append_data +from hdmf.testing import TestCase + +import numpy as np +from numpy.testing import assert_array_equal +import zarr + +class TestAppendData(TestCase): + + def test_append_data_zarr(self): + zarr_array = zarr.array([1,2,3]) + new = append_data(zarr_array, 4) + + assert_array_equal(new[:], np.array([1,2,3,4])) From 5e13d64efabedb039ed4eb49abd4a9c9e848f72a Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Mon, 1 Jul 2024 12:33:29 -0400 Subject: [PATCH 21/33] Warn when unexpected keys in specs (#1134) * Warn when unexpected keys in specs * Update changelog --- CHANGELOG.md | 7 ++++--- src/hdmf/spec/spec.py | 4 ++++ tests/unit/spec_tests/test_attribute_spec.py | 12 ++++++++++++ tests/unit/spec_tests/test_dataset_spec.py | 12 ++++++++++++ tests/unit/spec_tests/test_group_spec.py | 10 ++++++++++ tests/unit/spec_tests/test_link_spec.py | 12 ++++++++++++ 6 files changed, 54 insertions(+), 3 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 5f5db4918..74e048a19 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,12 +2,13 @@ ## HDMF 3.14.2 (Upcoming) -### Bug fixes -- Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128) - ### Enhancements +- Warn when unexpected keys are present in specs. @rly [#1134](https://github.com/hdmf-dev/hdmf/pull/1134) - Support appending to zarr arrays. @mavaylon1 [#1136](https://github.com/hdmf-dev/hdmf/pull/1136) +### Bug fixes +- Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128) + ## HDMF 3.14.1 (June 6, 2024) ### Bug fixes diff --git a/src/hdmf/spec/spec.py b/src/hdmf/spec/spec.py index 1a6e8d987..6d7d29e49 100644 --- a/src/hdmf/spec/spec.py +++ b/src/hdmf/spec/spec.py @@ -93,9 +93,13 @@ def build_spec(cls, spec_dict): vargs = cls.build_const_args(spec_dict) kwargs = dict() # iterate through the Spec docval and construct kwargs based on matching values in spec_dict + unused_vargs = list(vargs) for x in get_docval(cls.__init__): if x['name'] in vargs: kwargs[x['name']] = vargs.get(x['name']) + unused_vargs.remove(x['name']) + if unused_vargs: + warn(f'Unexpected keys {unused_vargs} in spec {spec_dict}') return cls(**kwargs) diff --git a/tests/unit/spec_tests/test_attribute_spec.py b/tests/unit/spec_tests/test_attribute_spec.py index 15102e728..bac8e12a3 100644 --- a/tests/unit/spec_tests/test_attribute_spec.py +++ b/tests/unit/spec_tests/test_attribute_spec.py @@ -91,3 +91,15 @@ def test_build_spec_no_doc(self): msg = "AttributeSpec.__init__: missing argument 'doc'" with self.assertRaisesWith(TypeError, msg): AttributeSpec.build_spec(spec_dict) + + def test_build_warn_extra_args(self): + spec_dict = { + 'name': 'attribute1', + 'doc': 'test attribute', + 'dtype': 'int', + 'quantity': '?', + } + msg = ("Unexpected keys ['quantity'] in spec {'name': 'attribute1', 'doc': 'test attribute', " + "'dtype': 'int', 'quantity': '?'}") + with self.assertWarnsWith(UserWarning, msg): + AttributeSpec.build_spec(spec_dict) diff --git a/tests/unit/spec_tests/test_dataset_spec.py b/tests/unit/spec_tests/test_dataset_spec.py index 0309aced4..008e8c6fc 100644 --- a/tests/unit/spec_tests/test_dataset_spec.py +++ b/tests/unit/spec_tests/test_dataset_spec.py @@ -245,3 +245,15 @@ def test_data_type_property_value(self): group = GroupSpec('A group', name='group', data_type_inc=data_type_inc, data_type_def=data_type_def) self.assertEqual(group.data_type, data_type) + + def test_build_warn_extra_args(self): + spec_dict = { + 'name': 'dataset1', + 'doc': 'test dataset', + 'dtype': 'int', + 'required': True, + } + msg = ("Unexpected keys ['required'] in spec {'name': 'dataset1', 'doc': 'test dataset', " + "'dtype': 'int', 'required': True}") + with self.assertWarnsWith(UserWarning, msg): + DatasetSpec.build_spec(spec_dict) diff --git a/tests/unit/spec_tests/test_group_spec.py b/tests/unit/spec_tests/test_group_spec.py index 00a937538..31c00cfbb 100644 --- a/tests/unit/spec_tests/test_group_spec.py +++ b/tests/unit/spec_tests/test_group_spec.py @@ -314,6 +314,16 @@ def test_get_namespace_spec(self): expected = AttributeSpec('namespace', 'the namespace for the data type of this object', 'text', required=False) self.assertDictEqual(GroupSpec.get_namespace_spec(), expected) + def test_build_warn_extra_args(self): + spec_dict = { + 'name': 'group1', + 'doc': 'test group', + 'required': True, + } + msg = "Unexpected keys ['required'] in spec {'name': 'group1', 'doc': 'test group', 'required': True}" + with self.assertWarnsWith(UserWarning, msg): + GroupSpec.build_spec(spec_dict) + class TestNotAllowedConfig(TestCase): diff --git a/tests/unit/spec_tests/test_link_spec.py b/tests/unit/spec_tests/test_link_spec.py index e6c680b7c..38e10886b 100644 --- a/tests/unit/spec_tests/test_link_spec.py +++ b/tests/unit/spec_tests/test_link_spec.py @@ -67,3 +67,15 @@ def test_required_is_many(self): ) self.assertEqual(spec.required, req) self.assertEqual(spec.is_many(), many) + + def test_build_warn_extra_args(self): + spec_dict = { + 'name': 'link1', + 'doc': 'test link', + 'target_type': 'TestType', + 'required': True, + } + msg = ("Unexpected keys ['required'] in spec {'name': 'link1', 'doc': 'test link', " + "'target_type': 'TestType', 'required': True}") + with self.assertWarnsWith(UserWarning, msg): + LinkSpec.build_spec(spec_dict) From 8917eaffb7a10bb97233e484b78c801cfb57161e Mon Sep 17 00:00:00 2001 From: "dependabot[bot]" <49699333+dependabot[bot]@users.noreply.github.com> Date: Mon, 1 Jul 2024 16:53:00 +0000 Subject: [PATCH 22/33] Bump actions/add-to-project from 0.6.1 to 1.0.1 (#1097) Co-authored-by: dependabot[bot] <49699333+dependabot[bot]@users.noreply.github.com> Co-authored-by: Ryan Ly --- .github/workflows/project_action.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.github/workflows/project_action.yml b/.github/workflows/project_action.yml index 5d141d1d1..0f0d8f3ce 100644 --- a/.github/workflows/project_action.yml +++ b/.github/workflows/project_action.yml @@ -20,7 +20,7 @@ jobs: - name: Add to Developer Board env: TOKEN: ${{ steps.generate_token.outputs.token }} - uses: actions/add-to-project@v0.6.1 + uses: actions/add-to-project@v1.0.1 with: project-url: https://github.com/orgs/hdmf-dev/projects/7 github-token: ${{ env.TOKEN }} @@ -28,7 +28,7 @@ jobs: - name: Add to Community Board env: TOKEN: ${{ steps.generate_token.outputs.token }} - uses: actions/add-to-project@v0.6.1 + uses: actions/add-to-project@v1.0.1 with: project-url: https://github.com/orgs/hdmf-dev/projects/8 github-token: ${{ env.TOKEN }} From 62edbe44892d10d199393eae3f6c7645a2689ea4 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Fri, 5 Jul 2024 12:40:28 -0700 Subject: [PATCH 23/33] Make Zarr optional for testing (#1141) --- pyproject.toml | 1 - src/hdmf/data_utils.py | 13 ++++++++---- tests/unit/utils_test/test_data_utils.py | 25 +++++++++++++++++++++++- 3 files changed, 33 insertions(+), 6 deletions(-) diff --git a/pyproject.toml b/pyproject.toml index a089113c0..6ef9850a6 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -36,7 +36,6 @@ dependencies = [ "pandas>=1.0.5", "ruamel.yaml>=0.16", "scipy>=1.4", - "zarr >= 2.12.0", "importlib-resources; python_version < '3.9'", # TODO: remove when minimum python version is 3.9 ] dynamic = ["version"] diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index 71f5bdf6d..798a40973 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -5,7 +5,12 @@ from warnings import warn from typing import Tuple from itertools import product, chain -from zarr import Array as ZarrArray + +try: + from zarr import Array as ZarrArray + ZARR_INSTALLED = True +except ImportError: + ZARR_INSTALLED = False import h5py import numpy as np @@ -16,9 +21,6 @@ def append_data(data, arg): if isinstance(data, (list, DataIO)): data.append(arg) return data - elif isinstance(data, ZarrArray): - data.append([arg], axis=0) - return data elif type(data).__name__ == 'TermSetWrapper': # circular import data.append(arg) return data @@ -33,6 +35,9 @@ def append_data(data, arg): data.resize(shape) data[-1] = arg return data + elif ZARR_INSTALLED and isinstance(data, ZarrArray): + data.append([arg], axis=0) + return data else: msg = "Data cannot append to object of type '%s'" % type(data) raise ValueError(msg) diff --git a/tests/unit/utils_test/test_data_utils.py b/tests/unit/utils_test/test_data_utils.py index 2e0df7ba8..b5a5e50e7 100644 --- a/tests/unit/utils_test/test_data_utils.py +++ b/tests/unit/utils_test/test_data_utils.py @@ -3,9 +3,32 @@ import numpy as np from numpy.testing import assert_array_equal -import zarr + +try: + import zarr + ZARR_INSTALLED = True +except ImportError: + ZARR_INSTALLED = False + + +class MyIterable: + def __init__(self, data): + self.data = data + class TestAppendData(TestCase): + def test_append_exception(self): + data = MyIterable([1, 2, 3, 4, 5]) + with self.assertRaises(ValueError): + append_data(data, 4) + + +class TestZarrAppendData(TestCase): + + def setUp(self): + if not ZARR_INSTALLED: + self.skipTest("optional Zarr package is not installed") + def test_append_data_zarr(self): zarr_array = zarr.array([1,2,3]) From eb67626c1c251a9c64458f77a6d893a14209fc88 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Sun, 7 Jul 2024 19:09:57 -0400 Subject: [PATCH 24/33] Add support for numpy 2, update all deps (#1139) --- CHANGELOG.md | 1 + environment-ros3.yml | 12 ++++++------ pyproject.toml | 2 +- requirements-dev.txt | 17 +++++++++-------- requirements-opt.txt | 8 ++++---- requirements.txt | 12 +++++++----- src/hdmf/build/objectmapper.py | 2 +- src/hdmf/common/table.py | 2 +- tests/unit/test_io_hdf5.py | 4 ++-- tests/unit/utils_test/test_docval.py | 11 +++++++++-- 10 files changed, 41 insertions(+), 30 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74e048a19..74d7bd477 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Enhancements - Warn when unexpected keys are present in specs. @rly [#1134](https://github.com/hdmf-dev/hdmf/pull/1134) - Support appending to zarr arrays. @mavaylon1 [#1136](https://github.com/hdmf-dev/hdmf/pull/1136) +- Add support for numpy 2. @rly [#1139](https://github.com/hdmf-dev/hdmf/pull/1139) ### Bug fixes - Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128) diff --git a/environment-ros3.yml b/environment-ros3.yml index 458b899ba..34c37cc01 100644 --- a/environment-ros3.yml +++ b/environment-ros3.yml @@ -5,11 +5,11 @@ channels: - defaults dependencies: - python==3.12 - - h5py==3.10.0 - - matplotlib==3.8.0 - - numpy==1.26.0 - - pandas==2.1.2 + - h5py==3.11.0 + - matplotlib==3.8.4 + - numpy==2.0.0 + - pandas==2.2.2 - python-dateutil==2.8.2 - - pytest==7.4.3 - - pytest-cov==4.1.0 + - pytest==8.1.2 # regression introduced in pytest 8.2.*, will be fixed in 8.3.0 + - pytest-cov==5.0.0 - setuptools diff --git a/pyproject.toml b/pyproject.toml index 6ef9850a6..86e52a137 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -32,7 +32,7 @@ classifiers = [ dependencies = [ "h5py>=2.10", "jsonschema>=2.6.0", - 'numpy>=1.18, <2.0', # pin below 2.0 until HDMF supports numpy 2.0 + 'numpy>=1.18', "pandas>=1.0.5", "ruamel.yaml>=0.16", "scipy>=1.4", diff --git a/requirements-dev.txt b/requirements-dev.txt index 1d856e4e7..95cf0797e 100644 --- a/requirements-dev.txt +++ b/requirements-dev.txt @@ -2,12 +2,13 @@ # compute coverage, and create test environments. note that depending on the version of python installed, different # versions of requirements may be installed due to package incompatibilities. # -black==24.3.0 -codespell==2.2.6 -coverage==7.3.2 -pre-commit==3.5.0 -pytest==7.4.3 -pytest-cov==4.1.0 +black==24.4.2 +codespell==2.3.0 +coverage==7.5.4 +pre-commit==3.7.1; python_version >= "3.9" +pre-commit==3.5.0; python_version < "3.9" +pytest==8.1.2 # regression introduced in pytest 8.2.*, will be fixed in 8.3.0 +pytest-cov==5.0.0 python-dateutil==2.8.2 -ruff==0.1.3 -tox==4.11.3 +ruff==0.5.0 +tox==4.15.1 diff --git a/requirements-opt.txt b/requirements-opt.txt index c1d34220b..4831d1949 100644 --- a/requirements-opt.txt +++ b/requirements-opt.txt @@ -1,6 +1,6 @@ # pinned dependencies that are optional. used to reproduce an entire development environment to use HDMF -tqdm==4.66.3 -zarr==2.17.1 -linkml-runtime==1.7.4; python_version >= "3.9" +tqdm==4.66.4 +zarr==2.18.2 +linkml-runtime==1.7.7; python_version >= "3.9" schemasheets==0.2.1; python_version >= "3.9" -oaklib==0.5.32; python_version >= "3.9" +oaklib==0.6.10; python_version >= "3.9" diff --git a/requirements.txt b/requirements.txt index 5182d5c2e..30a596ada 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,8 +1,10 @@ # pinned dependencies to reproduce an entire development environment to use HDMF -h5py==3.10.0 +h5py==3.11.0 importlib-resources==6.1.0; python_version < "3.9" # TODO: remove when minimum python version is 3.9 -jsonschema==4.19.1 -numpy==1.26.1 -pandas==2.1.2 +jsonschema==4.22.0 +numpy==1.26.4 # TODO: numpy 2.0.0 is supported by hdmf but incompatible with pandas and scipy +pandas==2.2.2; python_version >= "3.9" +pandas==2.1.2; python_version < "3.8" # TODO: remove when minimum python version is 3.9 ruamel.yaml==0.18.2 -scipy==1.11.3 +scipy==1.14.0; python_version >= "3.10" +scipy==1.11.3; python_version < "3.10" diff --git a/src/hdmf/build/objectmapper.py b/src/hdmf/build/objectmapper.py index 52fbfec49..b0bd7d594 100644 --- a/src/hdmf/build/objectmapper.py +++ b/src/hdmf/build/objectmapper.py @@ -299,7 +299,7 @@ def __check_edgecases(cls, spec, value, spec_dtype): # noqa: C901 cls.__check_convert_numeric(value.dtype.type) if np.issubdtype(value.dtype, np.str_): ret_dtype = 'utf8' - elif np.issubdtype(value.dtype, np.string_): + elif np.issubdtype(value.dtype, np.bytes_): ret_dtype = 'ascii' elif np.issubdtype(value.dtype, np.dtype('O')): # Only variable-length strings should ever appear as generic objects. diff --git a/src/hdmf/common/table.py b/src/hdmf/common/table.py index 3b67ff19d..2e90b0cdf 100644 --- a/src/hdmf/common/table.py +++ b/src/hdmf/common/table.py @@ -235,7 +235,7 @@ def __eq__(self, other): if isinstance(search_ids, int): search_ids = [search_ids] # Find all matching locations - return np.in1d(self.data, search_ids).nonzero()[0] + return np.isin(self.data, search_ids).nonzero()[0] def _validate_new_data(self, data): # NOTE this may not cover all the many AbstractDataChunkIterator edge cases diff --git a/tests/unit/test_io_hdf5.py b/tests/unit/test_io_hdf5.py index 0dae1fbbe..29b7f2d7f 100644 --- a/tests/unit/test_io_hdf5.py +++ b/tests/unit/test_io_hdf5.py @@ -121,10 +121,10 @@ def __assert_helper(self, a, b): # if strings, convert before comparing if b_array: if b_sub.dtype.char in ('S', 'U'): - a_sub = [np.string_(s) for s in a_sub] + a_sub = [np.bytes_(s) for s in a_sub] else: if a_sub.dtype.char in ('S', 'U'): - b_sub = [np.string_(s) for s in b_sub] + b_sub = [np.bytes_(s) for s in b_sub] equal = np.array_equal(a_sub, b_sub) else: equal = a_sub == b_sub diff --git a/tests/unit/utils_test/test_docval.py b/tests/unit/utils_test/test_docval.py index 154a5c4b0..c766dcf46 100644 --- a/tests/unit/utils_test/test_docval.py +++ b/tests/unit/utils_test/test_docval.py @@ -736,8 +736,12 @@ def method(self, **kwargs): self.assertEqual(method(self, np.uint(1)), np.uint(1)) self.assertEqual(method(self, np.uint(2)), np.uint(2)) + # the string rep of uint changes from numpy 1 to 2 ("1" to "np.uint64(1)"), so do not hardcode the string + uint_str1 = np.uint(1).__repr__() + uint_str2 = np.uint(2).__repr__() + msg = ("TestDocValidator.test_enum_uint..method: " - "forbidden value for 'arg1' (got 3, expected (1, 2))") + "forbidden value for 'arg1' (got 3, expected (%s, %s))" % (uint_str1, uint_str2)) with self.assertRaisesWith(ValueError, msg): method(self, np.uint(3)) @@ -767,8 +771,11 @@ def method(self, **kwargs): self.assertEqual(method(self, 'true'), 'true') self.assertEqual(method(self, np.uint(1)), np.uint(1)) + # the string rep of uint changes from numpy 1 to 2 ("1" to "np.uint64(1)"), so do not hardcode the string + uint_str = np.uint(1).__repr__() + msg = ("TestDocValidator.test_enum_bool_mixed..method: " - "forbidden value for 'arg1' (got 0, expected (True, 1, 1.0, 'true', 1))") + "forbidden value for 'arg1' (got 0, expected (True, 1, 1.0, 'true', %s))" % uint_str) with self.assertRaisesWith(ValueError, msg): method(self, 0) From 639d0caa1951b63f271542a97473d50e53a5faaf Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Mon, 8 Jul 2024 00:00:46 -0400 Subject: [PATCH 25/33] Allow "value" in DatasetSpec (#1143) --- CHANGELOG.md | 1 + src/hdmf/spec/spec.py | 11 ++++++++++- tests/unit/spec_tests/test_dataset_spec.py | 4 ++++ 3 files changed, 15 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 74d7bd477..3edd7c36d 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -5,6 +5,7 @@ ### Enhancements - Warn when unexpected keys are present in specs. @rly [#1134](https://github.com/hdmf-dev/hdmf/pull/1134) - Support appending to zarr arrays. @mavaylon1 [#1136](https://github.com/hdmf-dev/hdmf/pull/1136) +- Support specifying "value" key in DatasetSpec. @rly [#1143](https://github.com/hdmf-dev/hdmf/pull/1143) - Add support for numpy 2. @rly [#1139](https://github.com/hdmf-dev/hdmf/pull/1139) ### Bug fixes diff --git a/src/hdmf/spec/spec.py b/src/hdmf/spec/spec.py index 6d7d29e49..358cc3256 100644 --- a/src/hdmf/spec/spec.py +++ b/src/hdmf/spec/spec.py @@ -648,6 +648,7 @@ def build_const_args(cls, spec_dict): {'name': 'linkable', 'type': bool, 'doc': 'whether or not this group can be linked', 'default': True}, {'name': 'quantity', 'type': (str, int), 'doc': 'the required number of allowed instance', 'default': 1}, {'name': 'default_value', 'type': None, 'doc': 'a default value for this dataset', 'default': None}, + {'name': 'value', 'type': None, 'doc': 'a fixed value for this dataset', 'default': None}, {'name': 'data_type_def', 'type': str, 'doc': 'the data type this specification represents', 'default': None}, {'name': 'data_type_inc', 'type': (str, 'DatasetSpec'), 'doc': 'the data type this specification extends', 'default': None}, @@ -662,7 +663,8 @@ class DatasetSpec(BaseStorageSpec): @docval(*_dataset_args) def __init__(self, **kwargs): - doc, shape, dims, dtype, default_value = popargs('doc', 'shape', 'dims', 'dtype', 'default_value', kwargs) + doc, shape, dims, dtype = popargs('doc', 'shape', 'dims', 'dtype', kwargs) + default_value, value = popargs('default_value', 'value', kwargs) if shape is not None: self['shape'] = shape if dims is not None: @@ -685,6 +687,8 @@ def __init__(self, **kwargs): super().__init__(doc, **kwargs) if default_value is not None: self['default_value'] = default_value + if value is not None: + self['value'] = value if self.name is not None: valid_quant_vals = [1, 'zero_or_one', ZERO_OR_ONE] if self.quantity not in valid_quant_vals: @@ -762,6 +766,11 @@ def default_value(self): '''The default value of the dataset or None if not specified''' return self.get('default_value', None) + @property + def value(self): + '''The fixed value of the dataset or None if not specified''' + return self.get('value', None) + @classmethod def dtype_spec_cls(cls): ''' The class to use when constructing DtypeSpec objects diff --git a/tests/unit/spec_tests/test_dataset_spec.py b/tests/unit/spec_tests/test_dataset_spec.py index 008e8c6fc..c9db14635 100644 --- a/tests/unit/spec_tests/test_dataset_spec.py +++ b/tests/unit/spec_tests/test_dataset_spec.py @@ -246,6 +246,10 @@ def test_data_type_property_value(self): data_type_inc=data_type_inc, data_type_def=data_type_def) self.assertEqual(group.data_type, data_type) + def test_constructor_value(self): + spec = DatasetSpec(doc='my first dataset', dtype='int', name='dataset1', value=42) + assert spec.value == 42 + def test_build_warn_extra_args(self): spec_dict = { 'name': 'dataset1', From dfb1df79bce3e34c52af7996429c3f561d96390a Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Mon, 8 Jul 2024 08:32:07 -0700 Subject: [PATCH 26/33] Update CHANGELOG.md (#1145) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 3edd7c36d..0624a5f3a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # HDMF Changelog -## HDMF 3.14.2 (Upcoming) +## HDMF 3.14.2 (July 7, 2024) ### Enhancements - Warn when unexpected keys are present in specs. @rly [#1134](https://github.com/hdmf-dev/hdmf/pull/1134) From 77be0cc6d23a8ddfa3b96120f9d7599a5614044a Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Fri, 26 Jul 2024 10:25:31 -0700 Subject: [PATCH 27/33] Add doc to H5DataIO.dataset setter (#1154) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> Co-authored-by: Oliver Ruebel --- src/hdmf/backends/hdf5/h5_utils.py | 21 +++++++++++++++++++++ 1 file changed, 21 insertions(+) diff --git a/src/hdmf/backends/hdf5/h5_utils.py b/src/hdmf/backends/hdf5/h5_utils.py index 8654e2b4b..aa68272c9 100644 --- a/src/hdmf/backends/hdf5/h5_utils.py +++ b/src/hdmf/backends/hdf5/h5_utils.py @@ -546,10 +546,31 @@ def __init__(self, **kwargs): @property def dataset(self): + """Get the cached h5py.Dataset.""" return self.__dataset @dataset.setter def dataset(self, val): + """Cache the h5py.Dataset written with the stored IO settings. + + This attribute can be used to cache a written, empty dataset and fill it in later. + This allows users to access the handle to the dataset *without* having to close + and reopen a file. + + For example:: + + dataio = H5DataIO(shape=(5,), dtype=int) + foo = Foo('foo1', dataio, "I am foo1", 17, 3.14) + bucket = FooBucket('bucket1', [foo]) + foofile = FooFile(buckets=[bucket]) + + io = HDF5IO(self.path, manager=self.manager, mode='w') + # write the object to disk, including initializing an empty int dataset with shape (5,) + io.write(foofile) + + foo.my_data.dataset[:] = [0, 1, 2, 3, 4] + io.close() + """ if self.__dataset is not None: raise ValueError("Cannot overwrite H5DataIO.dataset") self.__dataset = val From 4c32820f8cf8adb3fbad27b8553ec2842548e571 Mon Sep 17 00:00:00 2001 From: Ryan Ly Date: Fri, 26 Jul 2024 14:42:30 -0700 Subject: [PATCH 28/33] Write dimension labels to DatasetBuilder on build (#1081) Co-authored-by: Matthew Avaylon --- CHANGELOG.md | 7 + src/hdmf/build/builders.py | 16 +- src/hdmf/build/objectmapper.py | 99 +++++- src/hdmf/build/warnings.py | 7 + .../build_tests/mapper_tests/test_build.py | 286 +++++++++++++++++- 5 files changed, 404 insertions(+), 11 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 0624a5f3a..bb80f5336 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,12 @@ # HDMF Changelog +## HDMF 3.14.3 (Upcoming) + +### Enhancements +- Added new attribute "dimension_labels" on `DatasetBuilder` which specifies the names of the dimensions used in the +dataset based on the shape of the dataset data and the dimension names in the spec for the data type. This attribute +is available on build (during the write process), but not on read of a dataset from a file. @rly [#1081](https://github.com/hdmf-dev/hdmf/pull/1081) + ## HDMF 3.14.2 (July 7, 2024) ### Enhancements diff --git a/src/hdmf/build/builders.py b/src/hdmf/build/builders.py index 73c683bbd..cb658b6d4 100644 --- a/src/hdmf/build/builders.py +++ b/src/hdmf/build/builders.py @@ -330,6 +330,10 @@ class DatasetBuilder(BaseBuilder): 'doc': 'The datatype of this dataset.', 'default': None}, {'name': 'attributes', 'type': dict, 'doc': 'A dictionary of attributes to create in this dataset.', 'default': dict()}, + {'name': 'dimension_labels', 'type': tuple, + 'doc': ('A list of labels for each dimension of this dataset from the spec. Currently this is ' + 'supplied only on build.'), + 'default': None}, {'name': 'maxshape', 'type': (int, tuple), 'doc': 'The shape of this dataset. Use None for scalars.', 'default': None}, {'name': 'chunks', 'type': bool, 'doc': 'Whether or not to chunk this dataset.', 'default': False}, @@ -337,11 +341,14 @@ class DatasetBuilder(BaseBuilder): {'name': 'source', 'type': str, 'doc': 'The source of the data in this builder.', 'default': None}) def __init__(self, **kwargs): """ Create a Builder object for a dataset """ - name, data, dtype, attributes, maxshape, chunks, parent, source = getargs( - 'name', 'data', 'dtype', 'attributes', 'maxshape', 'chunks', 'parent', 'source', kwargs) + name, data, dtype, attributes, dimension_labels, maxshape, chunks, parent, source = getargs( + 'name', 'data', 'dtype', 'attributes', 'dimension_labels', 'maxshape', 'chunks', 'parent', 'source', + kwargs + ) super().__init__(name, attributes, parent, source) self['data'] = data self['attributes'] = _copy.copy(attributes) + self.__dimension_labels = dimension_labels self.__chunks = chunks self.__maxshape = maxshape if isinstance(data, BaseBuilder): @@ -361,6 +368,11 @@ def data(self, val): raise AttributeError("Cannot overwrite data.") self['data'] = val + @property + def dimension_labels(self): + """Labels for each dimension of this dataset from the spec.""" + return self.__dimension_labels + @property def chunks(self): """Whether or not this dataset is chunked.""" diff --git a/src/hdmf/build/objectmapper.py b/src/hdmf/build/objectmapper.py index b0bd7d594..b5815ee2c 100644 --- a/src/hdmf/build/objectmapper.py +++ b/src/hdmf/build/objectmapper.py @@ -10,14 +10,15 @@ from .errors import (BuildError, OrphanContainerBuildError, ReferenceTargetNotBuiltError, ContainerConfigurationError, ConstructError) from .manager import Proxy, BuildManager -from .warnings import MissingRequiredBuildWarning, DtypeConversionWarning, IncorrectQuantityBuildWarning +from .warnings import (MissingRequiredBuildWarning, DtypeConversionWarning, IncorrectQuantityBuildWarning, + IncorrectDatasetShapeBuildWarning) from ..container import AbstractContainer, Data, DataRegion from ..term_set import TermSetWrapper from ..data_utils import DataIO, AbstractDataChunkIterator from ..query import ReferenceResolver from ..spec import Spec, AttributeSpec, DatasetSpec, GroupSpec, LinkSpec, RefSpec from ..spec.spec import BaseStorageSpec -from ..utils import docval, getargs, ExtenderMeta, get_docval +from ..utils import docval, getargs, ExtenderMeta, get_docval, get_data_shape _const_arg = '__constructor_arg' @@ -721,19 +722,34 @@ def build(self, **kwargs): if not isinstance(container, Data): msg = "'container' must be of type Data with DatasetSpec" raise ValueError(msg) - spec_dtype, spec_shape, spec = self.__check_dset_spec(self.spec, spec_ext) + spec_dtype, spec_shape, spec_dims, spec = self.__check_dset_spec(self.spec, spec_ext) + dimension_labels = self.__get_dimension_labels_from_spec(container.data, spec_shape, spec_dims) if isinstance(spec_dtype, RefSpec): self.logger.debug("Building %s '%s' as a dataset of references (source: %s)" % (container.__class__.__name__, container.name, repr(source))) # create dataset builder with data=None as a placeholder. fill in with refs later - builder = DatasetBuilder(name, data=None, parent=parent, source=source, dtype=spec_dtype.reftype) + builder = DatasetBuilder( + name, + data=None, + parent=parent, + source=source, + dtype=spec_dtype.reftype, + dimension_labels=dimension_labels, + ) manager.queue_ref(self.__set_dataset_to_refs(builder, spec_dtype, spec_shape, container, manager)) elif isinstance(spec_dtype, list): # a compound dataset self.logger.debug("Building %s '%s' as a dataset of compound dtypes (source: %s)" % (container.__class__.__name__, container.name, repr(source))) # create dataset builder with data=None, dtype=None as a placeholder. fill in with refs later - builder = DatasetBuilder(name, data=None, parent=parent, source=source, dtype=spec_dtype) + builder = DatasetBuilder( + name, + data=None, + parent=parent, + source=source, + dtype=spec_dtype, + dimension_labels=dimension_labels, + ) manager.queue_ref(self.__set_compound_dataset_to_refs(builder, spec, spec_dtype, container, manager)) else: @@ -744,7 +760,14 @@ def build(self, **kwargs): % (container.__class__.__name__, container.name, repr(source))) # an unspecified dtype and we were given references # create dataset builder with data=None as a placeholder. fill in with refs later - builder = DatasetBuilder(name, data=None, parent=parent, source=source, dtype='object') + builder = DatasetBuilder( + name, + data=None, + parent=parent, + source=source, + dtype="object", + dimension_labels=dimension_labels, + ) manager.queue_ref(self.__set_untyped_dataset_to_refs(builder, container, manager)) else: # a dataset that has no references, pass the conversion off to the convert_dtype method @@ -760,7 +783,14 @@ def build(self, **kwargs): except Exception as ex: msg = 'could not resolve dtype for %s \'%s\'' % (type(container).__name__, container.name) raise Exception(msg) from ex - builder = DatasetBuilder(name, bldr_data, parent=parent, source=source, dtype=dtype) + builder = DatasetBuilder( + name, + data=bldr_data, + parent=parent, + source=source, + dtype=dtype, + dimension_labels=dimension_labels, + ) # Add attributes from the specification extension to the list of attributes all_attrs = self.__spec.attributes + getattr(spec_ext, 'attributes', tuple()) @@ -779,14 +809,67 @@ def __check_dset_spec(self, orig, ext): """ dtype = orig.dtype shape = orig.shape + dims = orig.dims spec = orig if ext is not None: if ext.dtype is not None: dtype = ext.dtype if ext.shape is not None: shape = ext.shape + dims = ext.dims spec = ext - return dtype, shape, spec + return dtype, shape, dims, spec + + def __get_dimension_labels_from_spec(self, data, spec_shape, spec_dims) -> tuple: + if spec_shape is None or spec_dims is None: + return None + data_shape = get_data_shape(data) + # if shape is a list of allowed shapes, find the index of the shape that matches the data + if isinstance(spec_shape[0], list): + match_shape_inds = list() + for i, s in enumerate(spec_shape): + # skip this shape if it has a different number of dimensions from the data + if len(s) != len(data_shape): + continue + # check each dimension. None means any length is allowed + match = True + for j, d in enumerate(data_shape): + if s[j] is not None and s[j] != d: + match = False + break + if match: + match_shape_inds.append(i) + # use the most specific match -- the one with the fewest Nones + if match_shape_inds: + if len(match_shape_inds) == 1: + return tuple(spec_dims[match_shape_inds[0]]) + else: + count_nones = [len([x for x in spec_shape[k] if x is None]) for k in match_shape_inds] + index_min_count = count_nones.index(min(count_nones)) + best_match_ind = match_shape_inds[index_min_count] + return tuple(spec_dims[best_match_ind]) + else: + # no matches found + msg = "Shape of data does not match any allowed shapes in spec '%s'" % self.spec.path + warnings.warn(msg, IncorrectDatasetShapeBuildWarning) + return None + else: + if len(data_shape) != len(spec_shape): + msg = "Shape of data does not match shape in spec '%s'" % self.spec.path + warnings.warn(msg, IncorrectDatasetShapeBuildWarning) + return None + # check each dimension. None means any length is allowed + match = True + for j, d in enumerate(data_shape): + if spec_shape[j] is not None and spec_shape[j] != d: + match = False + break + if not match: + msg = "Shape of data does not match shape in spec '%s'" % self.spec.path + warnings.warn(msg, IncorrectDatasetShapeBuildWarning) + return None + # shape is a single list of allowed dimension lengths + return tuple(spec_dims) def __is_reftype(self, data): if (isinstance(data, AbstractDataChunkIterator) or diff --git a/src/hdmf/build/warnings.py b/src/hdmf/build/warnings.py index 3d5f02126..6a6ea6986 100644 --- a/src/hdmf/build/warnings.py +++ b/src/hdmf/build/warnings.py @@ -15,6 +15,13 @@ class IncorrectQuantityBuildWarning(BuildWarning): pass +class IncorrectDatasetShapeBuildWarning(BuildWarning): + """ + Raised when a dataset has a shape that is not allowed by the spec. + """ + pass + + class MissingRequiredBuildWarning(BuildWarning): """ Raised when a required field is missing. diff --git a/tests/unit/build_tests/mapper_tests/test_build.py b/tests/unit/build_tests/mapper_tests/test_build.py index b90ad6f1a..28cc9518e 100644 --- a/tests/unit/build_tests/mapper_tests/test_build.py +++ b/tests/unit/build_tests/mapper_tests/test_build.py @@ -4,7 +4,7 @@ from hdmf import Container, Data, TermSet, TermSetWrapper from hdmf.common import VectorData, get_type_map from hdmf.build import ObjectMapper, BuildManager, TypeMap, GroupBuilder, DatasetBuilder -from hdmf.build.warnings import DtypeConversionWarning +from hdmf.build.warnings import DtypeConversionWarning, IncorrectDatasetShapeBuildWarning from hdmf.spec import GroupSpec, AttributeSpec, DatasetSpec, SpecCatalog, SpecNamespace, NamespaceCatalog, Spec from hdmf.testing import TestCase from hdmf.utils import docval, getargs @@ -650,3 +650,287 @@ def test_build_incorrect_dtype(self): msg = "could not resolve dtype for BarData 'my_bar'" with self.assertRaisesWith(Exception, msg): self.manager.build(bar_data_holder_inst, source='test.h5') + + +class BuildDatasetShapeMixin(TestCase, metaclass=ABCMeta): + + def setUp(self): + self.set_up_specs() + spec_catalog = SpecCatalog() + spec_catalog.register_spec(self.bar_data_spec, 'test.yaml') + spec_catalog.register_spec(self.bar_data_holder_spec, 'test.yaml') + namespace = SpecNamespace( + doc='a test namespace', + name=CORE_NAMESPACE, + schema=[{'source': 'test.yaml'}], + version='0.1.0', + catalog=spec_catalog + ) + namespace_catalog = NamespaceCatalog() + namespace_catalog.add_namespace(CORE_NAMESPACE, namespace) + type_map = TypeMap(namespace_catalog) + type_map.register_container_type(CORE_NAMESPACE, 'BarData', BarData) + type_map.register_container_type(CORE_NAMESPACE, 'BarDataHolder', BarDataHolder) + type_map.register_map(BarData, ExtBarDataMapper) + type_map.register_map(BarDataHolder, ObjectMapper) + self.manager = BuildManager(type_map) + + def set_up_specs(self): + shape, dims = self.get_base_shape_dims() + self.bar_data_spec = DatasetSpec( + doc='A test dataset specification with a data type', + data_type_def='BarData', + dtype='int', + shape=shape, + dims=dims, + ) + self.bar_data_holder_spec = GroupSpec( + doc='A container of multiple extended BarData objects', + data_type_def='BarDataHolder', + datasets=[self.get_dataset_inc_spec()], + ) + + @abstractmethod + def get_base_shape_dims(self): + pass + + @abstractmethod + def get_dataset_inc_spec(self): + pass + + +class TestBuildDatasetOneOptionBadShapeUnspecified1(BuildDatasetShapeMixin): + """Test dataset spec shape = 2D any length, data = 1D. Should raise warning and set dimension_labels to None.""" + + def get_base_shape_dims(self): + return [None, None], ['a', 'b'] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[1, 2, 3], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + msg = "Shape of data does not match shape in spec 'BarData'" + with self.assertWarnsWith(IncorrectDatasetShapeBuildWarning, msg): + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels is None + + +class TestBuildDatasetOneOptionBadShapeUnspecified2(BuildDatasetShapeMixin): + """Test dataset spec shape = (any, 2), data = (3, 1). Should raise warning and set dimension_labels to None.""" + + def get_base_shape_dims(self): + return [None, 2], ['a', 'b'] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1], [2], [3]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + msg = "Shape of data does not match shape in spec 'BarData'" + with self.assertWarnsWith(IncorrectDatasetShapeBuildWarning, msg): + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels is None + + +class TestBuildDatasetTwoOptionsBadShapeUnspecified(BuildDatasetShapeMixin): + """Test dataset spec shape = (any, 2) or (any, 3), data = (3, 1). + Should raise warning and set dimension_labels to None. + """ + + def get_base_shape_dims(self): + return [[None, 2], [None, 3]], [['a', 'b1'], ['a', 'b2']] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1], [2], [3]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + msg = "Shape of data does not match any allowed shapes in spec 'BarData'" + with self.assertWarnsWith(IncorrectDatasetShapeBuildWarning, msg): + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels is None + + +class TestBuildDatasetDimensionLabelsUnspecified(BuildDatasetShapeMixin): + + def get_base_shape_dims(self): + return None, None + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1, 2, 3], [4, 5, 6]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels is None + + +class TestBuildDatasetDimensionLabelsOneOption(BuildDatasetShapeMixin): + + def get_base_shape_dims(self): + return [None, None], ['a', 'b'] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1, 2, 3], [4, 5, 6]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels == ('a', 'b') + + +class TestBuildDatasetDimensionLabelsTwoOptionsOneMatch(BuildDatasetShapeMixin): + + def get_base_shape_dims(self): + return [[None], [None, None]], [['a'], ['a', 'b']] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1, 2, 3], [4, 5, 6]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels == ('a', 'b') + + +class TestBuildDatasetDimensionLabelsTwoOptionsTwoMatches(BuildDatasetShapeMixin): + + def get_base_shape_dims(self): + return [[None, None], [None, 3]], [['a', 'b1'], ['a', 'b2']] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1, 2, 3], [4, 5, 6]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels == ('a', 'b2') + + +class TestBuildDatasetDimensionLabelsOneOptionRefined(BuildDatasetShapeMixin): + + def get_base_shape_dims(self): + return [None, None], ['a', 'b1'] + + def get_dataset_inc_spec(self): + dataset_inc_spec = DatasetSpec( + doc='A BarData', + data_type_inc='BarData', + quantity='*', + shape=[None, 3], + dims=['a', 'b2'], + ) + return dataset_inc_spec + + def test_build(self): + """ + Test build of BarDataHolder which contains a BarData. + """ + # NOTE: attr1 doesn't map to anything but is required in the test container class + bar_data_inst = BarData(name='my_bar', data=[[1, 2, 3], [4, 5, 6]], attr1='a string') + bar_data_holder_inst = BarDataHolder( + name='my_bar_holder', + bar_datas=[bar_data_inst], + ) + + builder = self.manager.build(bar_data_holder_inst, source='test.h5') + assert builder.datasets['my_bar'].dimension_labels == ('a', 'b2') From 50aad2fe4a3e34f4d7b941e7f8aa668ad0672535 Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Mon, 29 Jul 2024 08:59:29 -0700 Subject: [PATCH 29/33] Update CHANGELOG.md (#1156) --- CHANGELOG.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index bb80f5336..4d77fbdad 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,6 +1,6 @@ # HDMF Changelog -## HDMF 3.14.3 (Upcoming) +## HDMF 3.14.3 (July 29, 2024) ### Enhancements - Added new attribute "dimension_labels" on `DatasetBuilder` which specifies the names of the dimensions used in the From a98e5e98cbfb6e824fb57eb5e7eac1dc6cd4532b Mon Sep 17 00:00:00 2001 From: Matthew Avaylon Date: Wed, 31 Jul 2024 10:08:55 -0700 Subject: [PATCH 30/33] Minimum Support for Appending References in Hdmf-Zarr (#1157) * Minimum Support * Update CHANGELOG.md --- CHANGELOG.md | 5 +++++ src/hdmf/data_utils.py | 3 ++- 2 files changed, 7 insertions(+), 1 deletion(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4d77fbdad..4a6369094 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,10 @@ # HDMF Changelog +## HDMF 3.14.4 (Upcoming) + +### Enhancements +- Added support to append to a dataset of references for HDMF-Zarr. @mavaylon1 [#1157](https://github.com/hdmf-dev/hdmf/pull/1157) + ## HDMF 3.14.3 (July 29, 2024) ### Enhancements diff --git a/src/hdmf/data_utils.py b/src/hdmf/data_utils.py index 798a40973..91400da84 100644 --- a/src/hdmf/data_utils.py +++ b/src/hdmf/data_utils.py @@ -18,7 +18,8 @@ from .utils import docval, getargs, popargs, docval_macro, get_data_shape def append_data(data, arg): - if isinstance(data, (list, DataIO)): + from hdmf.backends.hdf5.h5_utils import HDMFDataset + if isinstance(data, (list, DataIO, HDMFDataset)): data.append(arg) return data elif type(data).__name__ == 'TermSetWrapper': # circular import From 3cc7db39d3e7f8e21fbaaa2282c14cebd758d20e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Fri, 9 Aug 2024 15:01:11 -0700 Subject: [PATCH 31/33] [pre-commit.ci] pre-commit autoupdate (#1133) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 0f486273b..9011c9dc3 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: # hooks: # - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.4.9 + rev: v0.5.6 hooks: - id: ruff # - repo: https://github.com/econchick/interrogate From 49a60df21016fb4da83d64e84e7bc99dac03e94e Mon Sep 17 00:00:00 2001 From: "pre-commit-ci[bot]" <66853113+pre-commit-ci[bot]@users.noreply.github.com> Date: Mon, 12 Aug 2024 17:54:11 -0700 Subject: [PATCH 32/33] [pre-commit.ci] pre-commit autoupdate (#1169) Co-authored-by: pre-commit-ci[bot] <66853113+pre-commit-ci[bot]@users.noreply.github.com> --- .pre-commit-config.yaml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml index 9011c9dc3..0ad399734 100644 --- a/.pre-commit-config.yaml +++ b/.pre-commit-config.yaml @@ -18,7 +18,7 @@ repos: # hooks: # - id: black - repo: https://github.com/astral-sh/ruff-pre-commit - rev: v0.5.6 + rev: v0.5.7 hooks: - id: ruff # - repo: https://github.com/econchick/interrogate From b0f068ee956870ce640dabf118260a97b18b1564 Mon Sep 17 00:00:00 2001 From: Jeremy Magland Date: Mon, 19 Aug 2024 00:34:38 -0400 Subject: [PATCH 33/33] speed up loading of namespaces: skip register type when already registered when loading namespace (#1102) * skip register type when already registered when loading namespace * update CHANGELOG.md * Update CHANGELOG.md * Update CHANGELOG.md * Update namespace.py --------- Co-authored-by: Matthew Avaylon --- CHANGELOG.md | 1 + src/hdmf/spec/namespace.py | 20 ++++++++++++-------- 2 files changed, 13 insertions(+), 8 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 4a6369094..c83a7ac3b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -11,6 +11,7 @@ - Added new attribute "dimension_labels" on `DatasetBuilder` which specifies the names of the dimensions used in the dataset based on the shape of the dataset data and the dimension names in the spec for the data type. This attribute is available on build (during the write process), but not on read of a dataset from a file. @rly [#1081](https://github.com/hdmf-dev/hdmf/pull/1081) +- Speed up loading namespaces by skipping register_type when already registered. @magland [#1102](https://github.com/hdmf-dev/hdmf/pull/1102) ## HDMF 3.14.2 (July 7, 2024) diff --git a/src/hdmf/spec/namespace.py b/src/hdmf/spec/namespace.py index a2ae0bd37..f0417175f 100644 --- a/src/hdmf/spec/namespace.py +++ b/src/hdmf/spec/namespace.py @@ -466,15 +466,19 @@ def __load_namespace(self, namespace, reader, resolve=True): return included_types def __register_type(self, ndt, inc_ns, catalog, registered_types): - spec = inc_ns.get_spec(ndt) - spec_file = inc_ns.catalog.get_spec_source_file(ndt) - self.__register_dependent_types(spec, inc_ns, catalog, registered_types) - if isinstance(spec, DatasetSpec): - built_spec = self.dataset_spec_cls.build_spec(spec) + if ndt in registered_types: + # already registered + pass else: - built_spec = self.group_spec_cls.build_spec(spec) - registered_types.add(ndt) - catalog.register_spec(built_spec, spec_file) + spec = inc_ns.get_spec(ndt) + spec_file = inc_ns.catalog.get_spec_source_file(ndt) + self.__register_dependent_types(spec, inc_ns, catalog, registered_types) + if isinstance(spec, DatasetSpec): + built_spec = self.dataset_spec_cls.build_spec(spec) + else: + built_spec = self.group_spec_cls.build_spec(spec) + registered_types.add(ndt) + catalog.register_spec(built_spec, spec_file) def __register_dependent_types(self, spec, inc_ns, catalog, registered_types): """Ensure that classes for all types used by this type are registered