Merge branch 'dev' into patch-2

hdmf-dev · Aug 19, 2024 · da2c1d3 · da2c1d3
2 parents 0ccde91 + b0f068e
commit da2c1d3
Show file tree

Hide file tree

Showing 46 changed files with 1,225 additions and 2,361 deletions.
diff --git a/.github/workflows/project_action.yml b/.github/workflows/project_action.yml
@@ -20,15 +20,15 @@ jobs:
       - name: Add to Developer Board
         env:
           TOKEN: ${{ steps.generate_token.outputs.token }}
-        uses: actions/add-to-project@v0.6.1
+        uses: actions/add-to-project@v1.0.1
         with:
           project-url: https://github.com/orgs/hdmf-dev/projects/7
           github-token: ${{ env.TOKEN }}
 
       - name: Add to Community Board
         env:
           TOKEN: ${{ steps.generate_token.outputs.token }}
-        uses: actions/add-to-project@v0.6.1
+        uses: actions/add-to-project@v1.0.1
         with:
           project-url: https://github.com/orgs/hdmf-dev/projects/8
           github-token: ${{ env.TOKEN }}
diff --git a/.github/workflows/run_all_tests.yml b/.github/workflows/run_all_tests.yml
@@ -41,8 +41,8 @@ jobs:
           - { name: windows-python3.12            , test-tox-env: pytest-py312-pinned          , python-ver: "3.12", os: windows-latest }
           - { name: windows-python3.12-upgraded   , test-tox-env: pytest-py312-upgraded        , python-ver: "3.12", os: windows-latest }
           - { name: windows-python3.12-prerelease , test-tox-env: pytest-py312-prerelease      , python-ver: "3.12", os: windows-latest }
-          - { name: macos-python3.8-minimum       , test-tox-env: pytest-py38-minimum          , python-ver: "3.8" , os: macos-latest }
-          - { name: macos-python3.9               , test-tox-env: pytest-py39-pinned           , python-ver: "3.9" , os: macos-latest }
+          - { name: macos-python3.8-minimum       , test-tox-env: pytest-py38-minimum          , python-ver: "3.8" , os: macos-13 }
+          - { name: macos-python3.9               , test-tox-env: pytest-py39-pinned           , python-ver: "3.9" , os: macos-13 }
           - { name: macos-python3.10              , test-tox-env: pytest-py310-pinned          , python-ver: "3.10", os: macos-latest }
           - { name: macos-python3.11              , test-tox-env: pytest-py311-pinned          , python-ver: "3.11", os: macos-latest }
           - { name: macos-python3.11-optional     , test-tox-env: pytest-py311-optional-pinned , python-ver: "3.11", os: macos-latest }
@@ -105,7 +105,7 @@ jobs:
           - { name: windows-gallery-python3.11-optional  , test-tox-env: gallery-py311-optional-pinned , python-ver: "3.11", os: windows-latest }
           - { name: windows-gallery-python3.12-upgraded  , test-tox-env: gallery-py312-upgraded        , python-ver: "3.12", os: windows-latest }
           - { name: windows-gallery-python3.12-prerelease, test-tox-env: gallery-py312-prerelease      , python-ver: "3.12", os: windows-latest }
-          - { name: macos-gallery-python3.8-minimum      , test-tox-env: gallery-py38-minimum          , python-ver: "3.8" , os: macos-latest }
+          - { name: macos-gallery-python3.8-minimum      , test-tox-env: gallery-py38-minimum          , python-ver: "3.8" , os: macos-13 }
           - { name: macos-gallery-python3.11-optional    , test-tox-env: gallery-py311-optional-pinned , python-ver: "3.11", os: macos-latest }
           - { name: macos-gallery-python3.12-upgraded    , test-tox-env: gallery-py312-upgraded        , python-ver: "3.12", os: macos-latest }
           - { name: macos-gallery-python3.12-prerelease  , test-tox-env: gallery-py312-prerelease      , python-ver: "3.12", os: macos-latest }
@@ -197,7 +197,7 @@ jobs:
         run: |
           tox -e wheelinstall --installpkg dist/*.tar.gz
 
-  run-gallery-ros3-tests:
+  run-ros3-tests:
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
     defaults:
@@ -210,9 +210,9 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-gallery-python3.12-ros3   , python-ver: "3.12", os: ubuntu-latest }
-          - { name: windows-gallery-python3.12-ros3 , python-ver: "3.12", os: windows-latest }
-          - { name: macos-gallery-python3.12-ros3   , python-ver: "3.12", os: macos-latest }
+          - { name: linux-python3.12-ros3   , python-ver: "3.12", os: ubuntu-latest }
+          - { name: windows-python3.12-ros3 , python-ver: "3.12", os: windows-latest }
+          - { name: macos-python3.12-ros3   , python-ver: "3.12", os: macos-latest }
     steps:
       - name: Checkout repo with submodules
         uses: actions/checkout@v4
@@ -233,7 +233,7 @@ jobs:
 
       - name: Install run dependencies
         run: |
-          pip install -e .
+          pip install .
           pip list
 
       - name: Conda reporting

diff --git a/.github/workflows/run_coverage.yml b/.github/workflows/run_coverage.yml
@@ -55,18 +55,73 @@ jobs:
 
       - name: Install package
         run: |
-          python -m pip install -e .  # must install in editable mode for coverage to find sources
+          python -m pip install .
           python -m pip list
 
       - name: Run tests and generate coverage report
         run: |
-          pytest --cov
-          python -m coverage xml  # codecov uploader requires xml format
-          python -m coverage report -m
+          # coverage is configured in pyproject.toml
+          pytest --cov --cov-report=xml --cov-report=term  # codecov uploader requires xml format
 
       - name: Upload coverage to Codecov
         uses: codecov/codecov-action@v4
         with:
           fail_ci_if_error: true
+          file: ./coverage.xml
+        env:
+          CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
+
+  run-ros3-coverage:
+    name: ${{ matrix.name }}
+    runs-on: ${{ matrix.os }}
+    defaults:
+      run:
+        shell: bash -l {0}  # necessary for conda
+    concurrency:
+      group: ${{ github.workflow }}-${{ github.ref }}-${{ matrix.name }}
+      cancel-in-progress: true
+    strategy:
+      fail-fast: false
+      matrix:
+        include:
+          - { name: linux-python3.12-ros3  , python-ver: "3.12", os: ubuntu-latest }
+    steps:
+      - name: Checkout repo with submodules
+        uses: actions/checkout@v4
+        with:
+          submodules: 'recursive'
+          fetch-depth: 0  # tags are required to determine the version
+
+      - name: Set up Conda
+        uses: conda-incubator/setup-miniconda@v3
+        with:
+          auto-update-conda: true
+          activate-environment: ros3
+          environment-file: environment-ros3.yml
+          python-version: ${{ matrix.python-ver }}
+          channels: conda-forge
+          auto-activate-base: false
+          mamba-version: "*"
+
+      - name: Install run dependencies
+        run: |
+          pip install .
+          pip list
+
+      - name: Conda reporting
+        run: |
+          conda info
+          conda config --show-sources
+          conda list --show-channel-urls
+
+      - name: Run ros3 tests  # TODO include gallery tests after they are written
+        run: |
+          pytest --cov --cov-report=xml --cov-report=term tests/unit/test_io_hdf5_streaming.py
+
+      - name: Upload coverage to Codecov
+        uses: codecov/codecov-action@v4
+        with:
+          fail_ci_if_error: true
+          file: ./coverage.xml
         env:
           CODECOV_TOKEN: ${{ secrets.CODECOV_TOKEN }}
diff --git a/.github/workflows/run_hdmf_zarr_tests.yml b/.github/workflows/run_hdmf_zarr_tests.yml
@@ -32,8 +32,7 @@ jobs:
           git clone https://github.com/hdmf-dev/hdmf-zarr.git --recurse-submodules
           cd hdmf-zarr
           python -m pip install -r requirements-dev.txt  # do not install the pinned install requirements
-          # must install in editable mode for coverage to find sources
-          python -m pip install -e .  # this will install a different version of hdmf from the current one
+          python -m pip install .  # this will install a different version of hdmf from the current one
           cd ..
           python -m pip uninstall -y hdmf  # uninstall the other version of hdmf
           python -m pip install .  # reinstall current branch of hdmf

diff --git a/.github/workflows/run_pynwb_tests.yml b/.github/workflows/run_pynwb_tests.yml
@@ -32,8 +32,7 @@ jobs:
           git clone https://github.com/NeurodataWithoutBorders/pynwb.git --recurse-submodules
           cd pynwb
           python -m pip install -r requirements-dev.txt  # do not install the pinned install requirements
-          # must install in editable mode for coverage to find sources
-          python -m pip install -e .  # this will install a different version of hdmf from the current one
+          python -m pip install .  # this will install a different version of hdmf from the current one
           cd ..
           python -m pip uninstall -y hdmf  # uninstall the other version of hdmf
           python -m pip install .  # reinstall current branch of hdmf

diff --git a/.github/workflows/run_tests.yml b/.github/workflows/run_tests.yml
@@ -28,7 +28,7 @@ jobs:
           - { name: linux-python3.12-upgraded    , test-tox-env: pytest-py312-upgraded  , python-ver: "3.12", os: ubuntu-latest , upload-wheels: true }
           - { name: windows-python3.8-minimum    , test-tox-env: pytest-py38-minimum    , python-ver: "3.8" , os: windows-latest }
           - { name: windows-python3.12-upgraded  , test-tox-env: pytest-py312-upgraded  , python-ver: "3.12", os: windows-latest }
-          - { name: macos-python3.8-minimum      , test-tox-env: pytest-py38-minimum    , python-ver: "3.8" , os: macos-latest }
+          - { name: macos-python3.8-minimum      , test-tox-env: pytest-py38-minimum    , python-ver: "3.8" , os: macos-13 }
           - { name: macos-python3.12-upgraded    , test-tox-env: pytest-py312-upgraded  , python-ver: "3.12", os: macos-latest }
     steps:
       - name: Checkout repo with submodules
@@ -209,7 +209,7 @@ jobs:
               --token ${{ secrets.BOT_GITHUB_TOKEN }} \
               --re-upload
 
-  run-gallery-ros3-tests:
+  run-ros3-tests:
     name: ${{ matrix.name }}
     runs-on: ${{ matrix.os }}
     defaults:
@@ -222,7 +222,7 @@ jobs:
       fail-fast: false
       matrix:
         include:
-          - { name: linux-gallery-python3.12-ros3  , python-ver: "3.12", os: ubuntu-latest }
+          - { name: linux-python3.12-ros3  , python-ver: "3.12", os: ubuntu-latest }
     steps:
       - name: Checkout repo with submodules
         uses: actions/checkout@v4
@@ -243,7 +243,7 @@ jobs:
 
       - name: Install run dependencies
         run: |
-          pip install -e .
+          pip install .
           pip list
 
       - name: Conda reporting

diff --git a/.gitignore b/.gitignore
@@ -12,6 +12,8 @@
 /docs/source/hdmf*.rst
 /docs/gallery/*.hdf5
 /docs/gallery/*.sqlite
+/docs/gallery/expanded_example_dynamic_term_set.yaml
+/docs/gallery/schemasheets/nwb_static_enums.yaml
 
 # Auto-generated files after running tutorials
 mylab.*.yaml

diff --git a/.pre-commit-config.yaml b/.pre-commit-config.yaml
@@ -1,7 +1,7 @@
 # NOTE: run `pre-commit autoupdate` to update hooks to latest version
 repos:
 -   repo: https://github.com/pre-commit/pre-commit-hooks
-    rev: v4.5.0
+    rev: v4.6.0
     hooks:
     -   id: check-yaml
     -   id: end-of-file-fixer
@@ -18,15 +18,15 @@ repos:
 #     hooks:
 #     -   id: black
 -   repo: https://github.com/astral-sh/ruff-pre-commit
-    rev: v0.3.3
+    rev: v0.5.7
     hooks:
     -   id: ruff
 # -   repo: https://github.com/econchick/interrogate
 #     rev: 1.5.0
 #     hooks:
 #     -   id: interrogate
 -   repo: https://github.com/codespell-project/codespell
-    rev: v2.2.6
+    rev: v2.3.0
     hooks:
     -   id: codespell
         additional_dependencies:

diff --git a/CHANGELOG.md b/CHANGELOG.md
@@ -1,12 +1,51 @@
 # HDMF Changelog
 
-## HDMF 3.14.0 (Upcoming)
+## HDMF 3.14.4 (Upcoming)
+
+### Enhancements
+- Added support to append to a dataset of references for HDMF-Zarr. @mavaylon1 [#1157](https://github.com/hdmf-dev/hdmf/pull/1157)
+
+## HDMF 3.14.3 (July 29, 2024)
+
+### Enhancements
+- Added new attribute "dimension_labels" on `DatasetBuilder` which specifies the names of the dimensions used in the
+dataset based on the shape of the dataset data and the dimension names in the spec for the data type. This attribute
+is available on build (during the write process), but not on read of a dataset from a file. @rly [#1081](https://github.com/hdmf-dev/hdmf/pull/1081)
+- Speed up loading namespaces by skipping register_type when already registered. @magland [#1102](https://github.com/hdmf-dev/hdmf/pull/1102)
+- Speed up namespace loading: return a shallow copy rather than a deep copy in build_const_args. @magland [#1103](https://github.com/hdmf-dev/hdmf/pull/1103)
+
+## HDMF 3.14.2 (July 7, 2024)
+
+### Enhancements
+- Warn when unexpected keys are present in specs. @rly [#1134](https://github.com/hdmf-dev/hdmf/pull/1134)
+- Support appending to zarr arrays. @mavaylon1 [#1136](https://github.com/hdmf-dev/hdmf/pull/1136)
+- Support specifying "value" key in DatasetSpec. @rly [#1143](https://github.com/hdmf-dev/hdmf/pull/1143)
+- Add support for numpy 2. @rly [#1139](https://github.com/hdmf-dev/hdmf/pull/1139)
+
+### Bug fixes
+- Fix iterator increment causing an extra +1 added after the end of completion. @CodyCBakerPhD [#1128](https://github.com/hdmf-dev/hdmf/pull/1128)
+
+## HDMF 3.14.1 (June 6, 2024)
+
+### Bug fixes
+- Excluded unnecessary artifacts from sdist and wheel. @rly [#1119](https://github.com/hdmf-dev/hdmf/pull/1119)
+- Fixed issue with resolving attribute specs that have the same name at different levels of a spec hierarchy.
+  @rly [#1122](https://github.com/hdmf-dev/hdmf/pull/1122)
+
+## HDMF 3.14.0 (May 20, 2024)
 
 ### Enhancements
 - Updated `_field_config` to take `type_map` as an argument for APIs. @mavaylon1 [#1094](https://github.com/hdmf-dev/hdmf/pull/1094)
 - Added `TypeConfigurator` to automatically wrap fields with `TermSetWrapper` according to a configuration file. @mavaylon1 [#1016](https://github.com/hdmf-dev/hdmf/pull/1016)
 - Updated `TermSetWrapper` to support validating a single field within a compound array. @mavaylon1 [#1061](https://github.com/hdmf-dev/hdmf/pull/1061)
-- Speed up namespace loading: return a shallow copy rather than a deep copy in build_const_args.
+- Updated testing to not install in editable mode and not run `coverage` by default. @rly [#1107](https://github.com/hdmf-dev/hdmf/pull/1107)
+- Add `post_init_method` parameter when generating classes to perform post-init functionality, i.e., validation. @mavaylon1 [#1089](https://github.com/hdmf-dev/hdmf/pull/1089)
+- Exposed `aws_region` to `HDF5IO` and downstream passes to `h5py.File`. @codycbakerphd [#1040](https://github.com/hdmf-dev/hdmf/pull/1040)
+- Exposed `progress_bar_class` to the `GenericDataChunkIterator` for more custom control over display of progress while iterating. @codycbakerphd [#1110](https://github.com/hdmf-dev/hdmf/pull/1110)
+- Updated loading, unloading, and getting the `TypeConfigurator` to support a `TypeMap` parameter. @mavaylon1 [#1117](https://github.com/hdmf-dev/hdmf/pull/1117)
+
+### Bug Fixes
+- Fixed `TermSetWrapper` warning raised during the setters. @mavaylon1 [#1116](https://github.com/hdmf-dev/hdmf/pull/1116)
 
 ## HDMF 3.13.0 (March 20, 2024)
 
@@ -541,7 +580,7 @@ the fields (i.e., when the constructor sets some fields to fixed values). @rly
   Each sub-table is itself a DynamicTable that is aligned with the main table by row index. Each subtable
   defines a sub-category in the main table effectively creating a table with sub-headings to organize columns.
   @oruebel (#551)
-- Add tutoral for new `AlignedDynamicTable` type. @oruebel (#571)
+- Add tutorial for new `AlignedDynamicTable` type. @oruebel (#571)
 - Equality check for `DynamicTable` now also checks that the name and description of the table are the same. @rly (#566)
 
 ### Internal improvements

diff --git a/MANIFEST.in b/MANIFEST.in