From cf04f4c9b4ab62204df0982273f13dffde7863c8 Mon Sep 17 00:00:00 2001 From: "Adam J. Stewart" Date: Sat, 18 Jun 2022 08:00:45 -0700 Subject: [PATCH] Test minimum version of deps (#574) --- .github/dependabot.yml | 4 +- .github/workflows/release.yaml | 4 +- .github/workflows/style.yaml | 10 +-- .github/workflows/tests.yaml | 27 ++++++- .readthedocs.yaml | 2 +- environment.yml | 35 ++++----- pyproject.toml | 20 ++++- requirements/requirements-min.txt | 54 +++++++++++++ .../requirements.txt | 2 - setup.cfg | 78 ++++++++++--------- tests/datasets/test_advance.py | 2 +- tests/datasets/test_idtrees.py | 2 +- tests/datasets/test_openbuildings.py | 2 +- tests/datasets/test_so2sat.py | 2 +- tests/datasets/test_usavars.py | 2 +- tests/datasets/test_zuericrop.py | 2 +- tests/trainers/test_classification.py | 2 +- tests/transforms/test_transforms.py | 3 - torchgeo/datasets/advance.py | 2 +- torchgeo/datasets/cyclone.py | 2 +- torchgeo/datasets/seco.py | 2 +- 21 files changed, 171 insertions(+), 88 deletions(-) create mode 100644 requirements/requirements-min.txt rename requirements.txt => requirements/requirements.txt (97%) diff --git a/.github/dependabot.yml b/.github/dependabot.yml index 9371d0aa7d5..7ee5f4a5296 100644 --- a/.github/dependabot.yml +++ b/.github/dependabot.yml @@ -1,13 +1,13 @@ version: 2 updates: - package-ecosystem: "pip" - directory: "/" + directory: "requirements" schedule: interval: "daily" # Allow up to 2 open pull requests at a time open-pull-requests-limit: 2 ignore: - # tensorboard requires protobuf < 4 + # torch, tensorboard require protobuf < 4 - dependency-name: "protobuf" # segmentation-models-pytorch requires older timm, can't update - dependency-name: "timm" diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 887cec05a87..35cf9e7208b 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -36,7 +36,7 @@ jobs: python-version: '3.10' - name: Install pip dependencies run: | - pip install -r requirements.txt + pip install -r requirements/requirements.txt pip list - name: Run integration checks run: pytest -m slow @@ -52,7 +52,7 @@ jobs: python-version: '3.10' - name: Install pip dependencies run: | - pip install -r requirements.txt + pip install -r requirements/requirements.txt pip list - name: Run notebook checks env: diff --git a/.github/workflows/style.yaml b/.github/workflows/style.yaml index 462618c0ac4..55ce1eddac1 100644 --- a/.github/workflows/style.yaml +++ b/.github/workflows/style.yaml @@ -21,7 +21,7 @@ jobs: python-version: '3.10' - name: Install pip dependencies run: | - pip install -r requirements.txt + pip install -r requirements/requirements.txt pip list - name: Run black checks run: black . --check --diff @@ -37,7 +37,7 @@ jobs: python-version: '3.10' - name: Install pip dependencies run: | - pip install -r requirements.txt + pip install -r requirements/requirements.txt pip list - name: Run flake8 checks run: flake8 @@ -53,7 +53,7 @@ jobs: python-version: '3.10' - name: Install pip dependencies run: | - pip install -r requirements.txt + pip install -r requirements/requirements.txt pip list - name: Run isort checks run: isort . --check --diff @@ -69,7 +69,7 @@ jobs: python-version: '3.10' - name: Install pip dependencies run: | - pip install -r requirements.txt + pip install -r requirements/requirements.txt pip list - name: Run pydocstyle checks run: pydocstyle @@ -85,7 +85,7 @@ jobs: python-version: '3.10' - name: Install pip dependencies run: | - pip install -r requirements.txt + pip install -r requirements/requirements.txt pip list - name: Run pyupgrade checks run: pyupgrade --py37-plus $(find . -name "*.py") diff --git a/.github/workflows/tests.yaml b/.github/workflows/tests.yaml index 06a150ae2e4..f860891ce81 100644 --- a/.github/workflows/tests.yaml +++ b/.github/workflows/tests.yaml @@ -21,7 +21,7 @@ jobs: python-version: '3.10' - name: Install pip dependencies run: | - pip install -r requirements.txt + pip install -r requirements/requirements.txt pip list - name: Run mypy checks run: mypy . @@ -62,7 +62,30 @@ jobs: if: ${{ runner.os == 'Windows' }} - name: Install pip dependencies run: | - pip install -r requirements.txt + pip install -r requirements/requirements.txt + pip list + - name: Run pytest checks + run: pytest --cov=torchgeo --cov-report=xml + - name: Report coverage + uses: codecov/codecov-action@v2 + minimum: + name: minimum + runs-on: ubuntu-latest + env: + MPLBACKEND: Agg + steps: + - name: Clone repo + uses: actions/checkout@v2 + - name: Set up python + uses: actions/setup-python@v2 + with: + python-version: '3.7' + - name: Install apt dependencies (Linux) + run: sudo apt-get install libgeos-dev libhdf5-dev unrar + - name: Install pip dependencies + run: | + pip install cython numpy==1.17.2 # needed by pycocotools + pip install -r requirements/requirements-min.txt pip list - name: Run pytest checks run: pytest --cov=torchgeo --cov-report=xml diff --git a/.readthedocs.yaml b/.readthedocs.yaml index 0dfacbe3268..1fb10c61455 100644 --- a/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -13,7 +13,7 @@ build: # Configuration of the Python environment to be used python: install: - - requirements: requirements.txt + - requirements: requirements/requirements.txt - requirements: docs/requirements.txt - method: pip path: . diff --git a/environment.yml b/environment.yml index 1343021c7b9..d3880694ed8 100644 --- a/environment.yml +++ b/environment.yml @@ -4,48 +4,45 @@ channels: - conda-forge - open3d-admin # for open3d dependencies: - - einops - - fiona>=1.5 - - h5py - - numpy + - einops>=0.3 + - fiona>=1.8 + - h5py>=2.6 + - numpy>=1.17.2 - open3d>=0.11.2 - pip - - protobuf<4 - - pycocotools + - pycocotools>=2 - pyproj>=2.2 - python>=3.7 - - pytorch>=1.7 + - pytorch>=1.9 - rarfile>=3 - - rasterio>=1.0.16 + - rasterio>=1.0.20 - shapely>=1.3 - torchvision>=0.10 - pip: - - black>=21.4b0 + - black[jupyter]>=21.8 - flake8>=3.8 - - ipywidgets>=7 - isort[colors]>=5.8 - - jupyterlab - kornia>=0.6.4 - - laspy>=2.0.0 + - laspy>=2 - mypy>=0.900 - nbmake>=0.1 - nbsphinx>=0.8.5 - omegaconf>=2.1 - - opencv-python + - opencv-python>=3.4.2.17 - pandas>=0.23.2 - - pillow>=2.9 + - pillow>=6.2 - pydocstyle[toml]>=6.1 - - pytest>=6 + - pytest>=6.1.2 - pytest-cov>=2.4 - - pytorch-lightning>=1.3 + - pytorch-lightning>=1.5.1 - git+https://github.com/pytorch/pytorch_sphinx_theme - radiant-mlhub>=0.2.1 - rtree>=1 - - scikit-learn>=0.18 - - scipy>=0.9 + - scikit-learn>=0.21 + - scipy>=1.2 - segmentation-models-pytorch>=0.2 - setuptools>=42 - sphinx>=4 - - timm>=0.2.1 + - timm>=0.4.12 - torchmetrics>=0.7 - zipfile-deflate64>=0.2 diff --git a/pyproject.toml b/pyproject.toml index 021f0280643..e1f51d6f2be 100644 --- a/pyproject.toml +++ b/pyproject.toml @@ -1,7 +1,7 @@ [build-system] requires = [ + # setuptools 42+ required for metadata.license_files support in setup.cfg "setuptools>=42,<63", - "wheel<0.38", ] build-backend = "setuptools.build_meta" @@ -58,18 +58,30 @@ filterwarnings = [ "error", # Warnings raised by dependencies of dependencies, out of our control + # https://github.com/Cadene/pretrained-models.pytorch/issues/221 + "ignore:.* is deprecated and will be removed in Pillow 10:DeprecationWarning:pretrainedmodels.datasets.utils", # https://github.com/pytorch/vision/pull/5898 + "ignore:.* is deprecated and will be removed in Pillow 10:DeprecationWarning:torchvision.transforms.functional_pil", # https://github.com/rwightman/pytorch-image-models/pull/1256 - "ignore:.*is deprecated and will be removed in Pillow 10:DeprecationWarning", + "ignore:.* is deprecated and will be removed in Pillow 10:DeprecationWarning:timm.data", # https://github.com/pytorch/pytorch/issues/72906 # https://github.com/pytorch/pytorch/pull/69823 - "ignore:distutils Version classes are deprecated. Use packaging.version instead.:DeprecationWarning", + "ignore:distutils Version classes are deprecated. Use packaging.version instead:DeprecationWarning:torch.utils.tensorboard", # https://github.com/PyTorchLightning/pytorch-lightning/issues/13256 - "ignore:torch.distributed._sharded_tensor will be deprecated, use torch.distributed._shard.sharded_tensor instead:DeprecationWarning", + # https://github.com/PyTorchLightning/pytorch-lightning/pull/13261 + "ignore:torch.distributed._sharded_tensor will be deprecated:DeprecationWarning:torch.distributed._sharded_tensor", + # https://github.com/rasterio/rasterio/issues/1742 + # https://github.com/rasterio/rasterio/pull/1753 + "ignore:Using or importing the ABCs from 'collections' instead of from 'collections.abc' is deprecated:DeprecationWarning:rasterio.crs", + # https://github.com/pytorch/pytorch/issues/60053 + # https://github.com/pytorch/pytorch/pull/60059 + "ignore:Named tensors and all their associated APIs are an experimental feature and subject to change:UserWarning:torch.nn.functional", # Expected warnings # pytorch-lightning warns us about using num_workers=0, but it's faster on macOS "ignore:The dataloader, .*, does not have many workers which may be a bottleneck:UserWarning", + # pytorch-lightning warns us about using the CPU when a GPU is available + "ignore:GPU available but not used.:UserWarning", # Unexpected warnings, worth investigating # pytorch-lightning is having trouble inferring the batch size for ChesapeakeCVPRDataModule and CycloneDataModule for some reason diff --git a/requirements/requirements-min.txt b/requirements/requirements-min.txt new file mode 100644 index 00000000000..c93c5509b45 --- /dev/null +++ b/requirements/requirements-min.txt @@ -0,0 +1,54 @@ +# This file is not managed by dependabot. It documents the minimum version of our +# dependencies that is supported. + +# setup +setuptools==42.0.0 + +# install +einops==0.3.0 +fiona==1.8.0 +kornia==0.6.4 +matplotlib==3.3.0 +numpy==1.17.2 +omegaconf==2.1.0 +pillow==6.2.0 +pyproj==2.2.0 +pytorch-lightning==1.5.1 +rasterio==1.0.20 +rtree==1.0.0 +scikit-learn==0.21.0 +segmentation-models-pytorch==0.2.0 +shapely==1.3.0 +timm==0.4.12 +torch==1.9.0 +torchmetrics==0.7.0 +torchvision==0.10.0 + +# datasets +h5py==2.6.0 +laspy==2.0.0 +open3d==0.11.2 +opencv-python==3.4.2.17 +pandas==0.23.2 +pycocotools==2.0.0 +radiant-mlhub==0.2.1 +rarfile==3.0 +scipy==1.2.0 +zipfile-deflate64==0.2.0 + +# docs +nbsphinx==0.8.5 +sphinx==4.0.0 + +# style +black[jupyter]==21.8b0 +flake8==3.8.0 +isort[colors]==5.8.0 +pydocstyle[toml]==6.1.0 +pyupgrade==1.24.0 + +# tests +mypy==0.900 +nbmake==0.1 +pytest==6.1.2 +pytest-cov==2.4.0 diff --git a/requirements.txt b/requirements/requirements.txt similarity index 97% rename from requirements.txt rename to requirements/requirements.txt index a895444ff38..06117d187c2 100644 --- a/requirements.txt +++ b/requirements/requirements.txt @@ -4,7 +4,6 @@ # setup setuptools==62.4.0 -wheel==0.37.1 # install einops==0.4.1 @@ -47,7 +46,6 @@ scipy==1.7.3;python_version=='3.7' zipfile-deflate64==0.2.0 # docs -ipywidgets==7.7.0 nbsphinx==0.8.9 sphinx==5.0.2 diff --git a/setup.cfg b/setup.cfg index c06d264973b..08472a5635c 100644 --- a/setup.cfg +++ b/setup.cfg @@ -25,37 +25,38 @@ keywords = pytorch, deep learning, machine learning, remote sensing, satellite i [options] install_requires = - einops<0.5 - # fiona 1.5+ required for fiona.transform module - fiona~=1.5 + # einops 0.3+ required for einops.repeat + einops>=0.3,<0.5 + # fiona 1.8+ required for reading empty files + fiona>=1.8,<2 # kornia 0.6.4+ required for kornia.contrib.compute_padding kornia>=0.6.4,<0.7 - matplotlib<4 - numpy<2 + # matplotlib 3.3+ required for (H, W, 1) image support in plt.imshow + matplotlib>=3.3,<4 + # numpy 1.17.2+ required by pytorch-lightning + numpy>=1.17.2,<2 # omegaconf 2.1+ required for to_object method - omegaconf~=2.1 - # pillow 2.9+ required for height attribute - pillow>=2.9,<10 - # Required to address protocolbuffers/protobuf#10051 - protobuf<4 + omegaconf>=2.1,<3 + # pillow 6.2+ required by matplotlib + pillow>=6.2,<10 # pyproj 2.2+ required for CRS object pyproj>=2.2,<4 - # pytorch-lightning 1.3+ required for gradient_clip_algorithm argument to Trainer - pytorch-lightning~=1.3 - # rasterio 1.0.16+ required for CRS support - rasterio>=1.0.16,<2 - # rtree 1+ required for len(Index) - rtree>=1 - # scikit-learn 0.18+ required for sklearn.model_selection module - scikit-learn>=0.18,<2 + # pytorch-lightning 1.5.1+ required for apply_to_collection bugfix + pytorch-lightning>=1.5.1,<2 + # rasterio 1.0.20+ required for out_dtype parameter of DatasetReaderBase.read + rasterio>=1.0.20,<2 + # rtree 1+ required for len(index), index & index, index | index + rtree>=1,<2 + # scikit-learn 0.21+ required to fix murmurhash3_32 import bug + scikit-learn>=0.21,<2 # segmentation-models-pytorch 0.2+ required for smp.losses module segmentation-models-pytorch>=0.2,<0.3 # shapely 1.3+ required for Python 3 support - shapely~=1.3 - # timm 0.2.1+ required for `features_only` option in create_model - timm>=0.2.1,<0.5 - # torch 1.7+ required for typing - torch~=1.7 + shapely>=1.3,<2 + # timm 0.4.12 required by segmentation-models-pytorch + timm>=0.4.12,<0.5 + # torch 1.9+ required by torchvision + torch>=1.9,<2 # torchmetrics 0.7+ required for JaccardIndex torchmetrics>=0.7,<0.10 # torchvision 0.10+ required for torchvision.utils.draw_segmentation_masks @@ -71,31 +72,32 @@ include = torchgeo* [options.extras_require] datasets = - h5py!=3.7.0,<4 - # laspy 2+ required for Python 3.7+ support - laspy~=2.0 + # h5py 2.6+ required for bugfixes to HDF5 API + h5py>=2.6,<4 + # laspy 2+ required for laspy.read + laspy>=2,<3 # open3d 0.11.2+ required to avoid GLFW error: # https://github.com/isl-org/Open3D/issues/1550 open3d>=0.11.2,<0.15;python_version<'3.10' - opencv-python<5 - # pandas 0.23.2+ required for python 3.7 support + # opencv-python 3.4.2.17 is oldest buildable version on PyPI + opencv-python>=3.4.2.17,<5 + # pandas 0.23.2+ required for Python 3.7 wheels pandas>=0.23.2,<2 - pycocotools<3 + # pycocotools 2.0.0 is oldest version on PyPI + pycocotools>=2,<3 # radiant-mlhub 0.2.1+ required for api_key bugfix: # https://github.com/radiantearth/radiant-mlhub/pull/48 radiant-mlhub>=0.2.1,<0.6 # rarfile 3+ required for correct Rar file detection rarfile>=3,<5 - # scipy 0.9+ required for scipy.io.wavfile.read - scipy>=0.9,<2 + # scipy 1.2+ required to fix LinearNDInterpolator import bug + scipy>=1.2,<2 # zipfile-deflate64 0.2+ required for extraction bugfix: # https://github.com/brianhelba/zipfile-deflate64/issues/19 zipfile-deflate64>=0.2,<0.3 docs = - # ipywidgets 7+ required for nbsphinx - ipywidgets~=7.0 # nbsphinx 0.8.5 fixes bug with nbformat attributes - nbsphinx~=0.8.5 + nbsphinx>=0.8.5,<0.9 # release versions missing files, must install from master pytorch-sphinx-theme # sphinx 4+ required for autodoc_typehints_description_target = documented @@ -107,9 +109,9 @@ style = # https://github.com/PyCQA/pyflakes/pull/455 flake8>=3.8,<5 # isort 5.8+ required for extend_skip option - isort[colors]~=5.8 + isort[colors]>=5.8,<6 # pydocstyle 6.1+ required for pyproject.toml support - pydocstyle[toml]~=6.1 + pydocstyle[toml]>=6.1,<7 # pyupgrade 1.24+ required for --py37-plus flag pyupgrade>=1.24,<3 tests = @@ -117,8 +119,8 @@ tests = mypy>=0.900,<=0.961 # nbmake 0.1+ required to fix path_source bug nbmake>=0.1,<2 - # pytest 6+ required for pyproject.toml support - pytest>=6,<8 + # pytest 6.1.2+ required by nbmake + pytest>=6.1.2,<8 # pytest-cov 2.4+ required for pytest --cov flags pytest-cov>=2.4,<4 diff --git a/tests/datasets/test_advance.py b/tests/datasets/test_advance.py index 2eb05cee433..789772fe1b4 100644 --- a/tests/datasets/test_advance.py +++ b/tests/datasets/test_advance.py @@ -49,7 +49,7 @@ def mocked_import(name: str, *args: Any, **kwargs: Any) -> Any: monkeypatch.setattr(builtins, "__import__", mocked_import) def test_getitem(self, dataset: ADVANCE) -> None: - pytest.importorskip("scipy", minversion="0.9.0") + pytest.importorskip("scipy", minversion="1.2") x = dataset[0] assert isinstance(x, dict) assert isinstance(x["image"], torch.Tensor) diff --git a/tests/datasets/test_idtrees.py b/tests/datasets/test_idtrees.py index 881732e01b6..5abef719715 100644 --- a/tests/datasets/test_idtrees.py +++ b/tests/datasets/test_idtrees.py @@ -19,7 +19,7 @@ import torchgeo.datasets.utils from torchgeo.datasets import IDTReeS -pytest.importorskip("pandas", minversion="0.19.1") +pytest.importorskip("pandas", minversion="0.23.2") pytest.importorskip("laspy", minversion="2") diff --git a/tests/datasets/test_openbuildings.py b/tests/datasets/test_openbuildings.py index 7c081bb70b6..5fd879f9269 100644 --- a/tests/datasets/test_openbuildings.py +++ b/tests/datasets/test_openbuildings.py @@ -24,7 +24,7 @@ UnionDataset, ) -pytest.importorskip("pandas", minversion="0.19.1") +pytest.importorskip("pandas", minversion="0.23.2") class TestOpenBuildings: diff --git a/tests/datasets/test_so2sat.py b/tests/datasets/test_so2sat.py index e57644a3628..351dbefaee6 100644 --- a/tests/datasets/test_so2sat.py +++ b/tests/datasets/test_so2sat.py @@ -15,7 +15,7 @@ from torchgeo.datasets import So2Sat -pytest.importorskip("h5py") +pytest.importorskip("h5py", minversion="2.6") class TestSo2Sat: diff --git a/tests/datasets/test_usavars.py b/tests/datasets/test_usavars.py index d43ea794f47..15aa10ad3cf 100644 --- a/tests/datasets/test_usavars.py +++ b/tests/datasets/test_usavars.py @@ -18,7 +18,7 @@ import torchgeo.datasets.utils from torchgeo.datasets import USAVars -pytest.importorskip("pandas", minversion="0.19.1") +pytest.importorskip("pandas", minversion="0.23.2") def download_url(url: str, root: str, *args: str, **kwargs: str) -> None: diff --git a/tests/datasets/test_zuericrop.py b/tests/datasets/test_zuericrop.py index ec246ff15fc..a6c22e6c213 100644 --- a/tests/datasets/test_zuericrop.py +++ b/tests/datasets/test_zuericrop.py @@ -16,7 +16,7 @@ import torchgeo.datasets.utils from torchgeo.datasets import ZueriCrop -pytest.importorskip("h5py") +pytest.importorskip("h5py", minversion="2.6") def download_url(url: str, root: str, *args: str, **kwargs: str) -> None: diff --git a/tests/trainers/test_classification.py b/tests/trainers/test_classification.py index a78d1350f86..2fecd61aed8 100644 --- a/tests/trainers/test_classification.py +++ b/tests/trainers/test_classification.py @@ -42,7 +42,7 @@ def test_trainer( self, monkeypatch: MonkeyPatch, name: str, classname: Type[LightningDataModule] ) -> None: if name.startswith("so2sat"): - pytest.importorskip("h5py") + pytest.importorskip("h5py", minversion="2.6") conf = OmegaConf.load(os.path.join("tests", "conf", name + ".yaml")) conf_dict = OmegaConf.to_object(conf.experiment) diff --git a/tests/transforms/test_transforms.py b/tests/transforms/test_transforms.py index 0f38c63bef0..6cec37bf8cc 100644 --- a/tests/transforms/test_transforms.py +++ b/tests/transforms/test_transforms.py @@ -11,9 +11,6 @@ from torchgeo.transforms import indices, transforms -# Tests require newer version of Kornia for newer bounding box behavior -pytest.importorskip("kornia", minversion="0.6.3") - @pytest.fixture def batch_gray() -> Dict[str, Tensor]: diff --git a/torchgeo/datasets/advance.py b/torchgeo/datasets/advance.py index d87cd154390..bc099ea51aa 100644 --- a/torchgeo/datasets/advance.py +++ b/torchgeo/datasets/advance.py @@ -199,7 +199,7 @@ def _load_target(self, path: str) -> Tensor: "scipy is not installed and is required to use this dataset" ) - array = wavfile.read(path)[1] + array = wavfile.read(path, mmap=True)[1] tensor = torch.from_numpy(array) tensor = tensor.unsqueeze(0) return tensor diff --git a/torchgeo/datasets/cyclone.py b/torchgeo/datasets/cyclone.py index 1673b9e2302..8c7b12f83ba 100644 --- a/torchgeo/datasets/cyclone.py +++ b/torchgeo/datasets/cyclone.py @@ -146,7 +146,7 @@ def _load_image(self, directory: str) -> Tensor: # Moved in PIL 9.1.0 try: resample = Image.Resampling.BILINEAR - except AttributeError: # pragma: no cover + except AttributeError: resample = Image.BILINEAR img = img.resize(size=(self.size, self.size), resample=resample) array: "np.typing.NDArray[np.int_]" = np.array(img) diff --git a/torchgeo/datasets/seco.py b/torchgeo/datasets/seco.py index 6cafa483ea9..faf3875c166 100644 --- a/torchgeo/datasets/seco.py +++ b/torchgeo/datasets/seco.py @@ -183,7 +183,7 @@ def _load_patch(self, scene_name: str, patch_name: str) -> Tensor: # Moved in PIL 9.1.0 try: resample = Image.Resampling.BILINEAR - except AttributeError: # pragma: no cover + except AttributeError: resample = Image.BILINEAR band_data = np.array( pil_image.resize((264, 264), resample=resample)