From 21ea965a7aac1c8366928a5ab982bb834c995452 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Wed, 20 Mar 2024 21:42:34 +0100 Subject: [PATCH 01/12] pep 621 dev dependencies --- pdm.lock | 2 +- python/deptry/cli.py | 12 +++++ python/deptry/core.py | 5 +- python/deptry/dependency_getter/pep_621.py | 35 +++++++++++- tests/data/pep_621_project/pyproject.toml | 4 ++ tests/functional/cli/test_cli_pep_621.py | 56 ++++++-------------- tests/unit/dependency_getter/test_pep_621.py | 44 +++++++++++++++ tests/unit/test_core.py | 1 + 8 files changed, 116 insertions(+), 43 deletions(-) diff --git a/pdm.lock b/pdm.lock index 3a38d00d..218bfc85 100644 --- a/pdm.lock +++ b/pdm.lock @@ -5,7 +5,7 @@ groups = ["default", "dev", "docs", "typing"] strategy = ["cross_platform", "inherit_metadata"] lock_version = "4.4.1" -content_hash = "sha256:075a94a78d7250b4bdc64a8e66445f285b6afec47004ca0e45aab3b0744eec29" +content_hash = "sha256:8ebb6c1f321887e2f1b32d8b0cd92c661e9a8879b57df2f1a5aa777ac884bd32" [[package]] name = "babel" diff --git a/python/deptry/cli.py b/python/deptry/cli.py index a4d206bc..d134191a 100644 --- a/python/deptry/cli.py +++ b/python/deptry/cli.py @@ -224,6 +224,16 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b default={}, show_default=False, ) +@click.option( + "--pep621-dev-dependency-groups", + "-ddg", + type=COMMA_SEPARATED_TUPLE, + help="""For projects that use PEP621 and that do not use a build tool that has it's own method of declaring development dependencies, + the --pep621-dev-dependency-groups (-ddg) provides the option to specify which groups under [project.optional-dependencies] in pyproject.toml + should be considered development dependencies. For example, use `--pep621-dev-dependency-groups tests,docs` to mark 'tests' and 'docs' as development groups.""", + default=(), + show_default=False, +) def deptry( root: tuple[Path, ...], config: Path, @@ -238,6 +248,7 @@ def deptry( known_first_party: tuple[str, ...], json_output: str, package_module_name_map: MutableMapping[str, tuple[str, ...]], + pep621_dev_dependency_groups: tuple[str, ...], ) -> None: """Find dependency issues in your Python project. @@ -267,4 +278,5 @@ def deptry( known_first_party=known_first_party, json_output=json_output, package_module_name_map=package_module_name_map, + pep621_dev_dependency_groups=pep621_dev_dependency_groups, ).run() diff --git a/python/deptry/core.py b/python/deptry/core.py index ff8b0ec1..04c2db38 100644 --- a/python/deptry/core.py +++ b/python/deptry/core.py @@ -53,6 +53,7 @@ class Core: known_first_party: tuple[str, ...] json_output: str package_module_name_map: Mapping[str, tuple[str, ...]] + pep621_dev_dependency_groups: tuple[str, ...] def run(self) -> None: self._log_config() @@ -143,7 +144,9 @@ def _get_dependencies(self, dependency_management_format: DependencyManagementFo if dependency_management_format is DependencyManagementFormat.PDM: return PDMDependencyGetter(self.config, self.package_module_name_map).get() if dependency_management_format is DependencyManagementFormat.PEP_621: - return PEP621DependencyGetter(self.config, self.package_module_name_map).get() + return PEP621DependencyGetter( + self.config, self.package_module_name_map, self.pep621_dev_dependency_groups + ).get() if dependency_management_format is DependencyManagementFormat.REQUIREMENTS_TXT: return RequirementsTxtDependencyGetter( self.config, self.package_module_name_map, self.requirements_txt, self.requirements_txt_dev diff --git a/python/deptry/dependency_getter/pep_621.py b/python/deptry/dependency_getter/pep_621.py index b7001673..a46d814c 100644 --- a/python/deptry/dependency_getter/pep_621.py +++ b/python/deptry/dependency_getter/pep_621.py @@ -15,6 +15,7 @@ @dataclass class PEP621DependencyGetter(DependencyGetter): + pep621_dev_dependency_groups: tuple[str, ...] = () """ Class to extract dependencies from a pyproject.toml file in which dependencies are specified according to PEP 621. For example: @@ -39,8 +40,21 @@ class PEP621DependencyGetter(DependencyGetter): def get(self) -> DependenciesExtract: dependencies = [*self._get_dependencies(), *itertools.chain(*self._get_optional_dependencies().values())] - self._log_dependencies(dependencies) + dependencies = self._get_dependencies() + optional_dependencies = self._get_optional_dependencies() + + if self.pep621_dev_dependency_groups: + dev_dependencies, optional_dependencies = self._split_development_dependencies_from_optional_dependencies( + optional_dependencies + ) + dependencies = [*dependencies, *optional_dependencies] + self._log_dependencies(dependencies) + self._log_dependencies(dev_dependencies, is_dev=True) + return DependenciesExtract(dependencies, dev_dependencies) + + dependencies = [*dependencies, *itertools.chain(*optional_dependencies.values())] + self._log_dependencies(dependencies) return DependenciesExtract(dependencies, []) def _get_dependencies(self) -> list[Dependency]: @@ -56,6 +70,25 @@ def _get_optional_dependencies(self) -> dict[str, list[Dependency]]: for group, dependencies in pyproject_data["project"].get("optional-dependencies", {}).items() } + def _split_development_dependencies_from_optional_dependencies( + self, optional_dependencies: dict[str, list[Dependency]] + ) -> tuple[list[Dependency], list[Dependency]]: + """ + Split the optional dependencies into optional dependencies and development dependencies based on the `pep621_dev_dependency_groups` + parameter. Return a tuple with two values: a list of the development dependencies and a list of the remaining 'true' optional dependencies. + """ + dev_dependencies = list( + itertools.chain.from_iterable( + deps for group, deps in optional_dependencies.items() if group in self.pep621_dev_dependency_groups + ) + ) + regular_dependencies = list( + itertools.chain.from_iterable( + deps for group, deps in optional_dependencies.items() if group not in self.pep621_dev_dependency_groups + ) + ) + return dev_dependencies, regular_dependencies + def _extract_pep_508_dependencies( self, dependencies: list[str], package_module_name_map: Mapping[str, Sequence[str]] ) -> list[Dependency]: diff --git a/tests/data/pep_621_project/pyproject.toml b/tests/data/pep_621_project/pyproject.toml index 52de6916..d845bb95 100644 --- a/tests/data/pep_621_project/pyproject.toml +++ b/tests/data/pep_621_project/pyproject.toml @@ -19,10 +19,14 @@ dev = [ "mypy==0.982", ] test = ["pytest==7.2.0"] +plot = ["matplotlib"] [build-system] requires = ["setuptools>=61.0.0"] build-backend = "setuptools.build_meta" +[tool.deptry] +pep621_dev_dependency_groups = ["dev"] + [tool.deptry.per_rule_ignores] DEP002 = ["pkginfo"] diff --git a/tests/functional/cli/test_cli_pep_621.py b/tests/functional/cli/test_cli_pep_621.py index b81f67f1..3939d1f6 100644 --- a/tests/functional/cli/test_cli_pep_621.py +++ b/tests/functional/cli/test_cli_pep_621.py @@ -22,16 +22,9 @@ def test_cli_with_pep_621(pip_venv_factory: PipVenvFactory) -> None: assert result.returncode == 1 assert get_issues_report(Path(issue_report)) == [ { - "error": { - "code": "DEP002", - "message": "'isort' defined as a dependency but not used in the codebase", - }, + "error": {"code": "DEP002", "message": "'isort' defined as a dependency but not used in the codebase"}, "module": "isort", - "location": { - "file": str(Path("pyproject.toml")), - "line": None, - "column": None, - }, + "location": {"file": str(Path("pyproject.toml")), "line": None, "column": None}, }, { "error": { @@ -39,46 +32,29 @@ def test_cli_with_pep_621(pip_venv_factory: PipVenvFactory) -> None: "message": "'requests' defined as a dependency but not used in the codebase", }, "module": "requests", - "location": { - "file": str(Path("pyproject.toml")), - "line": None, - "column": None, - }, + "location": {"file": str(Path("pyproject.toml")), "line": None, "column": None}, }, { - "error": { - "code": "DEP002", - "message": "'mypy' defined as a dependency but not used in the codebase", - }, - "module": "mypy", - "location": { - "file": str(Path("pyproject.toml")), - "line": None, - "column": None, - }, + "error": {"code": "DEP002", "message": "'pytest' defined as a dependency but not used in the codebase"}, + "module": "pytest", + "location": {"file": str(Path("pyproject.toml")), "line": None, "column": None}, }, { "error": { "code": "DEP002", - "message": "'pytest' defined as a dependency but not used in the codebase", - }, - "module": "pytest", - "location": { - "file": str(Path("pyproject.toml")), - "line": None, - "column": None, + "message": "'matplotlib' defined as a dependency but not used in the codebase", }, + "module": "matplotlib", + "location": {"file": str(Path("pyproject.toml")), "line": None, "column": None}, }, { - "error": { - "code": "DEP001", - "message": "'white' imported but missing from the dependency definitions", - }, + "error": {"code": "DEP004", "message": "'black' imported but declared as a dev dependency"}, + "module": "black", + "location": {"file": str(Path("src/main.py")), "line": 4, "column": 8}, + }, + { + "error": {"code": "DEP001", "message": "'white' imported but missing from the dependency definitions"}, "module": "white", - "location": { - "file": str(Path("src/main.py")), - "line": 6, - "column": 8, - }, + "location": {"file": str(Path("src/main.py")), "line": 6, "column": 8}, }, ] diff --git a/tests/unit/dependency_getter/test_pep_621.py b/tests/unit/dependency_getter/test_pep_621.py index 55c77708..e955d7ec 100644 --- a/tests/unit/dependency_getter/test_pep_621.py +++ b/tests/unit/dependency_getter/test_pep_621.py @@ -79,3 +79,47 @@ def test_dependency_getter(tmp_path: Path) -> None: assert not dependencies[7].is_conditional assert not dependencies[7].is_optional assert "dep" in dependencies[7].top_levels + + +def test_dependency_getter_with_dev_dependencies(tmp_path: Path) -> None: + fake_pyproject_toml = """[project] +# PEP 621 project metadata +# See https://www.python.org/dev/peps/pep-0621/ +dependencies = [ +"qux", +] + +[project.optional-dependencies] +group1 = [ + "foobar", +] +group2 = [ + "barfoo", +] +""" + + with run_within_dir(tmp_path): + with Path("pyproject.toml").open("w") as f: + f.write(fake_pyproject_toml) + + getter = PEP621DependencyGetter(config=Path("pyproject.toml"), pep621_dev_dependency_groups=("group2")) + dependencies = getter.get().dependencies + dev_dependencies = getter.get().dev_dependencies + + assert len(dependencies) == 2 + + assert dependencies[0].name == "qux" + assert not dependencies[0].is_conditional + assert not dependencies[0].is_optional + assert "qux" in dependencies[0].top_levels + + assert dependencies[1].name == "foobar" + assert not dependencies[1].is_conditional + assert not dependencies[1].is_optional + assert "foobar" in dependencies[1].top_levels + + assert len(dev_dependencies) == 1 + assert dev_dependencies[0].name == "barfoo" + assert not dev_dependencies[0].is_conditional + assert not dev_dependencies[0].is_optional + assert "barfoo" in dev_dependencies[0].top_levels diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index 4a91aa27..159ceea5 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -91,6 +91,7 @@ def test__get_local_modules( known_first_party=known_first_party, json_output="", package_module_name_map={}, + pep621_dev_dependency_groups=(), )._get_local_modules() == expected ) From 160899dedf52051e63499f90db52c1829e15ca3a Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Wed, 20 Mar 2024 21:49:28 +0100 Subject: [PATCH 02/12] fix typing --- python/deptry/cli.py | 2 +- python/deptry/dependency_getter/pep_621.py | 6 +++--- tests/unit/dependency_getter/test_pep_621.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/python/deptry/cli.py b/python/deptry/cli.py index d134191a..2c7c7865 100644 --- a/python/deptry/cli.py +++ b/python/deptry/cli.py @@ -228,7 +228,7 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b "--pep621-dev-dependency-groups", "-ddg", type=COMMA_SEPARATED_TUPLE, - help="""For projects that use PEP621 and that do not use a build tool that has it's own method of declaring development dependencies, + help="""For projects that use PEP621 and that do not use a build tool that has its own method of declaring development dependencies, the --pep621-dev-dependency-groups (-ddg) provides the option to specify which groups under [project.optional-dependencies] in pyproject.toml should be considered development dependencies. For example, use `--pep621-dev-dependency-groups tests,docs` to mark 'tests' and 'docs' as development groups.""", default=(), diff --git a/python/deptry/dependency_getter/pep_621.py b/python/deptry/dependency_getter/pep_621.py index a46d814c..cc33c9dd 100644 --- a/python/deptry/dependency_getter/pep_621.py +++ b/python/deptry/dependency_getter/pep_621.py @@ -45,10 +45,10 @@ def get(self) -> DependenciesExtract: optional_dependencies = self._get_optional_dependencies() if self.pep621_dev_dependency_groups: - dev_dependencies, optional_dependencies = self._split_development_dependencies_from_optional_dependencies( - optional_dependencies + dev_dependencies, leftover_optional_dependencies = ( + self._split_development_dependencies_from_optional_dependencies(optional_dependencies) ) - dependencies = [*dependencies, *optional_dependencies] + dependencies = [*dependencies, *leftover_optional_dependencies] self._log_dependencies(dependencies) self._log_dependencies(dev_dependencies, is_dev=True) return DependenciesExtract(dependencies, dev_dependencies) diff --git a/tests/unit/dependency_getter/test_pep_621.py b/tests/unit/dependency_getter/test_pep_621.py index e955d7ec..bd33bf23 100644 --- a/tests/unit/dependency_getter/test_pep_621.py +++ b/tests/unit/dependency_getter/test_pep_621.py @@ -102,7 +102,7 @@ def test_dependency_getter_with_dev_dependencies(tmp_path: Path) -> None: with Path("pyproject.toml").open("w") as f: f.write(fake_pyproject_toml) - getter = PEP621DependencyGetter(config=Path("pyproject.toml"), pep621_dev_dependency_groups=("group2")) + getter = PEP621DependencyGetter(config=Path("pyproject.toml"), pep621_dev_dependency_groups=("group2",)) dependencies = getter.get().dependencies dev_dependencies = getter.get().dev_dependencies From 7bcc44259ee90185a4072e15704a5121ca58fbea Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Thu, 21 Mar 2024 07:17:18 +0100 Subject: [PATCH 03/12] add docs --- docs/usage.md | 44 +++++++++++++++++++++- python/deptry/dependency_getter/pep_621.py | 6 ++- 2 files changed, 47 insertions(+), 3 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 0305993a..788941ec 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -29,7 +29,8 @@ To determine the project's dependencies, _deptry_ will scan the directory it is - dependencies from `[project.dependencies]` and `[project.optional-dependencies]` sections - development dependencies from `[tool.pdm.dev-dependencies]` section. 3. If a `pyproject.toml` file with a `[project]` section is found, _deptry_ will assume it uses [PEP 621](https://peps.python.org/pep-0621/) for dependency specification and extract: - - dependencies from `[project.dependencies]` and `[project.optional-dependencies]` sections + - dependencies from `[project.dependencies]` and `[project.optional-dependencies]`. + - development dependecies from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument. 4. If a `requirements.txt` file is found, _deptry_ will extract: - dependencies from it - development dependencies from `dev-dependencies.txt` and `dependencies-dev.txt`, if any exist @@ -400,3 +401,44 @@ Multiple package name to module name mappings are joined by a comma (`,`): ```shell deptry . --package-module-name-map 'foo-python=foo,bar-python=bar' ``` + +#### PEP 621 dev dependency groups + +PEP 621 does [not define](https://peps.python.org/pep-0621/#recommend-that-tools-put-development-related-dependencies-into-a-dev-extra) a standard convention for specifying development dependencies. However, deptry offers a mechanism to interpret specific optional dependency groups as development dependencies. + +By default, all dependencies under `[project.dependencies]` and `[project.optional-dependencies]` are extracted as regular dependencies. By using the `--pep621-dev-dependency-groups` argument, users can specify which groups defined under `[project.optional-dependencies]` should be treated as development dependencies instead. This is particularly useful for projects that adhere to PEP 621 but do not employ a separate build tool for declaring development dependencies. + +For example, consider a project with the following `pyproject.toml`: + +``` +[project] +... +dependencies = [ + "httpx", +] + +[project.optional-dependencies] +test = [ + "pytest < 5.0.0", +] +plot = [ + "matplotlib", +] +``` + +By default, `https`, `pytest` and `matplotlib` are extracted as regular dependencies. By specifying `--pep621-dev-dependency-groups=test`, +the dependency `pytest` will be considered a development dependency instead. + +- Type: `List[str]` +- Default: `[]` +- `pyproject.toml` option name: `pep621_dev_dependency_groups` +- CLI option name: `--pep621-dev-dependency-groups` (short: `-ddg`) +- `pyproject.toml` example: +```toml +[tool.deptry] +pep621_dev_dependency_groups = ["test", "docs"] +``` +- CLI example: +```shell +deptry . --pep621-dev-dependency-groups "test,docs" +``` diff --git a/python/deptry/dependency_getter/pep_621.py b/python/deptry/dependency_getter/pep_621.py index cc33c9dd..95c38516 100644 --- a/python/deptry/dependency_getter/pep_621.py +++ b/python/deptry/dependency_getter/pep_621.py @@ -34,8 +34,10 @@ class PEP621DependencyGetter(DependencyGetter): "pytest-cov[all]" ] - Note that both dependencies and optional-dependencies are extracted as regular dependencies. Since PEP-621 does not specify - a recommended way to extract development dependencies, we do not attempt to extract any from the pyproject.toml file. + Note that by default both dependencies and optional-dependencies are extracted as regular dependencies, since PEP-621 does not specify + a recommended way to extract development dependencies. However, if a value is passed for the `pep621_dev_dependency_groups` + argument, all dependencies from groups in that argument are considered to be development dependencies. e.g. in the example above, when + `pep621_dev_dependency_groups=(test,)`, both `pytest` and `pytest-cov` are returned as development dependencies. """ def get(self) -> DependenciesExtract: From c19335199c1c545d098f2e476ad0cb896237c50a Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Thu, 21 Mar 2024 12:08:44 +0100 Subject: [PATCH 04/12] better docstring --- python/deptry/cli.py | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/python/deptry/cli.py b/python/deptry/cli.py index 1d77a319..93940482 100644 --- a/python/deptry/cli.py +++ b/python/deptry/cli.py @@ -229,8 +229,9 @@ def display_deptry_version(ctx: click.Context, _param: click.Parameter, value: b "-ddg", type=COMMA_SEPARATED_TUPLE, help="""For projects that use PEP621 and that do not use a build tool that has its own method of declaring development dependencies, - the --pep621-dev-dependency-groups (-ddg) provides the option to specify which groups under [project.optional-dependencies] in pyproject.toml - should be considered development dependencies. For example, use `--pep621-dev-dependency-groups tests,docs` to mark 'tests' and 'docs' as development groups.""", + this argument provides the option to specify which groups under [project.optional-dependencies] in pyproject.toml + should be considered development dependencies. For example, use `--pep621-dev-dependency-groups tests,docs` to mark the dependencies in + the groups 'tests' and 'docs' as development dependencies.""", default=(), show_default=False, ) From 2ffa0f41ee6acff90532c61e13df6deab170d3f9 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Thu, 21 Mar 2024 13:42:57 +0100 Subject: [PATCH 05/12] log a warning when group not found --- python/deptry/dependency_getter/pep_621.py | 12 +++++ tests/unit/dependency_getter/test_pep_621.py | 52 ++++++++++++++++++++ 2 files changed, 64 insertions(+) diff --git a/python/deptry/dependency_getter/pep_621.py b/python/deptry/dependency_getter/pep_621.py index 95c38516..2f31fe56 100644 --- a/python/deptry/dependency_getter/pep_621.py +++ b/python/deptry/dependency_getter/pep_621.py @@ -1,6 +1,7 @@ from __future__ import annotations import itertools +import logging import re from dataclasses import dataclass from typing import TYPE_CHECKING @@ -47,6 +48,7 @@ def get(self) -> DependenciesExtract: optional_dependencies = self._get_optional_dependencies() if self.pep621_dev_dependency_groups: + self._check_for_invalid_group_names(optional_dependencies) dev_dependencies, leftover_optional_dependencies = ( self._split_development_dependencies_from_optional_dependencies(optional_dependencies) ) @@ -72,6 +74,16 @@ def _get_optional_dependencies(self) -> dict[str, list[Dependency]]: for group, dependencies in pyproject_data["project"].get("optional-dependencies", {}).items() } + def _check_for_invalid_group_names(self, optional_dependencies: dict[str, list[Dependency]]) -> None: + missing_groups = set(self.pep621_dev_dependency_groups) - set(optional_dependencies.keys()) + if missing_groups: + logging.warning( + "Warning: Trying to extract the dependencies from the optional dependency groups %s as development dependencies, " + "but the following groups were not found: %s", + list(self.pep621_dev_dependency_groups), + list(missing_groups), + ) + def _split_development_dependencies_from_optional_dependencies( self, optional_dependencies: dict[str, list[Dependency]] ) -> tuple[list[Dependency], list[Dependency]]: diff --git a/tests/unit/dependency_getter/test_pep_621.py b/tests/unit/dependency_getter/test_pep_621.py index bd33bf23..5689f506 100644 --- a/tests/unit/dependency_getter/test_pep_621.py +++ b/tests/unit/dependency_getter/test_pep_621.py @@ -1,10 +1,15 @@ from __future__ import annotations +import logging from pathlib import Path +from typing import TYPE_CHECKING from deptry.dependency_getter.pep_621 import PEP621DependencyGetter from tests.utils import run_within_dir +if TYPE_CHECKING: + from _pytest.logging import LogCaptureFixture + def test_dependency_getter(tmp_path: Path) -> None: fake_pyproject_toml = """[project] @@ -123,3 +128,50 @@ def test_dependency_getter_with_dev_dependencies(tmp_path: Path) -> None: assert not dev_dependencies[0].is_conditional assert not dev_dependencies[0].is_optional assert "barfoo" in dev_dependencies[0].top_levels + + +def test_dependency_getter_with_incorrect_dev_group(tmp_path: Path, caplog: LogCaptureFixture) -> None: + fake_pyproject_toml = """[project] +# PEP 621 project metadata +# See https://www.python.org/dev/peps/pep-0621/ +dependencies = [ +"qux", +] + +[project.optional-dependencies] +group1 = [ + "foobar", +] +group2 = [ + "barfoo", +] +""" + + with run_within_dir(tmp_path), caplog.at_level(logging.INFO): + with Path("pyproject.toml").open("w") as f: + f.write(fake_pyproject_toml) + + getter = PEP621DependencyGetter(config=Path("pyproject.toml"), pep621_dev_dependency_groups=("group3",)) + dependencies = getter.get().dependencies + + assert ( + "Trying to extract the dependencies from the optional dependency groups ['group3'] as development dependencies, but the following groups were not found: ['group3']" + in caplog.text + ) + + assert len(dependencies) == 3 + + assert dependencies[0].name == "qux" + assert not dependencies[0].is_conditional + assert not dependencies[0].is_optional + assert "qux" in dependencies[0].top_levels + + assert dependencies[1].name == "foobar" + assert not dependencies[1].is_conditional + assert not dependencies[1].is_optional + assert "foobar" in dependencies[1].top_levels + + assert dependencies[2].name == "barfoo" + assert not dependencies[2].is_conditional + assert not dependencies[2].is_optional + assert "barfoo" in dependencies[2].top_levels From 227996994e09f724bfc3a4ffc599f1070aa58037 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Thu, 21 Mar 2024 13:53:29 +0100 Subject: [PATCH 06/12] fix issue --- python/deptry/dependency_getter/pep_621.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/python/deptry/dependency_getter/pep_621.py b/python/deptry/dependency_getter/pep_621.py index 2f31fe56..0e27ddac 100644 --- a/python/deptry/dependency_getter/pep_621.py +++ b/python/deptry/dependency_getter/pep_621.py @@ -42,8 +42,6 @@ class PEP621DependencyGetter(DependencyGetter): """ def get(self) -> DependenciesExtract: - dependencies = [*self._get_dependencies(), *itertools.chain(*self._get_optional_dependencies().values())] - dependencies = self._get_dependencies() optional_dependencies = self._get_optional_dependencies() From 482056f0cefe23a46140ad179b16c23eb9701378 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Thu, 21 Mar 2024 18:43:22 +0100 Subject: [PATCH 07/12] change List to list in docs --- docs/usage.md | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/docs/usage.md b/docs/usage.md index 788941ec..42d21b9d 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -140,7 +140,7 @@ deptry . --no-ansi List of patterns to exclude when searching for source files. -- Type: `List[str]` +- Type: `list[str]` - Default: `["venv", "\.venv", "\.direnv", "tests", "\.git", "setup\.py"]` - `pyproject.toml` option name: `exclude` - CLI option name: `--exclude` (short: `-e`) @@ -159,7 +159,7 @@ deptry . --exclude "a_directory|a_python_file\.py|a_pattern/.*" Additional list of patterns to exclude when searching for source files. This extends the patterns set in [Exclude](#exclude), to allow defining patterns while keeping the default list. -- Type: `List[str]` +- Type: `list[str]` - Default: `[]` - `pyproject.toml` option name: `extend_exclude` - CLI option name: `--extend-exclude` (short: `-ee`) @@ -177,7 +177,7 @@ deptry . --extend-exclude "a_directory|a_python_file\.py|a_pattern/.*" A comma-separated list of [rules](rules-violations.md) to ignore. -- Type: `List[str]` +- Type: `list[str]` - Default: `[]` - `pyproject.toml` option name: `ignore` - CLI option name: `--ignore` (short: `-i`) @@ -232,7 +232,7 @@ deptry . --ignore-notebooks List of [`pip` requirements files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) that contain the source dependencies. -- Type: `List[str]` +- Type: `list[str]` - Default: `["requirements.txt"]` - `pyproject.toml` option name: `requirements_txt` - CLI option name: `--requirements-txt` (short: `-rt`) @@ -250,7 +250,7 @@ deptry . --requirements-txt requirements.txt,requirements-private.txt List of [`pip` requirements files](https://pip.pypa.io/en/stable/user_guide/#requirements-files) that contain the source development dependencies. -- Type: `List[str]` +- Type: `list[str]` - Default: `["dev-requirements.txt", "requirements-dev.txt"]` - `pyproject.toml` option name: `requirements_txt_dev` - CLI option name: `--requirements-txt-dev` (short: `-rtd`) @@ -268,7 +268,7 @@ deptry . --requirements-txt-dev requirements-dev.txt,requirements-tests.txt List of Python modules that should be considered as first party ones. This is useful in case _deptry_ is not able to automatically detect modules that should be considered as local ones. -- Type: `List[str]` +- Type: `list[str]` - Default: `[]` - `pyproject.toml` option name: `known_first_party` - CLI option name: `--known-first-party` (short: `-kf`) @@ -429,7 +429,7 @@ plot = [ By default, `https`, `pytest` and `matplotlib` are extracted as regular dependencies. By specifying `--pep621-dev-dependency-groups=test`, the dependency `pytest` will be considered a development dependency instead. -- Type: `List[str]` +- Type: `list[str]` - Default: `[]` - `pyproject.toml` option name: `pep621_dev_dependency_groups` - CLI option name: `--pep621-dev-dependency-groups` (short: `-ddg`) From 8b8b4e1735184d1588e5cba9c23288b485946972 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Fri, 22 Mar 2024 07:45:59 +0100 Subject: [PATCH 08/12] Update docs/usage.md MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Co-authored-by: Edgar Ramírez Mondragón <16805946+edgarrmondragon@users.noreply.github.com> --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 42d21b9d..5fbce1b3 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -426,7 +426,7 @@ plot = [ ] ``` -By default, `https`, `pytest` and `matplotlib` are extracted as regular dependencies. By specifying `--pep621-dev-dependency-groups=test`, +By default, `httpx`, `pytest` and `matplotlib` are extracted as regular dependencies. By specifying `--pep621-dev-dependency-groups=test`, the dependency `pytest` will be considered a development dependency instead. - Type: `list[str]` From 7eead0ff485dd6192f89f4a74607833f59e6133f Mon Sep 17 00:00:00 2001 From: Mathieu Kniewallner Date: Sat, 23 Mar 2024 20:07:18 +0100 Subject: [PATCH 09/12] feat(pdm): handle PEP 621 dev groups (#637) * refactor: move dependencies logging to `core` * feat(pdm): handle PEP 621 dev groups --- python/deptry/core.py | 20 ++++++++- python/deptry/dependency_getter/base.py | 10 ----- python/deptry/dependency_getter/pdm.py | 8 ++-- python/deptry/dependency_getter/pep_621.py | 3 -- python/deptry/dependency_getter/poetry.py | 8 +--- .../dependency_getter/requirements_txt.py | 2 - tests/data/project_with_pdm/pyproject.toml | 3 ++ tests/functional/cli/test_cli.py | 1 + tests/functional/cli/test_cli_pdm.py | 12 ----- tests/unit/test_core.py | 44 +++++++++++++++++++ 10 files changed, 72 insertions(+), 39 deletions(-) diff --git a/python/deptry/core.py b/python/deptry/core.py index 04c2db38..2c017c6a 100644 --- a/python/deptry/core.py +++ b/python/deptry/core.py @@ -63,6 +63,8 @@ def run(self) -> None: ).detect() dependencies_extract = self._get_dependencies(dependency_management_format) + self._log_dependencies(dependencies_extract) + all_python_files = PythonFileFinder( self.exclude, self.extend_exclude, self.using_default_exclude, self.ignore_notebooks ).get_all_python_files_in(self.root) @@ -142,7 +144,9 @@ def _get_dependencies(self, dependency_management_format: DependencyManagementFo if dependency_management_format is DependencyManagementFormat.POETRY: return PoetryDependencyGetter(self.config, self.package_module_name_map).get() if dependency_management_format is DependencyManagementFormat.PDM: - return PDMDependencyGetter(self.config, self.package_module_name_map).get() + return PDMDependencyGetter( + self.config, self.package_module_name_map, self.pep621_dev_dependency_groups + ).get() if dependency_management_format is DependencyManagementFormat.PEP_621: return PEP621DependencyGetter( self.config, self.package_module_name_map, self.pep621_dev_dependency_groups @@ -191,6 +195,20 @@ def _log_config(self) -> None: logging.debug("%s: %s", key, value) logging.debug("") + @staticmethod + def _log_dependencies(dependencies_extract: DependenciesExtract) -> None: + if dependencies_extract.dependencies: + logging.debug("The project contains the following dependencies:") + for dependency in dependencies_extract.dependencies: + logging.debug(dependency) + logging.debug("") + + if dependencies_extract.dev_dependencies: + logging.debug("The project contains the following dev dependencies:") + for dependency in dependencies_extract.dev_dependencies: + logging.debug(dependency) + logging.debug("") + @staticmethod def _exit(violations: list[Violation]) -> None: sys.exit(bool(violations)) diff --git a/python/deptry/dependency_getter/base.py b/python/deptry/dependency_getter/base.py index c0482636..d4b70a44 100644 --- a/python/deptry/dependency_getter/base.py +++ b/python/deptry/dependency_getter/base.py @@ -1,6 +1,5 @@ from __future__ import annotations -import logging from abc import ABC, abstractmethod from dataclasses import dataclass, field from typing import TYPE_CHECKING @@ -36,12 +35,3 @@ class DependencyGetter(ABC): def get(self) -> DependenciesExtract: """Get extracted dependencies and dev dependencies.""" raise NotImplementedError() - - @staticmethod - def _log_dependencies(dependencies: list[Dependency], is_dev: bool = False) -> None: - logging.debug("The project contains the following %s:", "dev dependencies" if is_dev else "dependencies") - - for dependency in dependencies: - logging.debug(dependency) - - logging.debug("") diff --git a/python/deptry/dependency_getter/pdm.py b/python/deptry/dependency_getter/pdm.py index a18115f4..a79bafdf 100644 --- a/python/deptry/dependency_getter/pdm.py +++ b/python/deptry/dependency_getter/pdm.py @@ -21,10 +21,10 @@ class PDMDependencyGetter(PEP621DependencyGetter): def get(self) -> DependenciesExtract: pep_621_dependencies_extract = super().get() - dev_dependencies = self._get_pdm_dev_dependencies() - self._log_dependencies(dev_dependencies, is_dev=True) - - return DependenciesExtract(pep_621_dependencies_extract.dependencies, dev_dependencies) + return DependenciesExtract( + pep_621_dependencies_extract.dependencies, + [*pep_621_dependencies_extract.dev_dependencies, *self._get_pdm_dev_dependencies()], + ) def _get_pdm_dev_dependencies(self) -> list[Dependency]: """ diff --git a/python/deptry/dependency_getter/pep_621.py b/python/deptry/dependency_getter/pep_621.py index 0e27ddac..7c1cde08 100644 --- a/python/deptry/dependency_getter/pep_621.py +++ b/python/deptry/dependency_getter/pep_621.py @@ -51,12 +51,9 @@ def get(self) -> DependenciesExtract: self._split_development_dependencies_from_optional_dependencies(optional_dependencies) ) dependencies = [*dependencies, *leftover_optional_dependencies] - self._log_dependencies(dependencies) - self._log_dependencies(dev_dependencies, is_dev=True) return DependenciesExtract(dependencies, dev_dependencies) dependencies = [*dependencies, *itertools.chain(*optional_dependencies.values())] - self._log_dependencies(dependencies) return DependenciesExtract(dependencies, []) def _get_dependencies(self) -> list[Dependency]: diff --git a/python/deptry/dependency_getter/poetry.py b/python/deptry/dependency_getter/poetry.py index 31c88f23..c8094729 100644 --- a/python/deptry/dependency_getter/poetry.py +++ b/python/deptry/dependency_getter/poetry.py @@ -17,13 +17,7 @@ class PoetryDependencyGetter(DependencyGetter): """Extract Poetry dependencies from pyproject.toml.""" def get(self) -> DependenciesExtract: - dependencies = self._get_poetry_dependencies() - self._log_dependencies(dependencies) - - dev_dependencies = self._get_poetry_dev_dependencies() - self._log_dependencies(dev_dependencies, is_dev=True) - - return DependenciesExtract(dependencies, dev_dependencies) + return DependenciesExtract(self._get_poetry_dependencies(), self._get_poetry_dev_dependencies()) def _get_poetry_dependencies(self) -> list[Dependency]: pyproject_data = load_pyproject_toml(self.config) diff --git a/python/deptry/dependency_getter/requirements_txt.py b/python/deptry/dependency_getter/requirements_txt.py index 19da3e5d..d747c702 100644 --- a/python/deptry/dependency_getter/requirements_txt.py +++ b/python/deptry/dependency_getter/requirements_txt.py @@ -25,7 +25,6 @@ def get(self) -> DependenciesExtract: *(self._get_dependencies_from_requirements_file(file_name) for file_name in self.requirements_txt) ) ) - self._log_dependencies(dependencies=dependencies) dev_dependencies = list( itertools.chain( @@ -35,7 +34,6 @@ def get(self) -> DependenciesExtract: ) ) ) - self._log_dependencies(dependencies=dev_dependencies, is_dev=True) return DependenciesExtract(dependencies, dev_dependencies) diff --git a/tests/data/project_with_pdm/pyproject.toml b/tests/data/project_with_pdm/pyproject.toml index 90302648..d26df860 100644 --- a/tests/data/project_with_pdm/pyproject.toml +++ b/tests/data/project_with_pdm/pyproject.toml @@ -28,5 +28,8 @@ test = [ "pytest-cov>=4.0.0", ] +[tool.deptry] +pep621_dev_dependency_groups = ["bar"] + [tool.deptry.per_rule_ignores] DEP002 = ["pkginfo"] diff --git a/tests/functional/cli/test_cli.py b/tests/functional/cli/test_cli.py index cfad5a2c..e451703c 100644 --- a/tests/functional/cli/test_cli.py +++ b/tests/functional/cli/test_cli.py @@ -418,6 +418,7 @@ def test_cli_verbose(poetry_venv_factory: PoetryVenvFactory) -> None: assert result.returncode == 1 assert "The project contains the following dependencies:" in result.stderr + assert "The project contains the following dev dependencies:" in result.stderr assert get_issues_report(Path(issue_report)) == [ { "error": { diff --git a/tests/functional/cli/test_cli_pdm.py b/tests/functional/cli/test_cli_pdm.py index 9b317490..b7d2274f 100644 --- a/tests/functional/cli/test_cli_pdm.py +++ b/tests/functional/cli/test_cli_pdm.py @@ -33,18 +33,6 @@ def test_cli_with_pdm(pdm_venv_factory: PDMVenvFactory) -> None: "column": None, }, }, - { - "error": { - "code": "DEP002", - "message": "'requests' defined as a dependency but not used in the codebase", - }, - "module": "requests", - "location": { - "file": str(Path("pyproject.toml")), - "line": None, - "column": None, - }, - }, { "error": { "code": "DEP004", diff --git a/tests/unit/test_core.py b/tests/unit/test_core.py index 159ceea5..2349168d 100644 --- a/tests/unit/test_core.py +++ b/tests/unit/test_core.py @@ -1,5 +1,6 @@ from __future__ import annotations +import logging import re import sys from pathlib import Path @@ -9,6 +10,7 @@ from deptry.core import Core from deptry.dependency import Dependency +from deptry.dependency_getter.base import DependenciesExtract from deptry.exceptions import UnsupportedPythonVersionError from deptry.imports.location import Location from deptry.module import Module @@ -167,3 +169,45 @@ def test__exit_without_violations() -> None: assert e.type == SystemExit assert e.value.code == 0 + + +@pytest.mark.parametrize( + ("dependencies", "dev_dependencies", "expected_logs"), + [ + ( + [], + [], + [], + ), + ( + [ + Dependency("foo", Path("pyproject.toml")), + Dependency("bar", Path("pyproject.toml")), + ], + [ + Dependency("dev", Path("pyproject.toml")), + Dependency("another-dev", Path("pyproject.toml")), + ], + [ + "The project contains the following dependencies:", + "Dependency 'foo' with top-levels: {'foo'}.", + "Dependency 'bar' with top-levels: {'bar'}.", + "", + "The project contains the following dev dependencies:", + "Dependency 'dev' with top-levels: {'dev'}.", + "Dependency 'another-dev' with top-levels: {'another_dev'}.", + "", + ], + ), + ], +) +def test__log_dependencies( + dependencies: list[Dependency], + dev_dependencies: list[Dependency], + expected_logs: list[str], + caplog: pytest.LogCaptureFixture, +) -> None: + with caplog.at_level(logging.DEBUG): + Core._log_dependencies(DependenciesExtract(dependencies, dev_dependencies)) + + assert caplog.messages == expected_logs From c1f26863ae3ba39827b8c137a805fa68c32da047 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Sat, 23 Mar 2024 20:08:51 +0100 Subject: [PATCH 10/12] Update docs/usage.md Co-authored-by: Mathieu Kniewallner --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 5fbce1b3..52e0f8fe 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -410,7 +410,7 @@ By default, all dependencies under `[project.dependencies]` and `[project.option For example, consider a project with the following `pyproject.toml`: -``` +```toml [project] ... dependencies = [ From 6bd79c74874a0eaefb3b6dd052c33613bf561909 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Sat, 23 Mar 2024 20:08:57 +0100 Subject: [PATCH 11/12] Update docs/usage.md Co-authored-by: Mathieu Kniewallner --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index 52e0f8fe..df0efe56 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -30,7 +30,7 @@ To determine the project's dependencies, _deptry_ will scan the directory it is - development dependencies from `[tool.pdm.dev-dependencies]` section. 3. If a `pyproject.toml` file with a `[project]` section is found, _deptry_ will assume it uses [PEP 621](https://peps.python.org/pep-0621/) for dependency specification and extract: - dependencies from `[project.dependencies]` and `[project.optional-dependencies]`. - - development dependecies from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument. + - development dependencies from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument. 4. If a `requirements.txt` file is found, _deptry_ will extract: - dependencies from it - development dependencies from `dev-dependencies.txt` and `dependencies-dev.txt`, if any exist From a3949666dc260da7c23cb7de376ff5520baf9b92 Mon Sep 17 00:00:00 2001 From: Florian Maas Date: Sat, 23 Mar 2024 20:11:40 +0100 Subject: [PATCH 12/12] updated docs --- docs/usage.md | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/docs/usage.md b/docs/usage.md index df0efe56..a4596d97 100644 --- a/docs/usage.md +++ b/docs/usage.md @@ -27,7 +27,7 @@ To determine the project's dependencies, _deptry_ will scan the directory it is - development dependencies from `[tool.poetry.group.dev.dependencies]` or `[tool.poetry.dev-dependencies]` section 2. If a `pyproject.toml` file with a `[tool.pdm.dev-dependencies]` section is found, _deptry_ will assume it uses PDM and extract: - dependencies from `[project.dependencies]` and `[project.optional-dependencies]` sections - - development dependencies from `[tool.pdm.dev-dependencies]` section. + - development dependencies from `[tool.pdm.dev-dependencies]` section and from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument. 3. If a `pyproject.toml` file with a `[project]` section is found, _deptry_ will assume it uses [PEP 621](https://peps.python.org/pep-0621/) for dependency specification and extract: - dependencies from `[project.dependencies]` and `[project.optional-dependencies]`. - development dependencies from the groups under `[project.optional-dependencies]` passed via the [`--pep621-dev-dependency-groups`](#pep-621-dev-dependency-groups) argument.