From 0601e0fb99f3a96681ead633a1e796911b95f1bd Mon Sep 17 00:00:00 2001 From: gram Date: Tue, 23 Mar 2021 08:44:53 +0100 Subject: [PATCH 01/12] make test_input_formats extendable --- tests/conftest.py | 10 ++ tests/test_cli_compile.py | 196 ++++++++++++++++++-------------------- 2 files changed, 105 insertions(+), 101 deletions(-) diff --git a/tests/conftest.py b/tests/conftest.py index e824baf90..f08f958d0 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -322,3 +322,13 @@ def _make_sdist(package_dir, dist_dir, *args): return run_setup_file(package_dir, "sdist", "--dist-dir", str(dist_dir), *args) return _make_sdist + + +@pytest.fixture +def fake_dists(tmpdir, make_package, make_wheel): + dists_path = os.path.join(tmpdir, "dists") + pkg = make_package("small-fake-a", version="0.1") + make_wheel(pkg, dists_path) + pkg = make_package("small-fake-b", version="0.2") + make_wheel(pkg, dists_path) + return dists_path diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index e47eb0b14..b8c6fa289 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -1658,109 +1658,103 @@ def test_triple_equal_pinned_dependency_is_used( assert line in out.stderr -@pytest.mark.network -@pytest.mark.parametrize( - ("fname", "content"), - ( - pytest.param( - "setup.cfg", - """ - [metadata] - name = sample_lib - author = Vincent Driessen - author_email = me@nvie.com - - [options] - packages = find: - install_requires = - small-fake-a==0.1 - small-fake-b==0.2 - - [options.extras_require] - dev = - small-fake-c==0.3 - small-fake-d==0.4 - test = - small-fake-e==0.5 - small-fake-f==0.6 - """, - id="setup.cfg", - ), - pytest.param( - "setup.py", - """ - from setuptools import setup - - setup( - name="sample_lib", - version=0.1, - install_requires=["small-fake-a==0.1", "small-fake-b==0.2"], - extras_require={ - "dev": ["small-fake-c==0.3", "small-fake-d==0.4"], - "test": ["small-fake-e==0.5", "small-fake-f==0.6"], - }, - ) - """, - id="setup.py", - ), - pytest.param( - "pyproject.toml", - """ - [build-system] - requires = ["flit_core >=2,<4"] - build-backend = "flit_core.buildapi" - - [tool.flit.metadata] - module = "sample_lib" - author = "Vincent Driessen" - author-email = "me@nvie.com" - - requires = ["small-fake-a==0.1", "small-fake-b==0.2"] - - [tool.flit.metadata.requires-extra] - dev = ["small-fake-c==0.3", "small-fake-d==0.4"] - test = ["small-fake-e==0.5", "small-fake-f==0.6"] - """, - id="flit", - ), - pytest.param( - "pyproject.toml", - """ - [build-system] - requires = ["poetry_core>=1.0.0"] - build-backend = "poetry.core.masonry.api" - - [tool.poetry] - name = "sample_lib" - version = "0.1.0" - description = "" - authors = ["Vincent Driessen "] - - [tool.poetry.dependencies] - python = "*" - small-fake-a = "0.1" - small-fake-b = "0.2" - - small-fake-c = "0.3" - small-fake-d = "0.4" - small-fake-e = "0.5" - small-fake-f = "0.6" - - [tool.poetry.extras] - dev = ["small-fake-c", "small-fake-d"] - test = ["small-fake-e", "small-fake-f"] - """, - id="poetry", - ), +METADATA_TEST_CASES = ( + pytest.param( + "setup.cfg", + """ + [metadata] + name = sample_lib + author = Vincent Driessen + author_email = me@nvie.com + + [options] + packages = find: + install_requires = + small-fake-a==0.1 + small-fake-b==0.2 + + [options.extras_require] + dev = + small-fake-c==0.3 + small-fake-d==0.4 + test = + small-fake-e==0.5 + small-fake-f==0.6 + """, + id="setup.cfg", + ), + pytest.param( + "setup.py", + """ + from setuptools import setup + + setup( + name="sample_lib", + version=0.1, + install_requires=["small-fake-a==0.1", "small-fake-b==0.2"], + extras_require={ + "dev": ["small-fake-c==0.3", "small-fake-d==0.4"], + "test": ["small-fake-e==0.5", "small-fake-f==0.6"], + }, + ) + """, + id="setup.py", + ), + pytest.param( + "pyproject.toml", + """ + [build-system] + requires = ["flit_core >=2,<4"] + build-backend = "flit_core.buildapi" + + [tool.flit.metadata] + module = "sample_lib" + author = "Vincent Driessen" + author-email = "me@nvie.com" + + requires = ["small-fake-a==0.1", "small-fake-b==0.2"] + + [tool.flit.metadata.requires-extra] + dev = ["small-fake-c==0.3", "small-fake-d==0.4"] + test = ["small-fake-e==0.5", "small-fake-f==0.6"] + """, + id="flit", + ), + pytest.param( + "pyproject.toml", + """ + [build-system] + requires = ["poetry_core>=1.0.0"] + build-backend = "poetry.core.masonry.api" + + [tool.poetry] + name = "sample_lib" + version = "0.1.0" + description = "" + authors = ["Vincent Driessen "] + + [tool.poetry.dependencies] + python = "*" + small-fake-a = "0.1" + small-fake-b = "0.2" + + small-fake-c = "0.3" + small-fake-d = "0.4" + small-fake-e = "0.5" + small-fake-f = "0.6" + + [tool.poetry.extras] + dev = ["small-fake-c", "small-fake-d"] + test = ["small-fake-e", "small-fake-f"] + """, + id="poetry", ), ) -def test_input_formats(make_package, make_wheel, runner, tmpdir, fname, content): - dists_path = os.path.join(tmpdir, "dists") - pkg = make_package("small-fake-a", version="0.1") - make_wheel(pkg, dists_path) - pkg = make_package("small-fake-b", version="0.2") - make_wheel(pkg, dists_path) + +@pytest.mark.network +@pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) +def test_input_formats(fake_dists, runner, tmpdir, fname, content): path = os.path.join(tmpdir, "sample_lib") os.mkdir(path) path = os.path.join(tmpdir, "sample_lib", "__init__.py") @@ -1770,7 +1764,7 @@ def test_input_formats(make_package, make_wheel, runner, tmpdir, fname, content) with open(path, "w") as stream: stream.write(dedent(content)) - out = runner.invoke(cli, ["-n", "--find-links", dists_path, path]) + out = runner.invoke(cli, ["-n", "--find-links", fake_dists, path]) assert out.exit_code == 0, out.stderr assert "small-fake-a==0.1" in out.stderr assert "small-fake-b==0.2" in out.stderr From fef5d60fda9c5f97719aef33574ae8be38be1e43 Mon Sep 17 00:00:00 2001 From: gram Date: Tue, 23 Mar 2021 09:11:16 +0100 Subject: [PATCH 02/12] add --extras --- piptools/scripts/compile.py | 35 ++++++++++++++++++------------- piptools/utils.py | 12 +++++++++++ tests/conftest.py | 34 ++++++++++++++++++++++++++---- tests/test_cli_compile.py | 42 +++++++++++++++++++++++++++---------- 4 files changed, 93 insertions(+), 30 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 8534a0b1f..3333a2ea2 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -19,7 +19,13 @@ from ..repositories import LocalRequirementsRepository, PyPIRepository from ..repositories.base import BaseRepository from ..resolver import Resolver -from ..utils import UNSAFE_PACKAGES, dedup, is_pinned_requirement, key_from_ireq +from ..utils import ( + UNSAFE_PACKAGES, + dedup, + is_pinned_requirement, + key_from_ireq, + req_is_in_extras, +) from ..writer import OutputWriter DEFAULT_REQUIREMENTS_FILE = "requirements.in" @@ -61,6 +67,12 @@ def _get_default_option(option_name: str) -> Any: is_flag=True, help="Clear any caches upfront, rebuild from scratch", ) +@click.option( + "-e", + "--extras", + multiple=True, + help="names of extras_require to install", +) @click.option( "-f", "--find-links", @@ -204,22 +216,23 @@ def cli( dry_run: bool, pre: bool, rebuild: bool, - find_links: Tuple[str], + extras: Tuple[str, ...], + find_links: Tuple[str, ...], index_url: str, - extra_index_url: Tuple[str], + extra_index_url: Tuple[str, ...], cert: Optional[str], client_cert: Optional[str], - trusted_host: Tuple[str], + trusted_host: Tuple[str, ...], header: bool, emit_trusted_host: bool, annotate: bool, upgrade: bool, - upgrade_packages: Tuple[str], + upgrade_packages: Tuple[str, ...], output_file: Optional[LazyFile], allow_unsafe: bool, generate_hashes: bool, reuse_hashes: bool, - src_files: Tuple[str], + src_files: Tuple[str, ...], max_rounds: int, build_isolation: bool, emit_find_links: bool, @@ -386,15 +399,7 @@ def cli( ) # Filter out pip environment markers which do not match (PEP496) - constraints = [ - req - for req in constraints - if req.markers is None - # We explicitly set extra=None to filter out optional requirements - # since evaluating an extra marker with no environment raises UndefinedEnvironmentName - # (see https://packaging.pypa.io/en/latest/markers.html#usage) - or req.markers.evaluate({"extra": None}) - ] + constraints = [req for req in constraints if req_is_in_extras(req, extras=extras)] log.debug("Using indexes:") with log.indentation(): diff --git a/piptools/utils.py b/piptools/utils.py index d58b5c067..519da6072 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -163,6 +163,18 @@ def is_pinned_requirement(ireq: InstallRequirement) -> bool: return spec.operator in {"==", "==="} and not spec.version.endswith(".*") +def req_is_in_extras(ireq: InstallRequirement, extras: Tuple[str, ...]) -> bool: + """ + Check if the requirement isn't extra or is included into extras to install. + """ + if not ireq.markers or ireq.markers.evaluate({"extra": None}): + return True + for extra in extras: + if ireq.markers.evaluate({"extra": extra}): + return True + return False + + def as_tuple(ireq: InstallRequirement) -> Tuple[str, str, Tuple[str, ...]]: """ Pulls out the (name: str, version:str, extras:(str)) tuple from diff --git a/tests/conftest.py b/tests/conftest.py index f08f958d0..3dbf22126 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -324,11 +324,37 @@ def _make_sdist(package_dir, dist_dir, *args): return _make_sdist +@pytest.fixture +def make_module(tmpdir): + """ + Make a metadata file with the given name and content and a fake module. + """ + + def _make_module(fname, content): + path = os.path.join(tmpdir, "sample_lib") + os.mkdir(path) + path = os.path.join(tmpdir, "sample_lib", "__init__.py") + with open(path, "w") as stream: + stream.write("'example module'\n__version__ = '1.2.3'") + path = os.path.join(tmpdir, fname) + with open(path, "w") as stream: + stream.write(dedent(content)) + return path + + return _make_module + + @pytest.fixture def fake_dists(tmpdir, make_package, make_wheel): dists_path = os.path.join(tmpdir, "dists") - pkg = make_package("small-fake-a", version="0.1") - make_wheel(pkg, dists_path) - pkg = make_package("small-fake-b", version="0.2") - make_wheel(pkg, dists_path) + pkgs = [ + make_package("small-fake-a", version="0.1"), + make_package("small-fake-b", version="0.2"), + make_package("small-fake-c", version="0.3"), + make_package("small-fake-d", version="0.4"), + make_package("small-fake-e", version="0.5"), + make_package("small-fake-f", version="0.6"), + ] + for pkg in pkgs: + make_wheel(pkg, dists_path) return dists_path diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index b8c6fa289..992eb4998 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -1754,17 +1754,9 @@ def test_triple_equal_pinned_dependency_is_used( @pytest.mark.network @pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) -def test_input_formats(fake_dists, runner, tmpdir, fname, content): - path = os.path.join(tmpdir, "sample_lib") - os.mkdir(path) - path = os.path.join(tmpdir, "sample_lib", "__init__.py") - with open(path, "w") as stream: - stream.write("'example module'\n__version__ = '1.2.3'") - path = os.path.join(tmpdir, fname) - with open(path, "w") as stream: - stream.write(dedent(content)) - - out = runner.invoke(cli, ["-n", "--find-links", fake_dists, path]) +def test_input_formats(fake_dists, runner, make_module, fname, content): + meta_path = make_module(fname=fname, content=content) + out = runner.invoke(cli, ["-n", "--find-links", fake_dists, meta_path]) assert out.exit_code == 0, out.stderr assert "small-fake-a==0.1" in out.stderr assert "small-fake-b==0.2" in out.stderr @@ -1772,3 +1764,31 @@ def test_input_formats(fake_dists, runner, tmpdir, fname, content): assert "small-fake-d" not in out.stderr assert "small-fake-e" not in out.stderr assert "small-fake-f" not in out.stderr + + +@pytest.mark.network +@pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) +def test_one_extra(fake_dists, runner, make_module, fname, content): + meta_path = make_module(fname=fname, content=content) + out = runner.invoke(cli, ["-n", "-e", "dev", "--find-links", fake_dists, meta_path]) + assert out.exit_code == 0, out.stderr + assert "small-fake-a==0.1" in out.stderr + assert "small-fake-b==0.2" in out.stderr + assert "small-fake-c==0.3" in out.stderr + assert "small-fake-d==0.4" in out.stderr + assert "small-fake-e" not in out.stderr + assert "small-fake-f" not in out.stderr + + +@pytest.mark.network +@pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) +def test_multiple_extras(fake_dists, runner, make_module, fname, content): + meta_path = make_module(fname=fname, content=content) + out = runner.invoke(cli, ["-n", "-e", "dev", "--find-links", fake_dists, meta_path]) + assert out.exit_code == 0, out.stderr + assert "small-fake-a==0.1" in out.stderr + assert "small-fake-b==0.2" in out.stderr + assert "small-fake-c==0.3" in out.stderr + assert "small-fake-d==0.4" in out.stderr + assert "small-fake-e" not in out.stderr + assert "small-fake-f" not in out.stderr From 8d74a8b3a21e030088683195c4c8adde2dff85ac Mon Sep 17 00:00:00 2001 From: gram Date: Tue, 23 Mar 2021 10:43:20 +0100 Subject: [PATCH 03/12] test multiple -e support --- tests/test_cli_compile.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 992eb4998..2d4284cb3 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -1784,11 +1784,13 @@ def test_one_extra(fake_dists, runner, make_module, fname, content): @pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) def test_multiple_extras(fake_dists, runner, make_module, fname, content): meta_path = make_module(fname=fname, content=content) - out = runner.invoke(cli, ["-n", "-e", "dev", "--find-links", fake_dists, meta_path]) + out = runner.invoke( + cli, ["-n", "-e", "dev", "-e", "test", "--find-links", fake_dists, meta_path] + ) assert out.exit_code == 0, out.stderr assert "small-fake-a==0.1" in out.stderr assert "small-fake-b==0.2" in out.stderr assert "small-fake-c==0.3" in out.stderr assert "small-fake-d==0.4" in out.stderr - assert "small-fake-e" not in out.stderr - assert "small-fake-f" not in out.stderr + assert "small-fake-e==0.5" in out.stderr + assert "small-fake-f==0.6" in out.stderr From 146a538caa5d809837b7c1862438f307d35eccf6 Mon Sep 17 00:00:00 2001 From: gram Date: Tue, 23 Mar 2021 13:10:21 +0100 Subject: [PATCH 04/12] rename the function --- piptools/scripts/compile.py | 5 ++--- piptools/utils.py | 5 +++-- 2 files changed, 5 insertions(+), 5 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 3333a2ea2..560dd2847 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -24,7 +24,7 @@ dedup, is_pinned_requirement, key_from_ireq, - req_is_in_extras, + req_check_markers, ) from ..writer import OutputWriter @@ -398,8 +398,7 @@ def cli( ireq for key, ireq in upgrade_install_reqs.items() if key in allowed_upgrades ) - # Filter out pip environment markers which do not match (PEP496) - constraints = [req for req in constraints if req_is_in_extras(req, extras=extras)] + constraints = [req for req in constraints if req_check_markers(req, extras=extras)] log.debug("Using indexes:") with log.indentation(): diff --git a/piptools/utils.py b/piptools/utils.py index 519da6072..59ebb8977 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -163,9 +163,10 @@ def is_pinned_requirement(ireq: InstallRequirement) -> bool: return spec.operator in {"==", "==="} and not spec.version.endswith(".*") -def req_is_in_extras(ireq: InstallRequirement, extras: Tuple[str, ...]) -> bool: +def req_check_markers(ireq: InstallRequirement, extras: Tuple[str, ...]) -> bool: """ - Check if the requirement isn't extra or is included into extras to install. + 1. Check if the environment markers match (PEP-496). + 2. Check if the requirement isn't extra or is included into extras to install. """ if not ireq.markers or ireq.markers.evaluate({"extra": None}): return True From 0a7838133dcdfab7ed76c51520ec386bd0424c05 Mon Sep 17 00:00:00 2001 From: gram Date: Wed, 24 Mar 2021 11:16:03 +0100 Subject: [PATCH 05/12] rename -e/--extras into --extra --- piptools/scripts/compile.py | 4 ++-- tests/test_cli_compile.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 560dd2847..b263fe2a1 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -68,8 +68,8 @@ def _get_default_option(option_name: str) -> Any: help="Clear any caches upfront, rebuild from scratch", ) @click.option( - "-e", - "--extras", + "--extra", + "extras", multiple=True, help="names of extras_require to install", ) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 2d4284cb3..ff9ffec21 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -1770,7 +1770,7 @@ def test_input_formats(fake_dists, runner, make_module, fname, content): @pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) def test_one_extra(fake_dists, runner, make_module, fname, content): meta_path = make_module(fname=fname, content=content) - out = runner.invoke(cli, ["-n", "-e", "dev", "--find-links", fake_dists, meta_path]) + out = runner.invoke(cli, ["-n", "--extra", "dev", "--find-links", fake_dists, meta_path]) assert out.exit_code == 0, out.stderr assert "small-fake-a==0.1" in out.stderr assert "small-fake-b==0.2" in out.stderr @@ -1785,7 +1785,7 @@ def test_one_extra(fake_dists, runner, make_module, fname, content): def test_multiple_extras(fake_dists, runner, make_module, fname, content): meta_path = make_module(fname=fname, content=content) out = runner.invoke( - cli, ["-n", "-e", "dev", "-e", "test", "--find-links", fake_dists, meta_path] + cli, ["-n", "--extra", "dev", "--extra", "test", "--find-links", fake_dists, meta_path] ) assert out.exit_code == 0, out.stderr assert "small-fake-a==0.1" in out.stderr From 50f29aedc9a4f2507c04d4abc244c1ad79f530e5 Mon Sep 17 00:00:00 2001 From: gram Date: Wed, 24 Mar 2021 14:02:08 +0100 Subject: [PATCH 06/12] black --- tests/test_cli_compile.py | 16 ++++++++++++++-- 1 file changed, 14 insertions(+), 2 deletions(-) diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index ff9ffec21..868b98cf9 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -1770,7 +1770,9 @@ def test_input_formats(fake_dists, runner, make_module, fname, content): @pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) def test_one_extra(fake_dists, runner, make_module, fname, content): meta_path = make_module(fname=fname, content=content) - out = runner.invoke(cli, ["-n", "--extra", "dev", "--find-links", fake_dists, meta_path]) + out = runner.invoke( + cli, ["-n", "--extra", "dev", "--find-links", fake_dists, meta_path] + ) assert out.exit_code == 0, out.stderr assert "small-fake-a==0.1" in out.stderr assert "small-fake-b==0.2" in out.stderr @@ -1785,7 +1787,17 @@ def test_one_extra(fake_dists, runner, make_module, fname, content): def test_multiple_extras(fake_dists, runner, make_module, fname, content): meta_path = make_module(fname=fname, content=content) out = runner.invoke( - cli, ["-n", "--extra", "dev", "--extra", "test", "--find-links", fake_dists, meta_path] + cli, + [ + "-n", + "--extra", + "dev", + "--extra", + "test", + "--find-links", + fake_dists, + meta_path, + ], ) assert out.exit_code == 0, out.stderr assert "small-fake-a==0.1" in out.stderr From 99719d34532903f86d4cae045c5f9bb7b97c3c3b Mon Sep 17 00:00:00 2001 From: gram Date: Wed, 31 Mar 2021 13:30:25 +0200 Subject: [PATCH 07/12] fail on --extra with requirements.in --- piptools/scripts/compile.py | 6 ++++++ tests/test_cli_compile.py | 10 ++++++++++ 2 files changed, 16 insertions(+) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index b263fe2a1..220fc4867 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -348,6 +348,7 @@ def cli( ### constraints = [] + setup_file_found = False for src_file in src_files: is_setup_file = os.path.basename(src_file) in METADATA_FILENAMES if src_file == "-": @@ -371,6 +372,7 @@ def cli( req.comes_from = comes_from constraints.extend(reqs) elif is_setup_file: + setup_file_found = True dist = meta.load(os.path.dirname(os.path.abspath(src_file))) comes_from = f"{dist.metadata.get_all('Name')[0]} ({src_file})" constraints.extend( @@ -389,6 +391,10 @@ def cli( ) ) + if not setup_file_found: + msg = "--extra has effect only with setup.py and PEP-517 input formats" + raise click.BadParameter(msg) + primary_packages = { key_from_ireq(ireq) for ireq in constraints if not ireq.constraint } diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 868b98cf9..9c181bbe3 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -1806,3 +1806,13 @@ def test_multiple_extras(fake_dists, runner, make_module, fname, content): assert "small-fake-d==0.4" in out.stderr assert "small-fake-e==0.5" in out.stderr assert "small-fake-f==0.6" in out.stderr + + +def test_extras_fail_with_requirements_in(runner, tmpdir): + path = os.path.join(tmpdir, "requirements.in") + with open(path, "w") as stream: + stream.write("\n") + out = runner.invoke(cli, ["-n", "--extra", "something", path]) + assert out.exit_code == 2 + exp = "--extra has effect only with setup.py and PEP-517 input formats" + assert exp in out.stderr From a93377690c3922d2b37568f7251b196293f69732 Mon Sep 17 00:00:00 2001 From: gram Date: Wed, 31 Mar 2021 15:19:52 +0200 Subject: [PATCH 08/12] add docstrings for tests --- piptools/scripts/compile.py | 2 +- tests/conftest.py | 3 +++ tests/test_cli_compile.py | 12 ++++++++++++ 3 files changed, 16 insertions(+), 1 deletion(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 9f098dd91..2e8910f52 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -71,7 +71,7 @@ def _get_default_option(option_name: str) -> Any: "--extra", "extras", multiple=True, - help="names of extras_require to install", + help="Names of extras_require to install", ) @click.option( "-f", diff --git a/tests/conftest.py b/tests/conftest.py index 3dbf22126..3ea43ff24 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -346,6 +346,9 @@ def _make_module(fname, content): @pytest.fixture def fake_dists(tmpdir, make_package, make_wheel): + """ + Generate distribution packages `small-fake-{a..f}` + """ dists_path = os.path.join(tmpdir, "dists") pkgs = [ make_package("small-fake-a", version="0.1"), diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 9c181bbe3..206f2a9a9 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -1755,6 +1755,9 @@ def test_triple_equal_pinned_dependency_is_used( @pytest.mark.network @pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) def test_input_formats(fake_dists, runner, make_module, fname, content): + """ + Test different dependency formats as input file. + """ meta_path = make_module(fname=fname, content=content) out = runner.invoke(cli, ["-n", "--find-links", fake_dists, meta_path]) assert out.exit_code == 0, out.stderr @@ -1769,6 +1772,9 @@ def test_input_formats(fake_dists, runner, make_module, fname, content): @pytest.mark.network @pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) def test_one_extra(fake_dists, runner, make_module, fname, content): + """ + Test one `--extra` (dev) passed, other extras (test) must be ignored. + """ meta_path = make_module(fname=fname, content=content) out = runner.invoke( cli, ["-n", "--extra", "dev", "--find-links", fake_dists, meta_path] @@ -1785,6 +1791,9 @@ def test_one_extra(fake_dists, runner, make_module, fname, content): @pytest.mark.network @pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) def test_multiple_extras(fake_dists, runner, make_module, fname, content): + """ + Test passing multiple `--extra` params. + """ meta_path = make_module(fname=fname, content=content) out = runner.invoke( cli, @@ -1809,6 +1818,9 @@ def test_multiple_extras(fake_dists, runner, make_module, fname, content): def test_extras_fail_with_requirements_in(runner, tmpdir): + """ + Test that passing `--extra` with `requirements.in` input file fails. + """ path = os.path.join(tmpdir, "requirements.in") with open(path, "w") as stream: stream.write("\n") From e364b8453467e1e64e651ead76acda2caa9ef918 Mon Sep 17 00:00:00 2001 From: gram Date: Wed, 31 Mar 2021 15:22:33 +0200 Subject: [PATCH 09/12] fix tests --- piptools/scripts/compile.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 2e8910f52..c9a6cfc56 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -393,7 +393,7 @@ def cli( ) ) - if not setup_file_found: + if extras and not setup_file_found: msg = "--extra has effect only with setup.py and PEP-517 input formats" raise click.BadParameter(msg) From 2e0505324c543c787f0531635abd3e39fc14d61b Mon Sep 17 00:00:00 2001 From: gram Date: Fri, 2 Apr 2021 10:38:56 +0200 Subject: [PATCH 10/12] simplify extras matching --- piptools/scripts/compile.py | 15 +++++---------- piptools/utils.py | 13 ------------- 2 files changed, 5 insertions(+), 23 deletions(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index c9a6cfc56..eac222698 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -2,12 +2,13 @@ import shlex import sys import tempfile -from typing import Any, BinaryIO, Optional, Tuple, cast +from typing import Any, BinaryIO, List, Optional, Tuple, cast import click from click.utils import LazyFile, safecall from pep517 import meta from pip._internal.commands import create_command +from pip._internal.req import InstallRequirement from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.misc import redact_auth_from_url @@ -19,13 +20,7 @@ from ..repositories import LocalRequirementsRepository, PyPIRepository from ..repositories.base import BaseRepository from ..resolver import Resolver -from ..utils import ( - UNSAFE_PACKAGES, - dedup, - is_pinned_requirement, - key_from_ireq, - req_check_markers, -) +from ..utils import UNSAFE_PACKAGES, dedup, is_pinned_requirement, key_from_ireq from ..writer import OutputWriter DEFAULT_REQUIREMENTS_FILE = "requirements.in" @@ -349,7 +344,7 @@ def cli( # Parsing/collecting initial requirements ### - constraints = [] + constraints: List[InstallRequirement] = [] setup_file_found = False for src_file in src_files: is_setup_file = os.path.basename(src_file) in METADATA_FILENAMES @@ -406,7 +401,7 @@ def cli( ireq for key, ireq in upgrade_install_reqs.items() if key in allowed_upgrades ) - constraints = [req for req in constraints if req_check_markers(req, extras=extras)] + constraints = [req for req in constraints if req.match_markers(extras)] log.debug("Using indexes:") with log.indentation(): diff --git a/piptools/utils.py b/piptools/utils.py index 59ebb8977..d58b5c067 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -163,19 +163,6 @@ def is_pinned_requirement(ireq: InstallRequirement) -> bool: return spec.operator in {"==", "==="} and not spec.version.endswith(".*") -def req_check_markers(ireq: InstallRequirement, extras: Tuple[str, ...]) -> bool: - """ - 1. Check if the environment markers match (PEP-496). - 2. Check if the requirement isn't extra or is included into extras to install. - """ - if not ireq.markers or ireq.markers.evaluate({"extra": None}): - return True - for extra in extras: - if ireq.markers.evaluate({"extra": extra}): - return True - return False - - def as_tuple(ireq: InstallRequirement) -> Tuple[str, str, Tuple[str, ...]]: """ Pulls out the (name: str, version:str, extras:(str)) tuple from From 3b63aa94370185094b6a827defcd2bc9b2551b22 Mon Sep 17 00:00:00 2001 From: gram Date: Tue, 6 Apr 2021 14:54:47 +0200 Subject: [PATCH 11/12] drop `extra` from markers --- piptools/scripts/compile.py | 10 ++++++- piptools/utils.py | 51 ++++++++++++++++++++++++++++++++++++ tests/test_cli_compile.py | 3 +++ tests/test_utils.py | 52 +++++++++++++++++++++++++++++++++++++ 4 files changed, 115 insertions(+), 1 deletion(-) diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index eac222698..b880b80eb 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -20,7 +20,13 @@ from ..repositories import LocalRequirementsRepository, PyPIRepository from ..repositories.base import BaseRepository from ..resolver import Resolver -from ..utils import UNSAFE_PACKAGES, dedup, is_pinned_requirement, key_from_ireq +from ..utils import ( + UNSAFE_PACKAGES, + dedup, + drop_extras, + is_pinned_requirement, + key_from_ireq, +) from ..writer import OutputWriter DEFAULT_REQUIREMENTS_FILE = "requirements.in" @@ -402,6 +408,8 @@ def cli( ) constraints = [req for req in constraints if req.match_markers(extras)] + for req in constraints: + drop_extras(req) log.debug("Using indexes:") with log.indentation(): diff --git a/piptools/utils.py b/piptools/utils.py index d58b5c067..2e92772a9 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -6,6 +6,7 @@ Dict, Iterable, Iterator, + List, Optional, Set, Tuple, @@ -210,6 +211,56 @@ def dedup(iterable: Iterable[_T]) -> Iterable[_T]: return iter(dict.fromkeys(iterable)) +def drop_extras(ireq: InstallRequirement) -> None: + """Remove "extra" markers (PEP-508) from requirement.""" + if ireq.markers is None: + return + ireq.markers._markers = _drop_extras(ireq.markers._markers) + if not ireq.markers._markers: + ireq.markers = None + + +def _drop_extras(markers: List[_T]) -> List[_T]: + # drop `extra` tokens + to_remove: List[int] = [] + for i, token in enumerate(markers): + # operator (and/or) + if isinstance(token, str): + continue + # sub-expression (inside braces) + if isinstance(token, list): + markers[i] = _drop_extras(token) # type: ignore + if not markers[i]: + to_remove.append(i) + continue + # test expression (like `extra == "dev"`) + assert isinstance(token, tuple) + if token[0].value == "extra": + to_remove.append(i) + for i in reversed(to_remove): + markers.pop(i) + + # drop duplicate bool operators (and/or) + to_remove = [] + for i, (token1, token2) in enumerate(zip(markers, markers[1:])): + if not isinstance(token1, str): + continue + if not isinstance(token2, str): + continue + if token1 == "and": + to_remove.append(i) + else: + to_remove.append(i + 1) + for i in reversed(to_remove): + markers.pop(i) + if markers and isinstance(markers[0], str): + markers.pop(0) + if markers and isinstance(markers[-1], str): + markers.pop(-1) + + return markers + + def get_hashes_from_ireq(ireq: InstallRequirement) -> Set[str]: """ Given an InstallRequirement, return a set of string hashes in the format diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index 206f2a9a9..f52281657 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -1767,6 +1767,7 @@ def test_input_formats(fake_dists, runner, make_module, fname, content): assert "small-fake-d" not in out.stderr assert "small-fake-e" not in out.stderr assert "small-fake-f" not in out.stderr + assert "extra ==" not in out.stderr @pytest.mark.network @@ -1786,6 +1787,7 @@ def test_one_extra(fake_dists, runner, make_module, fname, content): assert "small-fake-d==0.4" in out.stderr assert "small-fake-e" not in out.stderr assert "small-fake-f" not in out.stderr + assert "extra ==" not in out.stderr @pytest.mark.network @@ -1815,6 +1817,7 @@ def test_multiple_extras(fake_dists, runner, make_module, fname, content): assert "small-fake-d==0.4" in out.stderr assert "small-fake-e==0.5" in out.stderr assert "small-fake-f==0.6" in out.stderr + assert "extra ==" not in out.stderr def test_extras_fail_with_requirements_in(runner, tmpdir): diff --git a/tests/test_utils.py b/tests/test_utils.py index d6f245c3d..becfc878f 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,6 +9,7 @@ from piptools.utils import ( as_tuple, dedup, + drop_extras, flat_map, format_requirement, format_specifier, @@ -367,3 +368,54 @@ def test_lookup_table_from_tuples_with_empty_values(): def test_lookup_table_with_empty_values(): assert lookup_table((), operator.itemgetter(0)) == {} + + +@pytest.mark.parametrize( + ("given", "expected"), + ( + ("", None), + ("extra == 'dev'", None), + ("extra == 'dev' or extra == 'test'", None), + ("os_name == 'nt' and extra == 'dev'", "os_name == 'nt'"), + ("extra == 'dev' and os_name == 'nt'", "os_name == 'nt'"), + ("os_name == 'nt' or extra == 'dev'", "os_name == 'nt'"), + ("extra == 'dev' or os_name == 'nt'", "os_name == 'nt'"), + ("(extra == 'dev') or os_name == 'nt'", "os_name == 'nt'"), + ("os_name == 'nt' and (extra == 'dev' or extra == 'test')", "os_name == 'nt'"), + ("os_name == 'nt' or (extra == 'dev' or extra == 'test')", "os_name == 'nt'"), + ("(extra == 'dev' or extra == 'test') or os_name == 'nt'", "os_name == 'nt'"), + ("(extra == 'dev' or extra == 'test') and os_name == 'nt'", "os_name == 'nt'"), + ( + "os_name == 'nt' or (os_name == 'unix' and extra == 'test')", + "os_name == 'nt' or os_name == 'unix'", + ), + ( + "(os_name == 'unix' and extra == 'test') or os_name == 'nt'", + "os_name == 'unix' or os_name == 'nt'", + ), + ( + "(os_name == 'unix' or extra == 'test') and os_name == 'nt'", + "os_name == 'unix' and os_name == 'nt'", + ), + ( + "(os_name == 'unix' and extra == 'test' or python_version < '3.5')" + " or os_name == 'nt'", + "(os_name == 'unix' or python_version < '3.5') or os_name == 'nt'", + ), + ( + "os_name == 'unix' and extra == 'test' or os_name == 'nt'", + "os_name == 'unix' or os_name == 'nt'", + ), + ( + "os_name == 'unix' or extra == 'test' and os_name == 'nt'", + "os_name == 'unix' or os_name == 'nt'", + ), + ), +) +def test_drop_extras(from_line, given, expected): + ireq = from_line(f"test;{given}") + drop_extras(ireq) + if expected is None: + assert ireq.markers is None + else: + assert str(ireq.markers).replace("'", '"') == expected.replace("'", '"') From 4881eccc7724b684c8b7e6962af6a84a2b0e7b7c Mon Sep 17 00:00:00 2001 From: gram Date: Tue, 6 Apr 2021 15:04:03 +0200 Subject: [PATCH 12/12] workaround for coverage.py missed branch coverage --- piptools/utils.py | 5 +++-- tests/test_utils.py | 4 ++++ 2 files changed, 7 insertions(+), 2 deletions(-) diff --git a/piptools/utils.py b/piptools/utils.py index 2e92772a9..b4841890a 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -230,8 +230,9 @@ def _drop_extras(markers: List[_T]) -> List[_T]: # sub-expression (inside braces) if isinstance(token, list): markers[i] = _drop_extras(token) # type: ignore - if not markers[i]: - to_remove.append(i) + if markers[i]: + continue + to_remove.append(i) continue # test expression (like `extra == "dev"`) assert isinstance(token, tuple) diff --git a/tests/test_utils.py b/tests/test_utils.py index becfc878f..e143a46d4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -397,6 +397,10 @@ def test_lookup_table_with_empty_values(): "(os_name == 'unix' or extra == 'test') and os_name == 'nt'", "os_name == 'unix' and os_name == 'nt'", ), + ( + "(os_name == 'unix' or os_name == 'nt') and extra == 'dev'", + "os_name == 'unix' or os_name == 'nt'", + ), ( "(os_name == 'unix' and extra == 'test' or python_version < '3.5')" " or os_name == 'nt'",