diff --git a/piptools/scripts/compile.py b/piptools/scripts/compile.py index 6c036d5ae..b880b80eb 100755 --- a/piptools/scripts/compile.py +++ b/piptools/scripts/compile.py @@ -2,12 +2,13 @@ import shlex import sys import tempfile -from typing import Any, BinaryIO, Optional, Tuple, cast +from typing import Any, BinaryIO, List, Optional, Tuple, cast import click from click.utils import LazyFile, safecall from pep517 import meta from pip._internal.commands import create_command +from pip._internal.req import InstallRequirement from pip._internal.req.constructors import install_req_from_line from pip._internal.utils.misc import redact_auth_from_url @@ -19,7 +20,13 @@ from ..repositories import LocalRequirementsRepository, PyPIRepository from ..repositories.base import BaseRepository from ..resolver import Resolver -from ..utils import UNSAFE_PACKAGES, dedup, is_pinned_requirement, key_from_ireq +from ..utils import ( + UNSAFE_PACKAGES, + dedup, + drop_extras, + is_pinned_requirement, + key_from_ireq, +) from ..writer import OutputWriter DEFAULT_REQUIREMENTS_FILE = "requirements.in" @@ -61,6 +68,12 @@ def _get_default_option(option_name: str) -> Any: is_flag=True, help="Clear any caches upfront, rebuild from scratch", ) +@click.option( + "--extra", + "extras", + multiple=True, + help="Names of extras_require to install", +) @click.option( "-f", "--find-links", @@ -206,22 +219,23 @@ def cli( dry_run: bool, pre: bool, rebuild: bool, - find_links: Tuple[str], + extras: Tuple[str, ...], + find_links: Tuple[str, ...], index_url: str, - extra_index_url: Tuple[str], + extra_index_url: Tuple[str, ...], cert: Optional[str], client_cert: Optional[str], - trusted_host: Tuple[str], + trusted_host: Tuple[str, ...], header: bool, emit_trusted_host: bool, annotate: bool, upgrade: bool, - upgrade_packages: Tuple[str], + upgrade_packages: Tuple[str, ...], output_file: Optional[LazyFile], allow_unsafe: bool, generate_hashes: bool, reuse_hashes: bool, - src_files: Tuple[str], + src_files: Tuple[str, ...], max_rounds: int, build_isolation: bool, emit_find_links: bool, @@ -336,7 +350,8 @@ def cli( # Parsing/collecting initial requirements ### - constraints = [] + constraints: List[InstallRequirement] = [] + setup_file_found = False for src_file in src_files: is_setup_file = os.path.basename(src_file) in METADATA_FILENAMES if src_file == "-": @@ -360,6 +375,7 @@ def cli( req.comes_from = comes_from constraints.extend(reqs) elif is_setup_file: + setup_file_found = True dist = meta.load(os.path.dirname(os.path.abspath(src_file))) comes_from = f"{dist.metadata.get_all('Name')[0]} ({src_file})" constraints.extend( @@ -378,6 +394,10 @@ def cli( ) ) + if extras and not setup_file_found: + msg = "--extra has effect only with setup.py and PEP-517 input formats" + raise click.BadParameter(msg) + primary_packages = { key_from_ireq(ireq) for ireq in constraints if not ireq.constraint } @@ -387,16 +407,9 @@ def cli( ireq for key, ireq in upgrade_install_reqs.items() if key in allowed_upgrades ) - # Filter out pip environment markers which do not match (PEP496) - constraints = [ - req - for req in constraints - if req.markers is None - # We explicitly set extra=None to filter out optional requirements - # since evaluating an extra marker with no environment raises UndefinedEnvironmentName - # (see https://packaging.pypa.io/en/latest/markers.html#usage) - or req.markers.evaluate({"extra": None}) - ] + constraints = [req for req in constraints if req.match_markers(extras)] + for req in constraints: + drop_extras(req) log.debug("Using indexes:") with log.indentation(): diff --git a/piptools/utils.py b/piptools/utils.py index d58b5c067..b4841890a 100644 --- a/piptools/utils.py +++ b/piptools/utils.py @@ -6,6 +6,7 @@ Dict, Iterable, Iterator, + List, Optional, Set, Tuple, @@ -210,6 +211,57 @@ def dedup(iterable: Iterable[_T]) -> Iterable[_T]: return iter(dict.fromkeys(iterable)) +def drop_extras(ireq: InstallRequirement) -> None: + """Remove "extra" markers (PEP-508) from requirement.""" + if ireq.markers is None: + return + ireq.markers._markers = _drop_extras(ireq.markers._markers) + if not ireq.markers._markers: + ireq.markers = None + + +def _drop_extras(markers: List[_T]) -> List[_T]: + # drop `extra` tokens + to_remove: List[int] = [] + for i, token in enumerate(markers): + # operator (and/or) + if isinstance(token, str): + continue + # sub-expression (inside braces) + if isinstance(token, list): + markers[i] = _drop_extras(token) # type: ignore + if markers[i]: + continue + to_remove.append(i) + continue + # test expression (like `extra == "dev"`) + assert isinstance(token, tuple) + if token[0].value == "extra": + to_remove.append(i) + for i in reversed(to_remove): + markers.pop(i) + + # drop duplicate bool operators (and/or) + to_remove = [] + for i, (token1, token2) in enumerate(zip(markers, markers[1:])): + if not isinstance(token1, str): + continue + if not isinstance(token2, str): + continue + if token1 == "and": + to_remove.append(i) + else: + to_remove.append(i + 1) + for i in reversed(to_remove): + markers.pop(i) + if markers and isinstance(markers[0], str): + markers.pop(0) + if markers and isinstance(markers[-1], str): + markers.pop(-1) + + return markers + + def get_hashes_from_ireq(ireq: InstallRequirement) -> Set[str]: """ Given an InstallRequirement, return a set of string hashes in the format diff --git a/tests/conftest.py b/tests/conftest.py index e824baf90..3ea43ff24 100644 --- a/tests/conftest.py +++ b/tests/conftest.py @@ -322,3 +322,42 @@ def _make_sdist(package_dir, dist_dir, *args): return run_setup_file(package_dir, "sdist", "--dist-dir", str(dist_dir), *args) return _make_sdist + + +@pytest.fixture +def make_module(tmpdir): + """ + Make a metadata file with the given name and content and a fake module. + """ + + def _make_module(fname, content): + path = os.path.join(tmpdir, "sample_lib") + os.mkdir(path) + path = os.path.join(tmpdir, "sample_lib", "__init__.py") + with open(path, "w") as stream: + stream.write("'example module'\n__version__ = '1.2.3'") + path = os.path.join(tmpdir, fname) + with open(path, "w") as stream: + stream.write(dedent(content)) + return path + + return _make_module + + +@pytest.fixture +def fake_dists(tmpdir, make_package, make_wheel): + """ + Generate distribution packages `small-fake-{a..f}` + """ + dists_path = os.path.join(tmpdir, "dists") + pkgs = [ + make_package("small-fake-a", version="0.1"), + make_package("small-fake-b", version="0.2"), + make_package("small-fake-c", version="0.3"), + make_package("small-fake-d", version="0.4"), + make_package("small-fake-e", version="0.5"), + make_package("small-fake-f", version="0.6"), + ] + for pkg in pkgs: + make_wheel(pkg, dists_path) + return dists_path diff --git a/tests/test_cli_compile.py b/tests/test_cli_compile.py index e47eb0b14..f52281657 100644 --- a/tests/test_cli_compile.py +++ b/tests/test_cli_compile.py @@ -1658,119 +1658,108 @@ def test_triple_equal_pinned_dependency_is_used( assert line in out.stderr -@pytest.mark.network -@pytest.mark.parametrize( - ("fname", "content"), - ( - pytest.param( - "setup.cfg", - """ - [metadata] - name = sample_lib - author = Vincent Driessen - author_email = me@nvie.com - - [options] - packages = find: - install_requires = - small-fake-a==0.1 - small-fake-b==0.2 - - [options.extras_require] - dev = - small-fake-c==0.3 - small-fake-d==0.4 - test = - small-fake-e==0.5 - small-fake-f==0.6 - """, - id="setup.cfg", - ), - pytest.param( - "setup.py", - """ - from setuptools import setup - - setup( - name="sample_lib", - version=0.1, - install_requires=["small-fake-a==0.1", "small-fake-b==0.2"], - extras_require={ - "dev": ["small-fake-c==0.3", "small-fake-d==0.4"], - "test": ["small-fake-e==0.5", "small-fake-f==0.6"], - }, - ) - """, - id="setup.py", - ), - pytest.param( - "pyproject.toml", - """ - [build-system] - requires = ["flit_core >=2,<4"] - build-backend = "flit_core.buildapi" - - [tool.flit.metadata] - module = "sample_lib" - author = "Vincent Driessen" - author-email = "me@nvie.com" - - requires = ["small-fake-a==0.1", "small-fake-b==0.2"] - - [tool.flit.metadata.requires-extra] - dev = ["small-fake-c==0.3", "small-fake-d==0.4"] - test = ["small-fake-e==0.5", "small-fake-f==0.6"] - """, - id="flit", - ), - pytest.param( - "pyproject.toml", - """ - [build-system] - requires = ["poetry_core>=1.0.0"] - build-backend = "poetry.core.masonry.api" - - [tool.poetry] - name = "sample_lib" - version = "0.1.0" - description = "" - authors = ["Vincent Driessen "] - - [tool.poetry.dependencies] - python = "*" - small-fake-a = "0.1" - small-fake-b = "0.2" - - small-fake-c = "0.3" - small-fake-d = "0.4" - small-fake-e = "0.5" - small-fake-f = "0.6" - - [tool.poetry.extras] - dev = ["small-fake-c", "small-fake-d"] - test = ["small-fake-e", "small-fake-f"] - """, - id="poetry", - ), +METADATA_TEST_CASES = ( + pytest.param( + "setup.cfg", + """ + [metadata] + name = sample_lib + author = Vincent Driessen + author_email = me@nvie.com + + [options] + packages = find: + install_requires = + small-fake-a==0.1 + small-fake-b==0.2 + + [options.extras_require] + dev = + small-fake-c==0.3 + small-fake-d==0.4 + test = + small-fake-e==0.5 + small-fake-f==0.6 + """, + id="setup.cfg", + ), + pytest.param( + "setup.py", + """ + from setuptools import setup + + setup( + name="sample_lib", + version=0.1, + install_requires=["small-fake-a==0.1", "small-fake-b==0.2"], + extras_require={ + "dev": ["small-fake-c==0.3", "small-fake-d==0.4"], + "test": ["small-fake-e==0.5", "small-fake-f==0.6"], + }, + ) + """, + id="setup.py", + ), + pytest.param( + "pyproject.toml", + """ + [build-system] + requires = ["flit_core >=2,<4"] + build-backend = "flit_core.buildapi" + + [tool.flit.metadata] + module = "sample_lib" + author = "Vincent Driessen" + author-email = "me@nvie.com" + + requires = ["small-fake-a==0.1", "small-fake-b==0.2"] + + [tool.flit.metadata.requires-extra] + dev = ["small-fake-c==0.3", "small-fake-d==0.4"] + test = ["small-fake-e==0.5", "small-fake-f==0.6"] + """, + id="flit", + ), + pytest.param( + "pyproject.toml", + """ + [build-system] + requires = ["poetry_core>=1.0.0"] + build-backend = "poetry.core.masonry.api" + + [tool.poetry] + name = "sample_lib" + version = "0.1.0" + description = "" + authors = ["Vincent Driessen "] + + [tool.poetry.dependencies] + python = "*" + small-fake-a = "0.1" + small-fake-b = "0.2" + + small-fake-c = "0.3" + small-fake-d = "0.4" + small-fake-e = "0.5" + small-fake-f = "0.6" + + [tool.poetry.extras] + dev = ["small-fake-c", "small-fake-d"] + test = ["small-fake-e", "small-fake-f"] + """, + id="poetry", ), ) -def test_input_formats(make_package, make_wheel, runner, tmpdir, fname, content): - dists_path = os.path.join(tmpdir, "dists") - pkg = make_package("small-fake-a", version="0.1") - make_wheel(pkg, dists_path) - pkg = make_package("small-fake-b", version="0.2") - make_wheel(pkg, dists_path) - - path = os.path.join(tmpdir, "sample_lib") - os.mkdir(path) - path = os.path.join(tmpdir, "sample_lib", "__init__.py") - with open(path, "w") as stream: - stream.write("'example module'\n__version__ = '1.2.3'") - path = os.path.join(tmpdir, fname) - with open(path, "w") as stream: - stream.write(dedent(content)) - out = runner.invoke(cli, ["-n", "--find-links", dists_path, path]) + +@pytest.mark.network +@pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) +def test_input_formats(fake_dists, runner, make_module, fname, content): + """ + Test different dependency formats as input file. + """ + meta_path = make_module(fname=fname, content=content) + out = runner.invoke(cli, ["-n", "--find-links", fake_dists, meta_path]) assert out.exit_code == 0, out.stderr assert "small-fake-a==0.1" in out.stderr assert "small-fake-b==0.2" in out.stderr @@ -1778,3 +1767,67 @@ def test_input_formats(make_package, make_wheel, runner, tmpdir, fname, content) assert "small-fake-d" not in out.stderr assert "small-fake-e" not in out.stderr assert "small-fake-f" not in out.stderr + assert "extra ==" not in out.stderr + + +@pytest.mark.network +@pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) +def test_one_extra(fake_dists, runner, make_module, fname, content): + """ + Test one `--extra` (dev) passed, other extras (test) must be ignored. + """ + meta_path = make_module(fname=fname, content=content) + out = runner.invoke( + cli, ["-n", "--extra", "dev", "--find-links", fake_dists, meta_path] + ) + assert out.exit_code == 0, out.stderr + assert "small-fake-a==0.1" in out.stderr + assert "small-fake-b==0.2" in out.stderr + assert "small-fake-c==0.3" in out.stderr + assert "small-fake-d==0.4" in out.stderr + assert "small-fake-e" not in out.stderr + assert "small-fake-f" not in out.stderr + assert "extra ==" not in out.stderr + + +@pytest.mark.network +@pytest.mark.parametrize(("fname", "content"), METADATA_TEST_CASES) +def test_multiple_extras(fake_dists, runner, make_module, fname, content): + """ + Test passing multiple `--extra` params. + """ + meta_path = make_module(fname=fname, content=content) + out = runner.invoke( + cli, + [ + "-n", + "--extra", + "dev", + "--extra", + "test", + "--find-links", + fake_dists, + meta_path, + ], + ) + assert out.exit_code == 0, out.stderr + assert "small-fake-a==0.1" in out.stderr + assert "small-fake-b==0.2" in out.stderr + assert "small-fake-c==0.3" in out.stderr + assert "small-fake-d==0.4" in out.stderr + assert "small-fake-e==0.5" in out.stderr + assert "small-fake-f==0.6" in out.stderr + assert "extra ==" not in out.stderr + + +def test_extras_fail_with_requirements_in(runner, tmpdir): + """ + Test that passing `--extra` with `requirements.in` input file fails. + """ + path = os.path.join(tmpdir, "requirements.in") + with open(path, "w") as stream: + stream.write("\n") + out = runner.invoke(cli, ["-n", "--extra", "something", path]) + assert out.exit_code == 2 + exp = "--extra has effect only with setup.py and PEP-517 input formats" + assert exp in out.stderr diff --git a/tests/test_utils.py b/tests/test_utils.py index d6f245c3d..e143a46d4 100644 --- a/tests/test_utils.py +++ b/tests/test_utils.py @@ -9,6 +9,7 @@ from piptools.utils import ( as_tuple, dedup, + drop_extras, flat_map, format_requirement, format_specifier, @@ -367,3 +368,58 @@ def test_lookup_table_from_tuples_with_empty_values(): def test_lookup_table_with_empty_values(): assert lookup_table((), operator.itemgetter(0)) == {} + + +@pytest.mark.parametrize( + ("given", "expected"), + ( + ("", None), + ("extra == 'dev'", None), + ("extra == 'dev' or extra == 'test'", None), + ("os_name == 'nt' and extra == 'dev'", "os_name == 'nt'"), + ("extra == 'dev' and os_name == 'nt'", "os_name == 'nt'"), + ("os_name == 'nt' or extra == 'dev'", "os_name == 'nt'"), + ("extra == 'dev' or os_name == 'nt'", "os_name == 'nt'"), + ("(extra == 'dev') or os_name == 'nt'", "os_name == 'nt'"), + ("os_name == 'nt' and (extra == 'dev' or extra == 'test')", "os_name == 'nt'"), + ("os_name == 'nt' or (extra == 'dev' or extra == 'test')", "os_name == 'nt'"), + ("(extra == 'dev' or extra == 'test') or os_name == 'nt'", "os_name == 'nt'"), + ("(extra == 'dev' or extra == 'test') and os_name == 'nt'", "os_name == 'nt'"), + ( + "os_name == 'nt' or (os_name == 'unix' and extra == 'test')", + "os_name == 'nt' or os_name == 'unix'", + ), + ( + "(os_name == 'unix' and extra == 'test') or os_name == 'nt'", + "os_name == 'unix' or os_name == 'nt'", + ), + ( + "(os_name == 'unix' or extra == 'test') and os_name == 'nt'", + "os_name == 'unix' and os_name == 'nt'", + ), + ( + "(os_name == 'unix' or os_name == 'nt') and extra == 'dev'", + "os_name == 'unix' or os_name == 'nt'", + ), + ( + "(os_name == 'unix' and extra == 'test' or python_version < '3.5')" + " or os_name == 'nt'", + "(os_name == 'unix' or python_version < '3.5') or os_name == 'nt'", + ), + ( + "os_name == 'unix' and extra == 'test' or os_name == 'nt'", + "os_name == 'unix' or os_name == 'nt'", + ), + ( + "os_name == 'unix' or extra == 'test' and os_name == 'nt'", + "os_name == 'unix' or os_name == 'nt'", + ), + ), +) +def test_drop_extras(from_line, given, expected): + ireq = from_line(f"test;{given}") + drop_extras(ireq) + if expected is None: + assert ireq.markers is None + else: + assert str(ireq.markers).replace("'", '"') == expected.replace("'", '"')